def train_AI_with_loaders(input_path, model_save_path):
data = load_data(input_path,
balance=args.balance,
shuffe_data=args.shuffle_data)
stat_dataset, _, _ = create_loader_sets(data, args.train_ratio,
args.validation_ratio,
args.test_ratio, None, False)
stat_loader = torch.utils.data.DataLoader(stat_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=0)
stats = findStats(logging, stat_loader, args.cal_stats)
stat_dataset = None
stat_loader = None
train_dataset, val_dataset, test_dataset = create_loader_sets(
data, args.train_ratio, args.validation_ratio, args.test_ratio, stats,
True)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=0)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=0)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=0)
logging.info("Number of Training Examples : {}".format(len(train_dataset)))
logging.info("Number of Validation Examples : {}".format(len(val_dataset)))
logging.info("Number of Test Examples : {}".format(len(test_dataset)))
logging.info("Creating Model Graph")
model = getattr(sys.modules[__name__], args.model)(config['model_dict'], 5)
logging.info("Model Created successfully")
logging.info("Starting Training")
trainer(logging, model, config['loss_weights'], train_loader, val_loader,
args.num_epochs, args.lr, args.gpu, args.gpu_number,
args.save_model, args.print_after, args.validate_after,
args.save_after)
logging.info("Training Completed")
if (len(val_loader) > 0):
logging.info("Testing on Validation Set")
test_with_loader(logging, model, config['loss_weights'], val_loader,
args.gpu, args.gpu_number)
if (len(test_loader) > 0):
logging.info("Testing on Test Set")
test_with_loader(logging, model, config['loss_weights'], test_loader,
args.gpu, args.gpu_number)
def gen_model():
params,shared_model,shared_grad_buffers,side = yield
while True:
act = np.asarray([0.0,0.0])
agent = trainer(params,shared_model,shared_grad_buffers)
tup = yield (act[0] * 500,act[1] * 500)
total_reward = 0.0
agent.pre_train()
tick = 0
while True:
tick += 1
move_order = (act[0] * 500,act[1] * 500)
tup = yield move_order
tup = parse_tup(side, tup)
print("origin input ", tup ,flush=True)
total_reward += tup[1]
act = get_action(agent.step(tup,None,0.0))
if tup[2]:
break
def train_loop():
model = registry_model.get(
config.model['name'])(num_classes=config.num_class)
model = torch.nn.DataParallel(model)
model = model.to(torch.device('cuda'))
optimizer = Adam(model.parameters(), lr=config.lr)
# step_scheduler = lr_scheduler.StepLR(optimizer, 3, gamma=0.7, last_epoch=-1)
cos_scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,
T_0=1,
T_mult=2,
eta_min=1e-6,
last_epoch=-1)
scheduler = cos_scheduler
criterion = registry_loss.get(config.criterion)()
print(config.criterion)
print(config.lr)
train_tf = transforms.Compose([
transforms.Resize(config.resize),
transforms.RandomHorizontalFlip(0.5),
# transforms.RandomVerticalFlip(0.5),
transforms.ToTensor(),
])
test_tf = transforms.Compose([
transforms.Resize(config.resize),
transforms.ToTensor(),
])
GIC_train_dataset = ImageFolder(root='/home/youliang/datasets/GIC/train',
transform=train_tf)
GIC_train_loader = DataLoader(GIC_train_dataset,
batch_size=config.batch_size,
shuffle=True,
pin_memory=True)
GIC_val_dataset = ImageFolder(root='/home/youliang/datasets/GIC/val',
transform=test_tf)
GIC_val_loader = DataLoader(GIC_val_dataset,
batch_size=config.test_batch_size,
pin_memory=True)
tqdm_length = math.ceil(len(GIC_train_dataset) / config.batch_size)
trainer(model, optimizer, criterion, scheduler, GIC_train_loader,
GIC_val_loader, tqdm_length)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--logs', type=str, default=None, help='logs by tensorboardX')
parser.add_argument('--local_test', type=str2bool, default=False, help='local test verbose')
parser.add_argument('--model_name', type=str, default="dcgan", help='model name for saving')
parser.add_argument('--test', type=str2bool, default=False, help='call tester.py')
parser.add_argument('--use_visdom', type=str2bool, default=False, help='visualization by visdom')
args = parser.parse_args()
parameters.print_params()
if args.test == False:
trainer(args)
else:
tester(args)
def main(data_path, dict_path, save_path, batch_size, reload_, reload_path):
os.environ["THEANO_FLAGS"] = "floatX=float32"
file_names = get_file_list(data_path, ['txt'])
train_sent = load_txt_sent(file_names)
if not os.path.exists(dict_path):
print "Dictionary not found, recreating"
worddict, wordcount = vocab.build_dictionary(train_sent)
print "Built. Saving to: {}".format(dict_path)
vocab.save_dictionary(worddict, wordcount, dict_path)
else:
print "Found dictionary at {}... Loading...".format(dict_path)
worddict = vocab.load_dictionary(dict_path)
print "Beginning Training..."
train.trainer(train_sent, batch_size=batch_size, reload_=reload_, dictionary=dict_path, saveto=save_path, reload_path=reload_path, saveFreq=10000)
def main(params):
# build dataset
train_data = pd.read_csv('./data/train_final.csv')
tokenizer = get_tokenizer('spacy', language='en')
if params.emb_type == "GloVe":
embedding = GloVe(
name=params.emb_data, dim=params.emb_dim
) # use glove embedding with default option(name='840B', dim=300)
elif params.emb_type == "CharNGram":
embedding = CharNGram()
elif params.emb_type == "FastText":
embedding = FastText(name=params.emb_data, dim=params.emb_dim)
else:
print("Wrong embedding type")
exit()
train_data, val_data = train_data[1000:], train_data[:1000]
train_dataset = SentimentDataset(train_data, tokenizer, embedding)
val_dataset = SentimentDataset(val_data, tokenizer, embedding)
train_dataloader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
val_dataloader = DataLoader(dataset=val_dataset,
batch_size=batch_size,
shuffle=False)
model = SentimentClassificationModel(params.emb_dim, params.hidden_dim,
params.dropout).to(device)
crit = nn.CrossEntropyLoss().to(device)
optim = torch.optim.Adam(params=model.parameters(), lr=1e-3)
best_val_acc = 0
early_stop_cnt = 0
epoch = 0
train_loss_list = []
train_acc_list = []
val_acc_list = []
while early_stop_cnt != 5:
loss_list, train_acc = train.trainer(epoch, model, train_dataloader,
crit, optim, device)
val_acc = train.eval(epoch, model, val_dataloader, device, False)
if val_acc > best_val_acc and epoch > 0:
torch.save(model.state_dict(), './model/lstm_best.pt')
best_val_acc = val_acc
early_stop_cnt = 0
early_stop_cnt += 1
epoch += 1
train_loss_list.extend(loss_list)
train_acc_list.append(train_acc)
val_acc_list.append(val_acc)
print("Early stopping condition satisfied")
plotting("train_loss", "steps", "loss", train_loss_list)
plotting("train_accuracy", "epoch", "accuracy", train_acc_list)
plotting('validation_accuracy', "epoch", "accuracy", val_acc_list)
def Determination(self):
seq = self.lineEdit.text()
peo1 = self.lineEdit_2.text()
peo2 = self.lineEdit_3.text()
self.haha = trainer()
ans = self.haha.test(seq, peo1, peo2)
self.lineEdit_4.setText(ans)
def testTrain(self):
global model
global embed_map
main.save_vocab('test_dir', 'test_dir/dict.dat')
self.assertTrue(os.path.exists('test_dir/dict.dat'))
embed_map = tools.load_googlenews_vectors('word2vec.w2v', binary = True)
train.trainer(list(main.load_corpus('test_dir')), saveto = 'test_dir/model', saveFreq = 10, n_words = 10) #you may want to change parameters saveFreq or n_words if you use other test corpus texts
os.rename('test_dir/model.pkl', 'test_dir/model.npz.pkl')
self.assertTrue(os.path.exists('test_dir/model.npz'))
self.assertTrue(os.path.exists('test_dir/model.npz.pkl'))
model = tools.load_model('test_dir/model.npz', 'test_dir/dict.dat', 'word2vec.w2v', embed_map)
X_train, y_train = main.training_set(model, ['test_dir/train.csv'])
self.assertEqual(len(X_train.shape), 2)
self.assertEqual(len(y_train.shape), 1)
self.assertEqual(X_train.shape[0], y_train.shape[0])
self.assertEqual(X_train.shape[1], 4800)
def init_model_and_train(hidden_size, batch_size, train_size, n_epochs, lr, weight_decay,
betas0, betas1, seed):
# Parameters
num_hidden_layers = 1
out_channels = 1
if run_mode == "unfrozen_convolution_relu":
model = supervised_convnet.SupervisedConvNet(filter_size = 3, square_size = 3, \
hidden_size = hidden_size, out_channels = out_channels,
first_activation = "tanh", activation_func = "relu",
num_hidden_layers = num_hidden_layers, seed = seed)
results = train.trainer(model = model, batch_size = batch_size, train_size = train_size, n_epochs = n_epochs, lr = lr,
weight_decay = weight_decay,
betas0 = 1-betas0, betas1 = 1-betas1)
elif run_mode == "frozen_convolution_no_center_relu":
model = frozen.SupervisedConvNet(filter_size = 3, square_size = 3, \
hidden_size = hidden_size, out_channels = out_channels,
center = "omit", first_activation = "tanh",
activation_func = "relu", num_hidden_layers = num_hidden_layers)
results = train.trainer(model = model, batch_size = batch_size, train_size = train_size, n_epochs = n_epochs, lr = lr,
weight_decay = weight_decay,
betas0 = 1-betas0, betas1 = 1-betas1)
elif run_mode == "frozen_convolution_pretrained_relu":
model = frozen.SupervisedConvNet(filter_size = 3, square_size = 3, \
hidden_size = hidden_size, out_channels = out_channels,
center = "pre_trained", first_activation = "tanh",
activation_func = "relu", num_hidden_layers = num_hidden_layers)
results = train.trainer(model = model, batch_size = batch_size, train_size = train_size, n_epochs = n_epochs, lr = lr,
weight_decay = weight_decay,
betas0 = 1-betas0, betas1 = 1-betas1)
elif run_mode == "unfrozen_convolution_3_channels":
out_channels = 3
model = supervised_convnet.SupervisedConvNet(filter_size = 3, square_size = 3,
hidden_size = hidden_size, out_channels = out_channels,
first_activation = "tanh", activation_func = "relu",
num_hidden_layers = num_hidden_layers, seed = seed)
results = train.trainer(model = model, batch_size = batch_size, train_size = train_size, n_epochs = n_epochs, lr = lr,
weight_decay = weight_decay,
betas0 = 1-betas0, betas1 = 1-betas1)
return results
def start_training_thread(user, project, saving_step, train_enc_ids,
train_dec_ids):
trainer = train.trainer()
thread = Thread(target=trainer.train,
args=(user, project, saving_step, train_enc_ids,
train_dec_ids))
thread.start()
trainer_thread.append({
"user": user,
"project": project,
"trainer": trainer
})
def main():
"""Train"""
print("\nPyTorch Version:", torch.__version__)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Device found = {device}")
if device.type == "cuda":
print(" Number of GPUs:", torch.cuda.device_count())
print(" Device properties:", torch.cuda.get_device_properties(0))
print("\nDownload data")
start = time.time()
download_data(date_partitions=[EXECUTION_DATE])
print(f" Time taken = {time.time() - start:.0f} secs")
print("\nTrain model")
params = {
'weights': '',
'cfg': './models/custom_yolov5s.yaml',
'data': 'data.yaml',
'epochs': int(os.getenv("NUM_EPOCHS")),
'batch_size': int(os.getenv("BATCH_SIZE")),
'img_size': [416],
'cache_images': True,
'name': 'yolov5s_results',
}
trainer(set_params(params))
print("\nEvaluate")
run_dir = f"./runs/exp0_{params['name']}/"
log_metrics(run_dir)
print("\nSave artefacts and results")
for fpath in os.listdir(run_dir):
if fpath == "weights":
# Copy best weights
shutil.copy2(run_dir + "weights/best.pt", "/artefact/best.pt")
elif os.path.isfile(run_dir + fpath):
shutil.copy2(run_dir + fpath, "/artefact/" + fpath)
def train_loop():
model = registry_model.get(config.model['name'])()
model = torch.nn.DataParallel(model)
model = model.to(torch.device('cuda'))
print(model)
# optimizer = SGD(model.parameters(), lr=config.lr, momentum=config.momentum, weight_decay=config.weight_decay) #
optimizer = Adam(model.parameters(), lr=config.lr)
# optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)
criterion = registry_loss.get(config.criterion)()
step_scheduler = lr_scheduler.StepLR(optimizer,
8,
gamma=0.1,
last_epoch=-1)
train_data = MyDataset('train')
val_data = MyDataset('val')
tqdm_length = math.ceil(len(train_data) / config.batch_size)
if config.train_keep == -1:
ok_count = len(os.listdir(f"{config.data_path}/train/ok/")) - 1
else:
ok_count = config.train_keep
ng_count = len(os.listdir(f"{config.data_path}/train/ng/"))
weights = [1 / ok_count for i in range(ok_count)
] + [1 / ng_count for i in range(ng_count)]
sampler = WeightedRandomSampler(weights, len(train_data), True)
train_loader = DataLoader(train_data,
batch_size=config.batch_size,
sampler=sampler,
pin_memory=True)
val_loader = DataLoader(val_data,
batch_size=config.batch_size,
shuffle=False,
pin_memory=True)
trainer(model, optimizer, criterion, step_scheduler, train_loader,
val_loader, tqdm_length)
def recog(fname_or_list, train=None):
"""
识别 demo，输入一个或多个视频文件路径，返回识别结果
"""
if train is None:
train = trainer(is_recog=True)
train.saver.restore(train.sess, train.model_fname)
if not isinstance(fname_or_list, list):
fname_or_list = [fname_or_list]
with timer('识别视频') as t:
probs, labels = train.recog(fname_or_list)
delta = t.duration()
is_all_test_set = False
if fname_or_list == train.batch_generator.test_fname_list:
is_all_test_set = True
top_accs = [0 for _ in range(5)]
for i in range(len(probs)):
print('--------------------------------------------------')
fname = fname_or_list[i]
prob = probs[i]
label = labels[i]
cls_name = train.batch_generator.cls_list[label]
str_format = '%s 的预测标签为：%d，预测类别为：%s' % (fname, label, cls_name)
print(str_format)
prob = prob.tolist()
if is_all_test_set:
true_label = train.batch_generator.test_fname_to_label[fname]
print('Top-5 预测概率值分别为：')
top_5_index = np.argsort(prob)[::-1][:5]
for i in range(0, len(top_5_index), 2):
index = top_5_index[i]
if is_all_test_set:
if true_label in top_5_index[:i+1]:
top_accs[i] += 1
str_format = '标签：%d - 类别名：%s - 概率值：%f'
cls_name = train.batch_generator.cls_list[index]
p = prob[index]
print(str_format % (index, cls_name, p))
print('--------------------------------------------------')
print('以上为所有输入视频的预测标签、类别名和预测概率值')
print('总视频数为：%d' % len(fname_or_list))
print('视频总用时：%.3f' % delta)
speed = delta / len(fname_or_list)
print('识别平均速度为：%.3f sec/视频' % speed)
if is_all_test_set:
print('++++++++++++++++++++++++++++++++++++++++++++++++++')
print('当前使用所有测试集作为演示，可计算准确率，准确率如下：')
for i in range(0, len(top_accs), 2):
print('Top-%d准确率为：%.4f' % (i+1, top_accs[i] / len(fname_or_list)))
train.close()
def main():
parser = argparse.ArgumentParser(
description='Pass target style genre to train decoder')
parser.add_argument('-s',
'--style_genre',
help='the name of style corpus',
required='True',
default='localhost')
flag = parser.parse_args()
style_corpus_path = "/media/VSlab3/kuanchen_arxiv/artistic_style_corpora/{}".format(
flag.style_genre)
style_genre = flag.style_genre.split(".")[0]
X = []
with open(style_corpus_path, 'r') as handle:
for line in handle.readlines():
X.append(line.strip())
C = X
if not os.path.isfile("./vocab_save/{}.pkl".format(style_genre)):
print "Get vocabulary..."
worddict, wordcount = vocab.build_dictionary(X)
vocab.save_dictionary(worddict=worddict,
wordcount=wordcount,
loc="vocab_save/{}.pkl".format(style_genre))
else:
pass
savepath = "./logs_{}".format(style_genre)
if not os.path.exists(savepath):
os.mkdir(savepath)
skmodel = skipthoughts.load_model()
train.trainer(X,
C,
skmodel,
dictionary="vocab_save/{}.pkl".format(style_genre),
savepath=savepath,
saveto="model.npz")
def run():
params = Params()
grid_data = get_data(params.data_params)
data_dict = dataset_split(grid_data,
params.data_params['test_ratio'],
params.data_params['val_ratio'])
batch_gens = create_generator(data_dict, params.model_params['TRAJGRU']['batch_params'])
# for batch_idx, (x, y) in enumerate(batch_gens['train'].batch_next()):
# print(batch_idx, x.shape)
trained_model = trainer(batch_gens, **params.model_params['TRAJGRU'])
print('Training finished, saving the model')
model_file = open('results/conv_lstm.pkl', 'wb')
pickle.dump(trained_model, model_file)
(target_name, target_name))
"""
Step 2: Generating style vector for the target text.
"""
print("Generating style vector for the target text...")
nltk.download(
'punkt') # Natural Language Toolkit for skipthoughts encoder.
print("The lenth of X is:")
print len(X)
skip_vector = skipthoughts.encode(skmodel, X)
style_vector = skip_vector.mean(
0
) # 0 indicate that mean method is performed over multiple axes, see numpy.mean document
#np.save('./target_style/%s_style.npy'%target_name, style_vector)
#print("Done! Saved style under ./target_style/ as %s_style.npy"%target_name)
np.save('./%s/%s_style.npy' % (target_name, target_name), style_vector)
print("Done! Saved style under ./%s/ as %s_style.npy" %
(target_name, target_name))
"""
Step 3: Training model for the target text.
"""
print("Training model for the target text...")
C = X
#train.trainer(X, C, skmodel, './target_dict/%s.pkl'%target_name, './target_model/%s.npz'%target_name)
#print("Done! Saved model under ./target_model as %s.npz and %s.npz.pkl"%(target_name, target_name))
train.trainer(X, C, skmodel, './%s/%s.pkl' % (target_name, target_name),
'./%s/%s.npz' % (target_name, target_name))
print("Done! Saved model under ./%s/ as %s.npz and %s.npz.pkl" %
(target_name, target_name, target_name))
except:
results = []
results = []
first_epoch_validate_accuracy_list = []
running_mean = 0.0
for _ in range(500):
model = supervised_convnet.SupervisedConvNet(filter_size = 3, square_size = 3, \
hidden_size = hidden_size, out_channels = out_channels,
first_activation = "tanh", activation_func = "relu",
num_hidden_layers = num_hidden_layers, seed = time.time() + _)
model = model_to_cuda(model)
best_val_acc, param_dict = train.trainer(
model=model,
batch_size=2825,
betas0=1 - 0.14837031829213393,
betas1=1 - 0.00044025104003604366,
lr=0.005938797909346845,
n_epochs=233,
train_size=5000,
weight_decay=0.000119,
use_cuda=use_cuda)
running_mean = running_mean + (best_val_acc - running_mean) / (_ + 1)
print("Running mean is", colored(running_mean, "red"))
results.append(best_val_acc)
conv_params["weight"].append(param_dict["conv1.weight"])
conv_params["bias"].append(param_dict["conv1.bias"])
if (_ % save_loops) == (0):
save_progress(results, conv_params, filename, _)
elif run_mode == "unfrozen_convolution_relu9x9":
"""
"""
out_channels = 1
"""
all_sent = []
for txt_file in flist_txt:
print "Reading file: {}".format(txt_file)
with open(txt_file, 'r') as f:
data = f.read()
sent = data.split('\n')
all_sent += sent
print "File loading complete. Cleaning..."
#all_sent = map(clean_string, all_sent)
return all_sent
if __name__ == "__main__":
os.environ["THEANO_FLAGS"] = "floatX=float32"
file_names = get_file_list(data_path, ['txt'])
train_sent = load_txt_sent(file_names)
if not os.path.exists(dict_path):
print "Dictionary not found, recreating"
worddict, wordcount = vocab.build_dictionary(train_sent)
print "Built. Saving to: {}".format(dict_path)
vocab.save_dictionary(worddict, wordcount, dict_path)
else:
print "Found dictionary at {}... Loading...".format(dict_path)
worddict = vocab.load_dictionary(dict_path)
print "Beginning Training..."
train.trainer(train_sent, n_words=20000, dim=2400, batch_size=128, reload_=False, dictionary=dict_path, saveto=save_path)
# -*- coding: utf-8 -*-
import roc
import sys
sys.path.append('...')
from conf import knn as kn
import train as tr
if __name__ == "__main__":
trainer = tr.trainer()
predict_result = trainer.model.predict(trainer.data.x_test)
#print(predict_result)
for value in kn.CONFIG['data']['parameter']['category_set']:
roc.roc_picture(predict_result, trainer.data.y_test, value)
with open(filename, "rb") as handle:
results = pickle.load(handle)
except:
results = []
results = []
for _ in range(500):
model = supervised_convnet.SupervisedConvNet(filter_size = 3, square_size = 3, \
hidden_size = hidden_size, out_channels = out_channels,
first_activation = "tanh", activation_func = "relu",
num_hidden_layers = num_hidden_layers, seed = time.time() + _)
model = model_to_cuda(model)
best_val_acc, param_dict = train.trainer(
model=model,
batch_size=100,
betas0=1 - 0.0018179320494754046,
betas1=1 - 0.001354073715524798,
lr=0.004388469485690077,
n_epochs=150,
train_size=5000,
weight_decay=0,
use_cuda=use_cuda)
results.append(best_val_acc)
conv_params["weight"].append(param_dict["conv1.weight"])
conv_params["bias"].append(param_dict["conv1.bias"])
if (_ % save_loops) == (0):
save_progress(results, conv_params, filename, _)
elif run_mode == "frozen_convolution_no_center_relu":
"""
"""
filename = "frozen_convolution_no_center_relu.pl"
out_channels = 1
try:
'data': 'coco',
#'cnn': '10crop',
'cnn': 'vgg_attr',
#'dim_image': 4096,
'dim_image': 1000,
'encoder': 'gru',
'dispFreq': 10,
'grad_clip': 2.,
'optimizer': 'adam',
'batch_size': 128,
'dim': 1024,
'dim_word': 300,
'lrate': 0.001,
'validFreq': 300
#'v_norm': 'l2'
}
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('model', choices=['order', 'symmetric'])
parser.add_argument('name')
args = parser.parse_args()
model_params = eval(args.model)
model_params.update(default_params)
name = args.name
train.trainer(name=name, **model_params)
def start_cppSimulator():
time.sleep(0.5)
params,shared_model,shared_grad_buffers = yield
num_iter, canvas = yield
count = num_iter
discount_factor = 1.0
while True:
_engine = cppSimulator(canvas)
count += 1
print("%d simulated game starts!"%count)
dire_act = np.asarray([0.0,0.0])
rad_act = np.asarray([0.0,0.0])
dire_agent = trainer(params,shared_model,shared_grad_buffers)
rad_agent = trainer(params,shared_model,shared_grad_buffers)
shared_model = dire_agent.shared_model
d_tup = _engine.get_state_tup("Dire", 0)
r_tup = _engine.get_state_tup("Radiant", 0)
last_dire_location = hero_location_by_tup(d_tup)
last_rad_location = hero_location_by_tup(r_tup)
r_total_reward = 0.0
d_total_reward = 0.0
#discount_factor -= 0.001
dire_agent.pre_train()
rad_agent.pre_train()
if discount_factor < 0.0:
discount_factor = 0.0
tick = 0
p_dire_act = _engine.predefined_step("Dire",0)
p_rad_act = _engine.predefined_step("Radiant",0)
while _engine.get_time() < param.game_duriation:
tick += 1
d_move_order = (dire_act[0] * 1000,dire_act[1] * 1000)
r_move_order = (rad_act[0] * 1000,rad_act[1] * 1000)
_engine.set_move_order("Dire",0,dire_act[0] * 1000,dire_act[1] * 1000)
_engine.set_move_order("Radiant",0,rad_act[0] * 1000,rad_act[1] * 1000)
_engine.loop()
d_tup = _engine.get_state_tup("Dire", 0)
r_tup = _engine.get_state_tup("Radiant", 0)
if tick % param.tick_per_action != 0 and not(d_tup[2] or r_tup[2]):
continue#for faster training
if canvas != None:
#_engine.draw()
canvas.update_idletasks()
print("origin output ", d_tup , r_tup,flush=True)
r_total_reward += r_tup[1]
d_total_reward += d_tup[1]
#r_tup = (r_tup[0],r_tup[1] + dotproduct(p_rad_act,rad_act,1),r_tup[2])
#d_tup = (d_tup[0],d_tup[1] + dotproduct(p_dire_act,dire_act,1),d_tup[2])
dire_act = get_action(dire_agent.step(d_tup,p_dire_act,0))
rad_act = get_action(rad_agent.step(r_tup,p_rad_act,0))
p_dire_act = _engine.predefined_step("Dire",0)
p_rad_act = _engine.predefined_step("Radiant",0)
#print(d_tup,r_tup)
print("game %d t=%f,r_act=%s,r_reward=%f,d_act=%s,d_reward=%f"\
%(count, _engine.get_time(),str(rad_act),r_tup[1],str(dire_act),d_tup[1]))
last_dire_location = hero_location_by_tup(d_tup)
last_rad_location = hero_location_by_tup(r_tup)
yield
if d_tup[2] or r_tup[2]:
break
print("total reward %f %f"%(r_total_reward, d_total_reward))
rad_agent.fill_memory()
dire_agent.fill_memory()
if count > 0:
for it in range(1):
shared_grad_buffers = rad_agent.shared_grad_buffers
start_t = time.time()
rad_agent.train()
dire_agent.train()
t1 = time.time()
print("trianing x2 : %fs"%(t1 - start_t))
num_iter = num_iter + 1
optimizer.zero_grad()
for n,p in shared_model.named_parameters():
p._grad = Variable(shared_grad_buffers.grads[n+'_grad'])
p.data -= param.lr * p.grad.data
#optimizer.step()
shared_grad_buffers.reset()
print("opt time: %fs"%(time.time() - t1))
torch.save(shared_model.state_dict(),"./model/%d"%int(count))
print('update')
rad_agent.memory.clear()
dire_agent.memory.clear()
import train # training on new dataset train.trainer(data='43', saveto='./data/43/43.npz', batch_size=10)
# coding: utf-8
import vocab
import train
import tools
import numpy as np
with open("../../wikipedia_txt/result_wakati.txt") as f:
fdata = [line.rstrip() for i, line in enumerate(f)]
print '# lines: ', len(fdata)
worddict, wordcount = vocab.build_dictionary(fdata)
vocab.save_dictionary(worddict, wordcount, "word_dict")
print '# vocab: ', len(worddict)
train.trainer(fdata, dictionary="word_dict", saveFreq=100, saveto="model", reload_=True, n_words=40000)
model = tools.load_model()
vectors = tools.encode(model, fdata, use_norm=False)
np.savez('vecs.npz', vectors)
import train
import time
data = 'f30k'
saveto = 'vse/%s' % data
encoder = 'lstm'
if __name__ == "__main__":
begin_time = time.time()
train.trainer(data=data,
dim_image=4096,
lrate=1e-3,
margin=0.2,
encoder=encoder,
max_epochs=100,
batch_size=16,
dim=1000,
dim_word=300,
maxlen_w=150,
dispFreq=10,
validFreq=100,
early_stop=40,
saveto=saveto)
print('Using %.2f s' % (time.time() - begin_time))
import math, transforms3d
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# second to last argument False for nclt True for others
#weight_path = 'success_models/ShopFacade_weights/model_epoch_3.ckpt'
#image_path = '.ShopFacade/'
#fig_name = 'ShopFacade Trajectory.png'
weight_path = 'success_models/KingsCollege/model_epoch_90.ckpt'
image_path = './KingsCollege/'
fig_name = 'KingsCollege Trajectory.png'
trainer = train.trainer(weight_path, image_path, True, True, True)
datasource = gen_data.get_data()
# initialize plot tool
fig = plt.figure(1)
error = np.zeros([len(datasource.images), 3])
for i in range(len(datasource.images)):
np_image = datasource.images[i]
feed = {
tf.get_default_graph().get_tensor_by_name('Placeholder:0'):
np.expand_dims(np_image, axis=0)
}
# ground truth x y z
def train_AI(input_path, model_save_path):
data = load_data(input_path,
balance=args.balance,
shuffe_data=args.shuffle_data)
batched_data_X1, batched_data_X2, batched_data_X3, batched_data_X4, batched_data_Y = create_batches(
data, args.batch_size)
train_data_X1, validation_data_X1, test_data_X1 = create_sets(
batched_data_X1, args.train_ratio, args.validation_ratio,
args.test_ratio)
train_data_X2, validation_data_X2, test_data_X2 = create_sets(
batched_data_X2, args.train_ratio, args.validation_ratio,
args.test_ratio)
train_data_X3, validation_data_X3, test_data_X3 = create_sets(
batched_data_X3, args.train_ratio, args.validation_ratio,
args.test_ratio)
train_data_X4, validation_data_X4, test_data_X4 = create_sets(
batched_data_X4, args.train_ratio, args.validation_ratio,
args.test_ratio)
stats = {
"X1": {
"mean": np.mean(train_data_X1 / 255, axis=(0, 2, 3)),
"std": np.std(train_data_X1 / 255, axis=(0, 2, 3))
},
"X2": {
"mean": np.mean(train_data_X2 / 255, axis=(0, 2, 3)),
"std": np.std(train_data_X2 / 255, axis=(0, 2, 3))
},
"X3": {
"mean": np.mean(train_data_X3 / 255, axis=(0, 2, 3)),
"std": np.std(train_data_X3 / 255, axis=(0, 2, 3))
}
}
print(stats)
train_data_X = [train_data_X1, train_data_X2, train_data_X3, train_data_X4]
validation_data_X = [
validation_data_X1, validation_data_X2, validation_data_X3,
validation_data_X4
]
test_data_X = [test_data_X1, test_data_X2, test_data_X3, test_data_X4]
train_data_Y, validation_data_Y, test_data_Y = create_sets(
batched_data_Y, args.train_ratio, args.validation_ratio,
args.test_ratio)
logging.info("Number of Training Examples : {}".format(
train_data_X1.shape[0] * train_data_X1.shape[1]))
logging.info("Number of Validation Examples : {}".format(
validation_data_X1.shape[0] * validation_data_X1.shape[1]))
logging.info("Number of Test Examples : {}".format(test_data_X1.shape[0] *
test_data_X1.shape[1]))
logging.info("Creating Model Graph")
model = getattr(sys.modules[__name__], args.model)(config['model_dict'], 5)
logging.info("Model Created successfully")
logging.info("Starting Training")
trainer()
train(logging, model, config['loss_weights'], train_data_X, train_data_Y,
validation_data_X, validation_data_Y, args.num_epochs, args.lr,
args.gpu, args.gpu_number, args.save_model, args.print_after,
args.validate_after, args.save_after)
logging.info("Training Completed")
if (validation_data_X.shape[0] * validation_data_X.shape[1] > 0):
logging.info("Testing on Validation Set")
test(logging, model, config['loss_weights'], validation_data_X,
validation_data_Y, args.gpu)
if (test_data_X.shape[0] * test_data_X.shape[1] > 0):
logging.info("Testing on Test Set")
test(logging, model, config['loss_weights'], test_data_X, test_data_Y,
args.gpu, args.gpu_number)
default_params = {
'max_epochs': 100,
'data': 'coco',
'cnn': '10crop',
'dim_image': 4096,
'encoder': 'gru',
'dispFreq': 10,
'grad_clip': 2.,
'optimizer': 'adam',
'batch_size': 128,
'dim': 1024,
'dim_word': 300,
'lrate': 0.001,
'validFreq': 300
}
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('model', choices=['order', 'symmetric'])
args = parser.parse_args()
model_params = eval(args.model)
model_params.update(default_params)
name = args.model
train.trainer(name=name, **model_params)
# -*- coding: utf-8 -*-
import get_data as gt
import train
if __name__ == "__main__":
data = gt.data() # 得到数据
model = train.trainer(data.all_data) # 训练模型
result_model = model.model # 聚类结果
print(result_model.predict(data.all_data))
sent = data.split('\n')
all_sent += sent
print "File loading complete. Cleaning..."
#all_sent = map(clean_string, all_sent)
return all_sent
if __name__ == "__main__":
os.environ["THEANO_FLAGS"] = "floatX=float32"
file_names = get_file_list(data_path, ['txt'])
train_sent = load_txt_sent(file_names)
if not os.path.exists(dict_path):
print "Dictionary not found, recreating"
worddict, wordcount = vocab.build_dictionary(train_sent)
print "Built. Saving to: {}".format(dict_path)
vocab.save_dictionary(worddict, wordcount, dict_path)
else:
print "Found dictionary at {}... Loading...".format(dict_path)
worddict = vocab.load_dictionary(dict_path)
print "Beginning Training..."
train.trainer(train_sent,
batch_size=64,
reload_=True,
reload_path=reload_path,
dictionary=dict_path,
saveto=save_path,
saveFreq=10000)
def main(args):
logging.info("Parameters:")
for arg in vars(args):
logging.info(arg.rjust(15) + " : " + str(getattr(args, arg)))
# create dataset
if not args.test_only:
dataset_Train = SoccerNetReplayClips(
path=args.SoccerNet_path,
features=args.features,
split="train",
framerate=args.framerate,
chunk_size=args.chunk_size * args.framerate,
receptive_field=args.receptive_field * args.framerate,
chunks_per_epoch=args.chunks_per_epoch,
loop=args.loop)
dataset_Valid = SoccerNetReplayClips(
path=args.SoccerNet_path,
features=args.features,
split="valid",
framerate=args.framerate,
chunk_size=args.chunk_size * args.framerate,
receptive_field=args.receptive_field * args.framerate,
chunks_per_epoch=args.chunks_per_epoch,
loop=args.loop)
dataset_Valid_metric = SoccerNetReplayClipsTesting(
path=args.SoccerNet_path,
features=args.features,
split="valid",
framerate=args.framerate,
chunk_size=args.chunk_size * args.framerate,
receptive_field=args.receptive_field * args.framerate)
dataset_Test = SoccerNetReplayClipsTesting(
path=args.SoccerNet_path,
features=args.features,
split="test",
framerate=args.framerate,
chunk_size=args.chunk_size * args.framerate,
receptive_field=args.receptive_field * args.framerate)
# create model
model = Model(weights=args.load_weights,
input_size=args.num_features,
chunk_size=args.chunk_size * args.framerate,
dim_capsule=args.dim_capsule,
receptive_field=args.receptive_field * args.framerate,
framerate=args.framerate).cuda()
logging.info(model)
total_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
parameters_per_layer = [
p.numel() for p in model.parameters() if p.requires_grad
]
logging.info("Total number of parameters: " + str(total_params))
# create dataloader
if not args.test_only:
train_loader = torch.utils.data.DataLoader(
dataset_Train,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.max_num_worker,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(
dataset_Valid,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.max_num_worker,
pin_memory=True)
val_metric_loader = torch.utils.data.DataLoader(dataset_Valid_metric,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(dataset_Test,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
# training parameters
if not args.test_only:
criterion_segmentation = SymmetricContextAwareLoss(
K=K_MATRIX * args.K_multiplier * args.framerate,
receptive_field=args.receptive_field * args.framerate)
criterion_spotting = ReplayGroundingSpottingLoss(
lambda_coord=args.lambda_coord, lambda_noobj=args.lambda_noobj)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.LR,
betas=(0.9, 0.999),
eps=1e-07,
weight_decay=0,
amsgrad=False)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, 'min', verbose=True, patience=args.patience)
# start training
trainer(train_loader,
val_loader,
val_metric_loader,
test_loader,
model,
optimizer,
scheduler, [criterion_segmentation, criterion_spotting],
[args.loss_weight_segmentation, args.loss_weight_detection],
model_name=args.model_name,
max_epochs=args.max_epochs,
evaluation_frequency=args.evaluation_frequency,
annotation_path=args.SoccerNet_path,
detection_path=args.detection_path,
save_results=args.save_results)
# For the best model only
checkpoint = torch.load(
s.path.join("models", args.model_name, "model.pth.tar"))
model.load_state_dict(checkpoint['state_dict'])
average_mAP = test(test_loader,
model=model,
model_name=args.model_name,
split='test',
annotation_path=args.SoccerNet_path,
detection_path=args.detection_path,
save_results=args.save_results)
logging.info("Best Performance at end of training " + str(average_mAP))
return average_mAP
max_w = 10
saveF = 1000
batch = 128
clen = 6
if not reload_:
# load the data and put in list
f = open(data_path, 'r')
X = f.read().splitlines()
# preprocess
X = preprocess.prepareentitylist(X, stop_path, clen)
# store for future
f = open(proc_data_path,'w')
for item in X:
f.write('%s\n' % item)
else:
f = open(proc_data_path,'r')
X = f.read().splitlines()
# subset
X = X[:N]
# build dictionary
worddict, wordcount = vocab.build_dictionary(X)
vocab.save_dictionary(worddict, wordcount, dict_path)
# train
train.trainer(X, saveto=out_path, dictionary=dict_path, saveFreq=saveF, max_epochs=max_e, dispFreq=dispF, maxlen_w=max_w, batch_size=batch)
# Keep only the arguments for train function
train_arg = arg_dict.copy()
del train_arg['dataset_name']
print('Train arguments collected from launcher:')
for k, v in train_arg.items():
print('{:>26}: {}'.format(k, v))
sys.stdout.flush()
# Parameters for the results of the experiment
params = {}
# TRAIN
t_start = time.time()
solution = trainer(**train_arg)
params['time_computing'] = str(timedelta(seconds=time.time() - t_start))
for k, v in solution.items():
params[k] = v
# TEST
n_fold = 1
model = load_model(args.save_dir, args.model_name, best=True)
print('VALIDATION:')
if n_fold < 2:
print(
'No need to compute n-fold validation, using training results on one fold'
)
else:
print('Computing', n_fold, '-fold validation')
params['best_val_res'], params['best_val_score'] = ranking_eval_Nfold(
def main():
args = parameters.get_parser()
print(args.config)
Experiment = trainer(args=args, time_stp=time_stp)
Experiment.train()
