def _worker_loop(dataset, index_queue, data_queue, collate_fn, seed, init_fn, worker_id):
global _use_shared_memory
_use_shared_memory = True
# Intialize C side signal handlers for SIGBUS and SIGSEGV. Python signal
# module's handlers are executed after Python returns from C low-level
# handlers, likely when the same fatal signal happened again already.
# https://docs.python.org/3/library/signal.html Sec. 18.8.1.1
_set_worker_signal_handlers()
torch.set_num_threads(1)
torch.manual_seed(seed)
np.random.seed(seed)
if init_fn is not None:
init_fn(worker_id)
while True:
r = index_queue.get()
if r is None:
break
idx, batch_indices = r
try:
samples = collate_fn([dataset[i] for i in batch_indices])
except Exception:
data_queue.put((idx, ExceptionWrapper(sys.exc_info())))
else:
data_queue.put((idx, samples))
def __init__(self, registry, ob_space, action_space, config,
name="local", summarize=True):
self.registry = registry
self.local_steps = 0
self.config = config
self.summarize = summarize
self._setup_graph(ob_space, action_space)
torch.set_num_threads(2)
self.lock = Lock()
def _worker_loop(dataset, index_queue, data_queue, collate_fn):
global _use_shared_memory
_use_shared_memory = True
torch.set_num_threads(1)
while True:
r = index_queue.get()
if r is None:
data_queue.put(None)
break
idx, batch_indices = r
try:
samples = collate_fn([dataset[i] for i in batch_indices])
except Exception:
data_queue.put((idx, ExceptionWrapper(sys.exc_info())))
else:
data_queue.put((idx, samples))
def main(depth=2, width=512, nb_epoch=30):
prefer_gpu()
torch.set_num_threads(1)
train_data, dev_data, _ = datasets.mnist()
train_X, train_y = Model.ops.unzip(train_data)
dev_X, dev_y = Model.ops.unzip(dev_data)
dev_y = to_categorical(dev_y)
model = PyTorchWrapper(
PyTorchFeedForward(
depth=depth,
width=width,
input_size=train_X.shape[1],
output_size=dev_y.shape[1],
)
)
with model.begin_training(train_X, train_y, L2=1e-6) as (trainer, optimizer):
epoch_loss = [0.0]
def report_progress():
# with model.use_params(optimizer.averages):
print(epoch_loss[-1], model.evaluate(dev_X, dev_y), trainer.dropout)
epoch_loss.append(0.0)
trainer.each_epoch.append(report_progress)
trainer.nb_epoch = nb_epoch
trainer.dropout = 0.3
trainer.batch_size = 128
trainer.dropout_decay = 0.0
train_X = model.ops.asarray(train_X, dtype="float32")
y_onehot = to_categorical(train_y)
for X, y in trainer.iterate(train_X, y_onehot):
yh, backprop = model.begin_update(X, drop=trainer.dropout)
loss = ((yh - y) ** 2.0).sum() / y.shape[0]
backprop(yh - y, optimizer)
epoch_loss[-1] += loss
with model.use_params(optimizer.averages):
print("Avg dev.: %.3f" % model.evaluate(dev_X, dev_y))
with open("out.pickle", "wb") as file_:
pickle.dump(model, file_, -1)
def _worker_loop(dataset, index_queue, data_queue, collate_fn):
'''As torch.utils.data.dataloader._worker_loop but works on
full batches instead of iterating through the batch.
'''
global _use_shared_memory
_use_shared_memory = True
torch.set_num_threads(1)
while True:
r = index_queue.get()
if r is None:
data_queue.put(None)
break
idx, batch_indices = r
try:
# samples = collate_fn([dataset[i] for i in batch_indices])
samples = collate_fn(dataset[batch_indices])
except Exception:
data_queue.put((idx, ExceptionWrapper(sys.exc_info())))
else:
data_queue.put((idx, samples))
from joblib import Parallel, delayed
import util
import torch
import torch as T
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from collections import OrderedDict
from config_reader import config_reader
from scipy.ndimage.filters import gaussian_filter
#parser = argparse.ArgumentParser()
#parser.add_argument('--t7_file', required=True)
#parser.add_argument('--pth_file', required=True)
#args = parser.parse_args()
torch.set_num_threads(torch.get_num_threads())
weight_name = './model/pose_model.pth'
blocks = {}
# find connection in the specified sequence, center 29 is in the position 15
limbSeq = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], \
[10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], \
[1,16], [16,18], [3,17], [6,18]]
# the middle joints heatmap correpondence
mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], \
[23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], \
[55,56], [37,38], [45,46]]
# visualize
def main():
args = get_config()
# cuda
if args.cuda and torch.cuda.is_available():
device = torch.device("cuda:0")
torch.set_num_threads(1)
else:
device = torch.device("cpu")
torch.set_num_threads(args.n_training_threads)
run_dir = Path(args.model_dir) / ("run" + str(args.seed)) / 'eval'
if os.path.exists(run_dir):
shutil.rmtree(run_dir)
os.mkdir(run_dir)
log_dir = run_dir / 'logs'
os.makedirs(str(log_dir))
logger = SummaryWriter(str(log_dir))
gifs_dir = run_dir / 'gifs'
os.makedirs(str(gifs_dir))
num_agents = args.num_agents
# actor_critic = torch.load('/home/chenjy/curriculum/results/MPE/simple_spread/check/run10/models/agent_model.pt')['model'].to(device)
actor_critic = torch.load(
'/home/tsing73/curriculum/results/MPE/simple_spread/homework/run4/models/agent_model.pt'
)['model'].to(device)
# filename = '/home/tsing73/curriculum/'
# for name, para in zip(actor_critic.actor_base.state_dict(),actor_critic.actor_base.parameters()):
# pdb.set_trace()
# a = para.transpose(0,1).reshape(1,-1)
# np.savetxt(filename+ name + '.txt',np.array(a.to('cpu').detach()),delimiter=',\n')
# pdb.set_trace()
actor_critic.agents_num = 2
actor_critic.boxes_num = 2
num_agents = 2
num_boxes = 2
all_frames = []
cover_rate = 0
random.seed(1)
np.random.seed(1)
# load files
dir_path = '/home/chenjy/curriculum/diversified_left/' + ('archive_' +
str(89))
if os.path.exists(dir_path):
with open(dir_path, 'r') as fp:
archive = fp.readlines()
for i in range(len(archive)):
archive[i] = np.array(archive[i][1:-2].split(), dtype=float)
starts = produce_good_case_grid_pb(500, [-0.6, 0.6, -0.6, 0.6], num_agents,
num_boxes)
for eval_episode in range(args.eval_episodes):
print(eval_episode)
eval_env = MPEEnv(args)
if args.save_gifs:
image = eval_env.render('rgb_array', close=False)[0]
all_frames.append(image)
# eval_obs, _ = eval_env.reset(num_agents,num_boxes)
eval_obs, _ = eval_env.reset(num_agents)
# eval_obs = eval_env.new_starts_obs(start,num_agents)
# eval_obs = eval_env.new_starts_obs_pb(starts[eval_episode],num_agents,num_boxes)
eval_obs = np.array(eval_obs)
eval_share_obs = eval_obs.reshape(1, -1)
eval_recurrent_hidden_states = np.zeros(
(num_agents, args.hidden_size)).astype(np.float32)
eval_recurrent_hidden_states_critic = np.zeros(
(num_agents, args.hidden_size)).astype(np.float32)
eval_masks = np.ones((num_agents, 1)).astype(np.float32)
step_cover_rate = np.zeros(shape=(args.episode_length))
for step in range(args.episode_length):
calc_start = time.time()
eval_actions = []
for agent_id in range(num_agents):
if args.share_policy:
actor_critic.eval()
_, action, _, recurrent_hidden_states, recurrent_hidden_states_critic = actor_critic.act(
agent_id,
torch.FloatTensor(eval_share_obs),
torch.FloatTensor(eval_obs[agent_id].reshape(1, -1)),
torch.FloatTensor(
eval_recurrent_hidden_states[agent_id]),
torch.FloatTensor(
eval_recurrent_hidden_states_critic[agent_id]),
torch.FloatTensor(eval_masks[agent_id]),
None,
deterministic=True)
else:
actor_critic[agent_id].eval()
_, action, _, recurrent_hidden_states, recurrent_hidden_states_critic = actor_critic[
agent_id].act(
agent_id,
torch.FloatTensor(eval_share_obs),
torch.FloatTensor(eval_obs[agent_id]),
torch.FloatTensor(
eval_recurrent_hidden_states[agent_id]),
torch.FloatTensor(
eval_recurrent_hidden_states_critic[agent_id]),
torch.FloatTensor(eval_masks[agent_id]),
None,
deterministic=True)
eval_actions.append(action.detach().cpu().numpy())
eval_recurrent_hidden_states[
agent_id] = recurrent_hidden_states.detach().cpu().numpy()
eval_recurrent_hidden_states_critic[
agent_id] = recurrent_hidden_states_critic.detach().cpu(
).numpy()
# rearrange action
eval_actions_env = []
for agent_id in range(num_agents):
one_hot_action = np.zeros(eval_env.action_space[0].n)
one_hot_action[eval_actions[agent_id][0]] = 1
eval_actions_env.append(one_hot_action)
pdb.set_trace()
# Obser reward and next obs
eval_obs, eval_rewards, eval_dones, eval_infos, _ = eval_env.step(
eval_actions_env)
step_cover_rate[step] = eval_infos[0]['cover_rate']
eval_obs = np.array(eval_obs)
eval_share_obs = eval_obs.reshape(1, -1)
if args.save_gifs:
image = eval_env.render('rgb_array', close=False)[0]
all_frames.append(image)
calc_end = time.time()
elapsed = calc_end - calc_start
if elapsed < args.ifi:
time.sleep(ifi - elapsed)
print('cover_rate: ', np.mean(step_cover_rate[-5:]))
cover_rate += np.mean(step_cover_rate[-5:])
if args.save_gifs:
gif_num = 0
imageio.mimsave(str(gifs_dir / args.scenario_name) +
'_%i.gif' % gif_num,
all_frames,
duration=args.ifi)
# if save_gifs:
# gif_num = 0
# while os.path.exists('./gifs/' + model_dir + '/%i_%i.gif' % (gif_num, ep_i)):
# gif_num += 1
# imageio.mimsave('./gifs/' + model_dir + '/%i_%i.gif' % (gif_num, ep_i),
# frames, duration=ifi)
print('average_cover_rate: ', cover_rate / args.eval_episodes)
eval_env.close()
def on_validation_epoch_start(self):
self.n_threads = torch.get_num_threads()
torch.set_num_threads(1)
self.coco_eval = self._get_coco_eval()
from mjrl.policies.gaussian_linear_lpg_ftw import LinearPolicyLPGFTW
from mjrl.baselines.mlp_baseline import MLPBaseline
from mjrl.algos.npg_cg_ftw import NPGFTW
from mjrl.utils.train_agent import train_agent
import time as timer
import numpy as np
import gym
import pickle
import torch
import os
from mjrl.utils.make_train_plots import make_multitask_train_plots, make_multitask_test_plots
SEED = 50 # initial value, 10 will be added for every iteration
job_name_mtl = 'results/walker_mtl_bodyparts_exp'
job_name_lpgftw = 'results/walker_lpgftw_bodyparts_exp'
torch.set_num_threads(5)
# MTL policy
# ==================================
num_tasks = 50
num_seeds = 5
num_cpu = 5
f = open(job_name_mtl+'/env_factors.pickle', 'rb')
size_factors_list = pickle.load(f)
f.close()
f = open(job_name_mtl+'/env_ids.pickle','rb')
env_ids = pickle.load(f)
f.close()
e_unshuffled = {}
def run(proc_id, n_gpus, n_cpus, args, devices, dataset, split, queue=None):
dev_id = devices[proc_id] if devices[proc_id] != 'cpu' else -1
g, node_feats, num_of_ntype, num_classes, num_rels, target_idx, \
train_idx, val_idx, test_idx, labels = dataset
if split is not None:
train_seed, val_seed, test_seed = split
train_idx = train_idx[train_seed]
val_idx = val_idx[val_seed]
test_idx = test_idx[test_seed]
fanouts = [int(fanout) for fanout in args.fanout.split(',')]
node_tids = g.ndata[dgl.NTYPE]
sampler = NeighborSampler(g, target_idx, fanouts)
loader = DataLoader(dataset=train_idx.numpy(),
batch_size=args.batch_size,
collate_fn=sampler.sample_blocks,
shuffle=True,
num_workers=args.num_workers)
# validation sampler
val_sampler = NeighborSampler(g, target_idx, fanouts)
val_loader = DataLoader(dataset=val_idx.numpy(),
batch_size=args.batch_size,
collate_fn=val_sampler.sample_blocks,
shuffle=False,
num_workers=args.num_workers)
# test sampler
test_sampler = NeighborSampler(g, target_idx, [None] * args.n_layers)
test_loader = DataLoader(dataset=test_idx.numpy(),
batch_size=args.eval_batch_size,
collate_fn=test_sampler.sample_blocks,
shuffle=False,
num_workers=args.num_workers)
world_size = n_gpus
if n_gpus > 1:
dist_init_method = 'tcp://{master_ip}:{master_port}'.format(
master_ip='127.0.0.1', master_port='12345')
backend = 'nccl'
# using sparse embedding or usig mix_cpu_gpu model (embedding model can not be stored in GPU)
if args.dgl_sparse is False:
backend = 'gloo'
print("backend using {}".format(backend))
th.distributed.init_process_group(backend=backend,
init_method=dist_init_method,
world_size=world_size,
rank=dev_id)
# node features
# None for one-hot feature, if not none, it should be the feature tensor.
#
embed_layer = RelGraphEmbedLayer(dev_id,
g.number_of_nodes(),
node_tids,
num_of_ntype,
node_feats,
args.n_hidden,
dgl_sparse=args.dgl_sparse)
# create model
# all model params are in device.
model = EntityClassify(dev_id,
g.number_of_nodes(),
args.n_hidden,
num_classes,
num_rels,
num_bases=args.n_bases,
num_hidden_layers=args.n_layers - 2,
dropout=args.dropout,
use_self_loop=args.use_self_loop,
low_mem=args.low_mem,
layer_norm=args.layer_norm)
if dev_id >= 0 and n_gpus == 1:
th.cuda.set_device(dev_id)
labels = labels.to(dev_id)
model.cuda(dev_id)
# with dgl_sparse emb, only node embedding is not in GPU
if args.dgl_sparse:
embed_layer.cuda(dev_id)
if n_gpus > 1:
labels = labels.to(dev_id)
model.cuda(dev_id)
model = DistributedDataParallel(model, device_ids=[dev_id], output_device=dev_id)
if args.dgl_sparse:
embed_layer.cuda(dev_id)
if len(list(embed_layer.parameters())) > 0:
embed_layer = DistributedDataParallel(embed_layer, device_ids=[dev_id], output_device=dev_id)
else:
if len(list(embed_layer.parameters())) > 0:
embed_layer = DistributedDataParallel(embed_layer, device_ids=None, output_device=None)
# optimizer
dense_params = list(model.parameters())
if args.node_feats:
if n_gpus > 1:
dense_params += list(embed_layer.module.embeds.parameters())
else:
dense_params += list(embed_layer.embeds.parameters())
optimizer = th.optim.Adam(dense_params, lr=args.lr, weight_decay=args.l2norm)
if args.dgl_sparse:
all_params = list(model.parameters()) + list(embed_layer.parameters())
optimizer = th.optim.Adam(all_params, lr=args.lr, weight_decay=args.l2norm)
if n_gpus > 1 and isinstance(embed_layer, DistributedDataParallel):
dgl_emb = embed_layer.module.dgl_emb
else:
dgl_emb = embed_layer.dgl_emb
emb_optimizer = dgl.optim.SparseAdam(params=dgl_emb, lr=args.sparse_lr, eps=1e-8) if len(dgl_emb) > 0 else None
else:
if n_gpus > 1:
embs = list(embed_layer.module.node_embeds.parameters())
else:
embs = list(embed_layer.node_embeds.parameters())
emb_optimizer = th.optim.SparseAdam(embs, lr=args.sparse_lr) if len(embs) > 0 else None
# training loop
print("start training...")
forward_time = []
backward_time = []
train_time = 0
validation_time = 0
test_time = 0
last_val_acc = 0.0
if n_gpus > 1 and n_cpus - args.num_workers > 0:
th.set_num_threads(n_cpus-args.num_workers)
for epoch in range(args.n_epochs):
tstart = time.time()
model.train()
embed_layer.train()
for i, sample_data in enumerate(loader):
seeds, blocks = sample_data
t0 = time.time()
feats = embed_layer(blocks[0].srcdata[dgl.NID],
blocks[0].srcdata['ntype'],
blocks[0].srcdata['type_id'],
node_feats)
logits = model(blocks, feats)
loss = F.cross_entropy(logits, labels[seeds])
t1 = time.time()
optimizer.zero_grad()
if emb_optimizer is not None:
emb_optimizer.zero_grad()
loss.backward()
if emb_optimizer is not None:
emb_optimizer.step()
optimizer.step()
t2 = time.time()
forward_time.append(t1 - t0)
backward_time.append(t2 - t1)
train_acc = th.sum(logits.argmax(dim=1) == labels[seeds]).item() / len(seeds)
if i % 100 and proc_id == 0:
print("Train Accuracy: {:.4f} | Train Loss: {:.4f}".
format(train_acc, loss.item()))
gc.collect()
print("Epoch {:05d}:{:05d} | Train Forward Time(s) {:.4f} | Backward Time(s) {:.4f}".
format(epoch, args.n_epochs, forward_time[-1], backward_time[-1]))
tend = time.time()
train_time += (tend - tstart)
def collect_eval():
eval_logits = []
eval_seeds = []
for i in range(n_gpus):
log = queue.get()
eval_l, eval_s = log
eval_logits.append(eval_l)
eval_seeds.append(eval_s)
eval_logits = th.cat(eval_logits)
eval_seeds = th.cat(eval_seeds)
eval_loss = F.cross_entropy(eval_logits, labels[eval_seeds].cpu()).item()
eval_acc = th.sum(eval_logits.argmax(dim=1) == labels[eval_seeds].cpu()).item() / len(eval_seeds)
return eval_loss, eval_acc
vstart = time.time()
if (queue is not None) or (proc_id == 0):
val_logits, val_seeds = evaluate(model, embed_layer, val_loader, node_feats)
if queue is not None:
queue.put((val_logits, val_seeds))
# gather evaluation result from multiple processes
if proc_id == 0:
val_loss, val_acc = collect_eval() if queue is not None else \
(F.cross_entropy(val_logits, labels[val_seeds].cpu()).item(), \
th.sum(val_logits.argmax(dim=1) == labels[val_seeds].cpu()).item() / len(val_seeds))
do_test = val_acc > last_val_acc
last_val_acc = val_acc
print("Validation Accuracy: {:.4f} | Validation loss: {:.4f}".
format(val_acc, val_loss))
if n_gpus > 1:
th.distributed.barrier()
if proc_id == 0:
for i in range(1, n_gpus):
queue.put(do_test)
else:
do_test = queue.get()
vend = time.time()
validation_time += (vend - vstart)
if (epoch + 1) > (args.n_epochs / 2) and do_test:
tstart = time.time()
if (queue is not None) or (proc_id == 0):
test_logits, test_seeds = evaluate(model, embed_layer, test_loader, node_feats)
if queue is not None:
queue.put((test_logits, test_seeds))
# gather evaluation result from multiple processes
if proc_id == 0:
test_loss, test_acc = collect_eval() if queue is not None else \
(F.cross_entropy(test_logits, labels[test_seeds].cpu()).item(), \
th.sum(test_logits.argmax(dim=1) == labels[test_seeds].cpu()).item() / len(test_seeds))
print("Test Accuracy: {:.4f} | Test loss: {:.4f}".format(test_acc, test_loss))
print()
tend = time.time()
test_time += (tend-tstart)
# sync for test
if n_gpus > 1:
th.distributed.barrier()
print("{}/{} Mean forward time: {:4f}".format(proc_id, n_gpus,
np.mean(forward_time[len(forward_time) // 4:])))
print("{}/{} Mean backward time: {:4f}".format(proc_id, n_gpus,
np.mean(backward_time[len(backward_time) // 4:])))
if proc_id == 0:
print("Test Accuracy: {:.4f} | Test loss: {:.4f}".format(test_acc, test_loss))
print("Train {}s, valid {}s, test {}s".format(train_time, validation_time, test_time))
return data,labels.long()
class Net(nn.Module):
def __init__(self,inputSize,hideSize,outputSize):
super(Net,self).__init__()
self.lstm = lstmTest.LstmCell(inputSize,hideSize)
self.fc = nn.Linear(hideSize,outputSize)
def forward(self,x,h,c):
hideState,cellState = self.lstm(x,h,c)
output = self.fc(hideState)
return output,hideState,cellState
if __name__ == '__main__':
t.set_num_threads(8)
device = t.device("cuda:0" if t.cuda.is_available() else "cpu")
net = Net(inputSize,hideSize,outputSize).to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(),lr=0.001)
#加载训练结果
#net.load_state_dict(t.load('save/net2.pkl'))
#ptimizer.load_state_dict(t.load('save/optimizer2.pkl'))
#生成训练集
trainInput, supposedOutputs = datasetGenerator(T,trainSize)
trainInput = trainInput.to(device)
supposedOutputs = supposedOutputs.to(device)
#生成测试集
testInputs,testOutputs = datasetGenerator(T,testSize)
testInputs = testInputs.to(device)
testOutputs = testOutputs.to(device)
def test_reasoning_v6():
torch.set_num_threads(multiprocessing.cpu_count())
embedding_size = 50
torch.manual_seed(0)
rs = np.random.RandomState(0)
triples = [('a', 'p', 'b'), ('b', 'q', 'c'), ('c', 'p', 'd'),
('d', 'q', 'e'), ('e', 'p', 'f'), ('f', 'q', 'g'),
('g', 'p', 'h'), ('h', 'q', 'i'), ('i', 'p', 'l'),
('l', 'q', 'm'), ('m', 'p', 'n'), ('n', 'q', 'o'),
('o', 'p', 'p'), ('p', 'q', 'q'), ('q', 'p', 'r'),
('r', 'q', 's'), ('s', 'p', 't'), ('t', 'q', 'u'),
('u', 'p', 'v'), ('v', 'q', 'w'), ('x', 'r', 'y'),
('x', 's', 'y')]
entity_lst = sorted({e
for (e, _, _) in triples}
| {e
for (_, _, e) in triples})
predicate_lst = sorted({p for (_, p, _) in triples})
nb_entities = len(entity_lst)
nb_predicates = len(predicate_lst)
entity_to_index = {e: i for i, e in enumerate(entity_lst)}
predicate_to_index = {p: i for i, p in enumerate(predicate_lst)}
for st in ['min', 'concat']:
with torch.no_grad():
kernel = GaussianKernel()
entity_embeddings = nn.Embedding(nb_entities,
embedding_size * 2,
sparse=True)
predicate_embeddings = nn.Embedding(nb_predicates,
embedding_size * 2,
sparse=True)
fact_rel = torch.LongTensor(
np.array([predicate_to_index[p] for (_, p, _) in triples]))
fact_arg1 = torch.LongTensor(
np.array([entity_to_index[s] for (s, _, _) in triples]))
fact_arg2 = torch.LongTensor(
np.array([entity_to_index[o] for (_, _, o) in triples]))
facts = [fact_rel, fact_arg1, fact_arg2]
model = NeuralKB(entity_embeddings=entity_embeddings,
predicate_embeddings=predicate_embeddings,
kernel=kernel,
facts=facts,
scoring_type=st)
indices = torch.LongTensor(
np.array([predicate_to_index['p'], predicate_to_index['q']]))
reformulator = SymbolicReformulator(predicate_embeddings, indices)
k = 5
rhoppy0 = RecursiveHoppy(model,
entity_embeddings,
hops=reformulator,
depth=0,
k=k)
rhoppy1 = RecursiveHoppy(model,
entity_embeddings,
hops=reformulator,
depth=1,
k=k)
rhoppy2 = RecursiveHoppy(model,
entity_embeddings,
hops=reformulator,
depth=2,
k=k)
rhoppy3 = RecursiveHoppy(model,
entity_embeddings,
hops=reformulator,
depth=3,
k=k)
rhoppy4 = RecursiveHoppy(model,
entity_embeddings,
hops=reformulator,
depth=4,
k=k)
xs_np = rs.randint(nb_entities, size=12)
xp_np = rs.randint(nb_predicates, size=12)
xo_np = rs.randint(nb_entities, size=12)
xs_np[0] = entity_to_index['a']
xp_np[0] = predicate_to_index['r']
xo_np[0] = entity_to_index['c']
xs_np[1] = entity_to_index['a']
xp_np[1] = predicate_to_index['r']
xo_np[1] = entity_to_index['e']
xs_np[2] = entity_to_index['a']
xp_np[2] = predicate_to_index['r']
xo_np[2] = entity_to_index['g']
xs_np[3] = entity_to_index['a']
xp_np[3] = predicate_to_index['r']
xo_np[3] = entity_to_index['i']
xs_np[4] = entity_to_index['a']
xp_np[4] = predicate_to_index['r']
xo_np[4] = entity_to_index['m']
xs_np[5] = entity_to_index['a']
xp_np[5] = predicate_to_index['r']
xo_np[5] = entity_to_index['o']
xs_np[6] = entity_to_index['a']
xp_np[6] = predicate_to_index['r']
xo_np[6] = entity_to_index['q']
xs_np[7] = entity_to_index['a']
xp_np[7] = predicate_to_index['r']
xo_np[7] = entity_to_index['s']
xs_np[8] = entity_to_index['a']
xp_np[8] = predicate_to_index['r']
xo_np[8] = entity_to_index['u']
# xs_np[9] = entity_to_index['a']
# xp_np[9] = predicate_to_index['r']
# xo_np[9] = entity_to_index['w']
xs = torch.LongTensor(xs_np)
xp = torch.LongTensor(xp_np)
xo = torch.LongTensor(xo_np)
xs_emb = entity_embeddings(xs)
xp_emb = predicate_embeddings(xp)
xo_emb = entity_embeddings(xo)
scores0 = rhoppy0.forward(xp_emb, xs_emb, xo_emb)
inf0 = rhoppy0.score(xp_emb, xs_emb, xo_emb)
for i in range(xs.shape[0]):
scores_sp, scores_po = scores0
inf_np = inf0.cpu().numpy()
scores_sp_np = scores_sp.cpu().numpy()
scores_po_np = scores_po.cpu().numpy()
np.testing.assert_allclose(inf_np[i],
scores_sp_np[i, xo[i]],
rtol=1e-5,
atol=1e-5)
np.testing.assert_allclose(inf_np[i],
scores_po_np[i, xs[i]],
rtol=1e-5,
atol=1e-5)
scores1 = rhoppy1.forward(xp_emb, xs_emb, xo_emb)
inf1 = rhoppy1.score(xp_emb, xs_emb, xo_emb)
for i in range(xs.shape[0]):
scores_sp, scores_po = scores1
inf_np = inf1.cpu().numpy()
scores_sp_np = scores_sp.cpu().numpy()
scores_po_np = scores_po.cpu().numpy()
np.testing.assert_allclose(inf_np[i],
scores_sp_np[i, xo[i]],
rtol=1e-5,
atol=1e-5)
np.testing.assert_allclose(inf_np[i],
scores_po_np[i, xs[i]],
rtol=1e-5,
atol=1e-5)
scores2 = rhoppy2.forward(xp_emb, xs_emb, xo_emb)
inf2 = rhoppy2.score(xp_emb, xs_emb, xo_emb)
for i in range(xs.shape[0]):
scores_sp, scores_po = scores2
inf_np = inf2.cpu().numpy()
scores_sp_np = scores_sp.cpu().numpy()
scores_po_np = scores_po.cpu().numpy()
np.testing.assert_allclose(inf_np[i],
scores_sp_np[i, xo[i]],
rtol=1e-1,
atol=1e-1)
np.testing.assert_allclose(inf_np[i],
scores_po_np[i, xs[i]],
rtol=1e-1,
atol=1e-1)
scores3 = rhoppy3.forward(xp_emb, xs_emb, xo_emb)
inf3 = rhoppy3.score(xp_emb, xs_emb, xo_emb)
for i in range(xs.shape[0]):
scores_sp, scores_po = scores3
inf_np = inf3.cpu().numpy()
scores_sp_np = scores_sp.cpu().numpy()
scores_po_np = scores_po.cpu().numpy()
np.testing.assert_allclose(inf_np[i],
scores_sp_np[i, xo[i]],
rtol=1e-1,
atol=1e-1)
np.testing.assert_allclose(inf_np[i],
scores_po_np[i, xs[i]],
rtol=1e-1,
atol=1e-1)
scores4 = rhoppy4.forward(xp_emb, xs_emb, xo_emb)
inf4 = rhoppy4.score(xp_emb, xs_emb, xo_emb)
for i in range(xs.shape[0]):
scores_sp, scores_po = scores4
inf_np = inf4.cpu().numpy()
scores_sp_np = scores_sp.cpu().numpy()
scores_po_np = scores_po.cpu().numpy()
np.testing.assert_allclose(inf_np[i],
scores_sp_np[i, xo[i]],
rtol=1e-1,
atol=1e-1)
np.testing.assert_allclose(inf_np[i],
scores_po_np[i, xs[i]],
rtol=1e-1,
atol=1e-1)
print(inf0)
print(inf1)
print(inf2)
print(inf3)
print(inf4)
inf0_np = inf0.cpu().numpy()
inf1_np = inf1.cpu().numpy()
inf2_np = inf2.cpu().numpy()
inf3_np = inf3.cpu().numpy()
inf4_np = inf4.cpu().numpy()
np.testing.assert_allclose(inf0_np,
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
rtol=1e-1,
atol=1e-1)
np.testing.assert_allclose(inf1_np,
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
rtol=1e-1,
atol=1e-1)
np.testing.assert_allclose(inf2_np,
[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
rtol=1e-1,
atol=1e-1)
np.testing.assert_allclose(inf3_np,
[1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
rtol=1e-1,
atol=1e-1)
np.testing.assert_allclose(inf4_np,
[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0],
rtol=1e-1,
atol=1e-1)
def parse_args():
parser.add_argument(
'--tag_filter',
help=
'tag_filter can be used to run the shapes which matches the tag. (all is used to run all the shapes)',
default='short')
# This option is used to filter test cases to run.
parser.add_argument(
'--operators',
help='Filter tests based on comma-delimited list of operators to test',
default=None)
parser.add_argument(
'--operator_range',
help='Filter tests based on operator_range(e.g. a-c or b,c-d)',
default=None)
parser.add_argument('--test_name',
help='Run tests that have the provided test_name',
default=None)
parser.add_argument('--list_ops',
help='List operators without running them',
action='store_true')
parser.add_argument('--list_tests',
help='List all test cases without running them',
action='store_true')
parser.add_argument(
"--iterations",
help="Repeat each operator for the number of iterations",
type=int)
parser.add_argument(
"--num_runs",
help=
"Run each test for num_runs. Each run executes an operator for number of <--iterations>",
type=int,
default=1,
)
parser.add_argument(
"--min_time_per_test",
help="Set the minimum time (unit: seconds) to run each test",
type=int,
default=0,
)
parser.add_argument(
"--warmup_iterations",
help="Number of iterations to ignore before measuring performance",
default=100,
type=int)
parser.add_argument(
"--omp_num_threads",
help="Number of OpenMP threads used in PyTorch/Caffe2 runtime",
default=None,
type=int)
parser.add_argument(
"--mkl_num_threads",
help="Number of MKL threads used in PyTorch/Caffe2 runtime",
default=None,
type=int)
parser.add_argument("--ai_pep_format",
type=benchmark_utils.str2bool,
nargs='?',
const=True,
default=False,
help="Print result when running on AI-PEP")
parser.add_argument("--use_jit",
type=benchmark_utils.str2bool,
nargs='?',
const=True,
default=False,
help="Run operators with PyTorch JIT mode")
parser.add_argument("--forward_only",
type=benchmark_utils.str2bool,
nargs='?',
const=True,
default=False,
help="Only run the forward path of operators")
parser.add_argument(
'--framework',
help='Comma-delimited list of frameworks to test (Caffe2, PyTorch)',
default="Caffe2,PyTorch")
parser.add_argument(
'--device',
help='Run tests on the provided architecture (cpu, cuda)',
default='None')
args, _ = parser.parse_known_args()
if args.omp_num_threads:
# benchmark_utils.set_omp_threads sets the env variable OMP_NUM_THREADS
# which doesn't have any impact as C2 init logic has already been called
# before setting the env var.
# In general, OMP_NUM_THREADS (and other OMP env variables) needs to be set
# before the program is started.
# From Chapter 4 in OMP standard: https://www.openmp.org/wp-content/uploads/openmp-4.5.pdf
# "Modifications to the environment variables after the program has started,
# even if modified by the program itself, are ignored by the OpenMP implementation"
benchmark_utils.set_omp_threads(args.omp_num_threads)
if benchmark_utils.is_pytorch_enabled(args.framework):
torch.set_num_threads(args.omp_num_threads)
if args.mkl_num_threads:
benchmark_utils.set_mkl_threads(args.mkl_num_threads)
return args
def _setup(self):
"""
Setup the inner attributes of the interface, initializing horovod and the callbacks handler. This method must be
called before train and evaluate.
"""
# Setup horovod:
if self._use_horovod:
# Import horovod:
self._hvd = importlib.import_module("horovod.torch")
# Initialize horovod:
self._hvd.init()
# Limit the number of CPU threads to be used per worker:
torch.set_num_threads(1)
# Setup additional multiprocessing related key word arguments for the data loaders initialization:
mp_data_loader_kwargs = {}
# Setup cuda:
if self._use_cuda and torch.cuda.is_available():
if self._use_horovod:
# Set the torch environment to use a specific GPU according to the horovod worker's local rank:
torch.cuda.set_device(self._hvd.local_rank())
# Log horovod worker device:
print(
f"Horovod worker #{self._hvd.rank()} is using GPU:{self._hvd.local_rank()}"
)
# Register the required multiprocessing arguments:
mp_data_loader_kwargs["num_workers"] = 1
mp_data_loader_kwargs["pin_memory"] = True
# Move the model and the stored objects to the GPU:
self._objects_to_cuda()
elif self._use_horovod:
# Log horovod worker device:
print(f"Horovod worker #{self._hvd.rank()} is using CPU")
# Initialize a callbacks handler:
if self._use_horovod:
self._callbacks_handler = CallbacksHandler(callbacks=[
callback for callback in self._callbacks
if callback.on_horovod_check(rank=self._hvd.rank())
])
else:
self._callbacks_handler = CallbacksHandler(
callbacks=self._callbacks)
# Prepare horovod for the run if needed:
if self._use_horovod:
# When supported, use 'forkserver' to spawn dataloader workers instead of 'fork' to prevent issues with
# Infiniband implementations that are not fork-safe:
if (mp_data_loader_kwargs.get("num_workers", 0) > 0
and hasattr(mp, "_supports_context")
and mp._supports_context
and "forkserver" in mp.get_all_start_methods()):
mp_data_loader_kwargs["multiprocessing_context"] = "forkserver"
# Partition dataset among workers using distributed samplers:
if self._training_set is not None:
self._training_sampler = DistributedSampler(
self._training_set.dataset,
num_replicas=self._hvd.size(),
rank=self._hvd.rank(),
)
self._training_set = self._insert_sampler_to_data_loader(
data_loader=self._training_set,
sampler=self._training_sampler,
multiprocessing_kwargs=mp_data_loader_kwargs,
)
if self._validation_set is not None:
self._validation_sampler = DistributedSampler(
self._validation_set.dataset,
num_replicas=self._hvd.size(),
rank=self._hvd.rank(),
)
self._validation_set = self._insert_sampler_to_data_loader(
data_loader=self._validation_set,
sampler=self._validation_sampler,
multiprocessing_kwargs=mp_data_loader_kwargs,
)
# Broadcast parameters and optimizer state:
self._hvd.broadcast_parameters(self._model.state_dict(),
root_rank=0)
if self._optimizer is not None:
self._hvd.broadcast_optimizer_state(self._optimizer,
root_rank=0)
# Add Horovod Distributed Optimizer:
self._optimizer = self._hvd.DistributedOptimizer(
self._optimizer,
named_parameters=self._model.named_parameters())
# Setup the callbacks functions:
self._callbacks_handler.on_setup(
model=self._model,
training_set=self._training_set,
validation_set=self._validation_set,
loss_function=self._loss_function,
optimizer=self._optimizer,
metric_functions=self._metric_functions,
scheduler=self._scheduler,
)
def test_reasoning_v5():
torch.set_num_threads(multiprocessing.cpu_count())
nb_entities = 10
nb_predicates = 5
embedding_size = 10
rs = np.random.RandomState(0)
triples = [('a', 'p', 'b'), ('b', 'q', 'c'), ('c', 'r', 'd'),
('d', 's', 'e')]
entity_to_index = {'a': 0, 'b': 1, 'c': 2, 'd': 3, 'e': 4}
predicate_to_index = {'p': 0, 'q': 1, 'r': 2, 's': 3}
for st in ['min', 'concat']:
with torch.no_grad():
kernel = GaussianKernel()
entity_embeddings = nn.Embedding(nb_entities,
embedding_size * 2,
sparse=True)
predicate_embeddings = nn.Embedding(nb_predicates,
embedding_size * 2,
sparse=True)
fact_rel = torch.LongTensor(
np.array([predicate_to_index[p] for (_, p, _) in triples]))
fact_arg1 = torch.LongTensor(
np.array([entity_to_index[s] for (s, _, _) in triples]))
fact_arg2 = torch.LongTensor(
np.array([entity_to_index[o] for (_, _, o) in triples]))
facts = [fact_rel, fact_arg1, fact_arg2]
model = NeuralKB(entity_embeddings=entity_embeddings,
predicate_embeddings=predicate_embeddings,
kernel=kernel,
facts=facts,
scoring_type=st)
indices = torch.LongTensor(
np.array([
predicate_to_index['p'], predicate_to_index['q'],
predicate_to_index['r'], predicate_to_index['s']
]))
reformulator = SymbolicReformulator(predicate_embeddings, indices)
hoppy = SimpleHoppy(model, entity_embeddings, hops=reformulator)
rhoppy = RecursiveHoppy(model,
entity_embeddings,
hops=reformulator,
depth=1)
xs_np = rs.randint(nb_entities, size=32)
xp_np = rs.randint(nb_predicates, size=32)
xo_np = rs.randint(nb_entities, size=32)
xs_np[0] = 0
xp_np[0] = 0
xo_np[0] = 1
xs_np[1] = 1
xp_np[1] = 1
xo_np[1] = 2
xs_np[2] = 0
xp_np[2] = 3
xo_np[2] = 4
xs = torch.LongTensor(xs_np)
xp = torch.LongTensor(xp_np)
xo = torch.LongTensor(xo_np)
xs_emb = entity_embeddings(xs)
xp_emb = predicate_embeddings(xp)
xo_emb = entity_embeddings(xo)
scores = hoppy.forward(xp_emb, xs_emb, xo_emb)
inf = hoppy.score(xp_emb, xs_emb, xo_emb)
scores_h = rhoppy.depth_r_forward(xp_emb, xs_emb, xo_emb, depth=1)
inf_h = rhoppy.depth_r_score(xp_emb, xs_emb, xo_emb, depth=1)
print(inf)
print(inf_h)
assert inf[2] > 0.95
scores_sp, scores_po = scores
scores_h_sp, scores_h_po = scores_h
inf = inf.cpu().numpy()
scores_sp = scores_sp.cpu().numpy()
scores_po = scores_po.cpu().numpy()
inf_h = inf_h.cpu().numpy()
scores_h_sp = scores_h_sp.cpu().numpy()
scores_h_po = scores_h_po.cpu().numpy()
np.testing.assert_allclose(inf, inf_h)
np.testing.assert_allclose(scores_sp, scores_h_sp)
np.testing.assert_allclose(scores_po, scores_h_po)
for i in range(xs.shape[0]):
np.testing.assert_allclose(inf[i],
scores_sp[i, xo[i]],
rtol=1e-5,
atol=1e-5)
np.testing.assert_allclose(inf[i],
scores_po[i, xs[i]],
rtol=1e-5,
atol=1e-5)
np.testing.assert_allclose(inf_h[i],
scores_h_sp[i, xo[i]],
rtol=1e-5,
atol=1e-5)
np.testing.assert_allclose(inf_h[i],
scores_h_po[i, xs[i]],
rtol=1e-5,
atol=1e-5)
def run(config):
model_dir = Path('./models') / config.env_id / config.model_name
if not model_dir.exists():
curr_run = 'run1'
else:
exst_run_nums = [
int(str(folder.name).split('run')[1])
for folder in model_dir.iterdir()
if str(folder.name).startswith('run')
]
if len(exst_run_nums) == 0:
curr_run = 'run1'
else:
curr_run = 'run%i' % (max(exst_run_nums) + 1)
run_dir = model_dir / curr_run
log_dir = run_dir / 'logs'
os.makedirs(str(log_dir))
logger = SummaryWriter(str(log_dir))
torch.manual_seed(config.seed)
np.random.seed(config.seed)
if not USE_CUDA:
torch.set_num_threads(config.n_training_threads)
env = make_parallel_env(config.env_id, config.n_rollout_threads,
config.seed, config.discrete_action)
maddpg = MADDPG.init_from_env(env,
agent_alg=config.agent_alg,
adversary_alg=config.adversary_alg,
tau=config.tau,
lr=config.lr,
hidden_dim=config.hidden_dim)
replay_buffer = ReplayBuffer(
config.buffer_length, maddpg.nagents,
[obsp.shape[0] for obsp in env.observation_space], [
acsp.shape[0] if isinstance(acsp, Box) else acsp.n
for acsp in env.action_space
])
t = 0
a_loss = []
c_loss = []
for ep_i in range(0, config.n_episodes, config.n_rollout_threads):
print(
"Episodes %i-%i of %i" %
(ep_i + 1, ep_i + 1 + config.n_rollout_threads, config.n_episodes))
obs = env.reset()
# obs.shape = (n_rollout_threads, nagent)(nobs), nobs differs per agent so not tensor
maddpg.prep_rollouts(device='cpu')
explr_pct_remaining = max(
0, config.n_exploration_eps - ep_i) / config.n_exploration_eps
maddpg.scale_noise(config.final_noise_scale +
(config.init_noise_scale -
config.final_noise_scale) * explr_pct_remaining)
maddpg.reset_noise()
for et_i in range(config.episode_length):
# rearrange observations to be per agent, and convert to torch Variable
torch_obs = [
Variable(torch.Tensor(np.vstack(obs[:, i])),
requires_grad=False) for i in range(maddpg.nagents)
]
# get actions as torch Variables
torch_agent_actions = maddpg.step(torch_obs, explore=True)
# convert actions to numpy arrays
agent_actions = [ac.data.numpy() for ac in torch_agent_actions]
# rearrange actions to be per environment
actions = [[ac[i] for ac in agent_actions]
for i in range(config.n_rollout_threads)]
next_obs, rewards, dones, infos = env.step(actions)
replay_buffer.push(obs, agent_actions, rewards, next_obs, dones)
obs = next_obs
t += config.n_rollout_threads
if (len(replay_buffer) >= config.batch_size and
(t % config.steps_per_update) < config.n_rollout_threads):
if USE_CUDA:
maddpg.prep_training(device='gpu')
else:
maddpg.prep_training(device='cpu')
for u_i in range(config.n_rollout_threads):
for a_i in range(maddpg.nagents):
sample = replay_buffer.sample(config.batch_size,
to_gpu=USE_CUDA)
maddpg.update(sample,
a_i,
logger=logger,
actor_loss_list=a_loss,
critic_loss_list=c_loss)
maddpg.update_all_targets()
maddpg.prep_rollouts(device='cpu')
ep_rews = replay_buffer.get_average_rewards(config.episode_length *
config.n_rollout_threads)
for a_i, a_ep_rew in enumerate(ep_rews):
logger.add_scalar('agent%i/mean_episode_rewards' % a_i, a_ep_rew,
ep_i)
# print('agent%i/mean_episode_rewards' % a_i, a_ep_rew, ep_i)
if ep_i % config.save_interval < config.n_rollout_threads:
os.makedirs(str(run_dir / 'incremental'), exist_ok=True)
maddpg.save(
str(run_dir / 'incremental' / ('model_ep%i.pt' % (ep_i + 1))))
maddpg.save(str(run_dir / 'model.pt'))
maddpg.save(str(run_dir / 'model.pt'))
env.close()
logger.export_scalars_to_json(str(log_dir / 'summary.json'))
logger.close()
index_aloss = list(range(1, len(a_loss) + 1))
plt.plot(index_aloss, a_loss)
plt.ylabel('actor_loss')
# plt.savefig("./results/" + config.env_id + "_" + config.model_name + "_actor_loss.jpg")
plt.savefig("./results/" + config.model_name + "_" + curr_run +
"_actor_loss.jpg")
plt.show()
index_closs = list(range(1, len(c_loss) + 1))
plt.plot(index_closs, c_loss)
plt.ylabel('critic_loss')
# plt.savefig("./results/" + config.env_id + "_" + config.model_name + "_critic_loss.jpg")
plt.savefig("./results/" + config.model_name + "_" + curr_run +
"_critic_loss.jpg")
plt.show()
# A trial run for debugging
if TRIAL == True:
data_size = -1
train_file = trial_file
dev_file = trial_file
test_file = trial_file
num_iter = 200
# setting network configurations
NetworkConfig.DEVICE = torch.device(device)
NetworkConfig.BUILD_GRAPH_WITH_FULL_BATCH = True
NetworkConfig.IGNORE_TRANSITION = False
NetworkConfig.NEUTRAL_BUILDER_ENABLE_NODE_TO_NN_OUTPUT_MAPPING = False
seed = 42
torch.manual_seed(seed)
torch.set_num_threads(40)
np.random.seed(seed)
random.seed(seed)
torch.cuda.manual_seed(seed)
UNK = '<UNK>'
PAD = '<PAD>'
if num_thread > 1:
NetworkConfig.NUM_THREADS = num_thread
print('Set NUM_THREADS = ', num_thread)
# read data
train_insts = TagReader.read_inst(train_file, True, num_train, opinion_offset)
dev_insts = TagReader.read_inst(dev_file, False, num_dev, opinion_offset)
test_insts = TagReader.read_inst(test_file, False, num_test, opinion_offset)
TagReader.label2id_map['<STOP>'] = len(TagReader.label2id_map)
print('Map: ', TagReader.label2id_map)
def run_train(): # 2019-11-24
args = Arguments()
gpu_id = args.gpu_id
env_name = args.env_name
mod_dir = args.mod_dir
memories_size = args.memories_size
batch_size = args.batch_size
update_gap = args.update_gap
soft_update_tau = args.soft_update_tau
actor_dim = args.actor_dim
critic_dim = args.critic_dim
show_gap = args.show_gap
max_step = args.max_step
max_epoch = args.max_epoch
gamma = args.gamma
explore_noise = args.explore_noise
policy_noise = args.policy_noise
random_seed = args.random_seed
smooth_kernel = args.smooth_kernel
is_remove = args.is_remove
'''PPO'''
num_episode = 500
batch_size = 2048
max_step_per_round = 2000
gamma = 0.995
lamda = 0.97
log_num_episode = 1
num_epoch = 10
minibatch_size = 256
clip = 0.2
loss_coeff_value = 0.5
loss_coeff_entropy = 0.02 # 0.01
lr = 3e-4
num_parallel_run = 5
layer_norm = True
state_norm = False
advantage_norm = True
lossvalue_norm = True
schedule_adam = 'linear'
schedule_clip = 'linear'
clip_now = clip
whether_remove_history(remove=is_remove, mod_dir=mod_dir)
'''env init'''
env = gym.make(env_name)
state_dim, action_dim, action_max, target_reward = get_env_info(env)
'''mod init'''
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
network = ActorCritic(state_dim, action_dim, layer_norm=True).to(device)
running_state = ZFilter((state_dim,), clip=5.0)
from torch.optim import Adam
optimizer = Adam(network.parameters(), lr=lr)
torch.set_num_threads(8)
torch.manual_seed(random_seed)
np.random.seed(random_seed)
'''train loop'''
rd_normal = np.random.normal
recorders = list()
rewards = list()
start_time = show_time = timer()
EPS = 1e-10
reward_record = []
global_steps = 0
from torch import Tensor
try:
for i_episode in range(num_episode):
# step1: perform current policy to collect trajectories
# this is an on-policy method!
memory = Memory()
num_steps = 0
reward_list = []
len_list = []
while num_steps < batch_size:
state = env.reset()
state = state2d_1d(state) # For CarRacing-v0
reward_sum = 0
t = 0
if state_norm:
state = running_state(state)
for t in range(max_step_per_round):
state_ten = torch.tensor((state,), dtype=torch.float32, device=device)
action_mean, action_logstd, value = network(state_ten)
action, logproba = network.select_action(action_mean, action_logstd)
action = action.cpu().data.numpy()[0]
logproba = logproba.cpu().data.numpy()[0]
next_state, reward, done, _ = env.step(action) # For CarRacing-v0
next_state = state2d_1d(next_state)
if np.sum(next_state > 1.1) > 185: # For CarRacing-v0, outside
reward = -100
done = True
reward_sum += reward
if state_norm:
next_state = running_state(next_state)
mask = 0 if done else 1
memory.push(state, value, action, logproba, mask, next_state, reward)
if done:
break
state = next_state
num_steps += (t + 1)
global_steps += (t + 1)
reward_list.append(reward_sum)
len_list.append(t + 1)
reward_record.append({
'episode': i_episode,
'steps': global_steps,
'meanepreward': np.mean(reward_list),
'meaneplen': np.mean(len_list)})
batch = memory.sample()
batch_size = len(memory)
rewards = torch.tensor(batch.reward, dtype=torch.float32, device=device)
values = torch.tensor(batch.value, dtype=torch.float32, device=device)
masks = torch.tensor(batch.mask, dtype=torch.float32, device=device)
actions = torch.tensor(batch.action, dtype=torch.float32, device=device)
states = torch.tensor(batch.state, dtype=torch.float32, device=device)
oldlogproba = torch.tensor(batch.logproba, dtype=torch.float32, device=device)
prev_return = 0
prev_value = 0
prev_advantage = 0
returns = torch.empty(batch_size, dtype=torch.float32, device=device)
deltas = torch.empty(batch_size, dtype=torch.float32, device=device)
advantages = torch.empty(batch_size, dtype=torch.float32, device=device)
for i in reversed(range(batch_size)):
returns[i] = rewards[i] + gamma * prev_return * masks[i]
deltas[i] = rewards[i] + gamma * prev_value * masks[i] - values[i]
# ref: https://arxiv.org/pdf/1506.02438.pdf (generalization advantage estimate)
advantages[i] = deltas[i] + gamma * lamda * prev_advantage * masks[i]
prev_return = returns[i]
prev_value = values[i]
prev_advantage = advantages[i]
if advantage_norm:
advantages = (advantages - advantages.mean()) / (advantages.std() + EPS)
for i_epoch in range(int(num_epoch * batch_size / minibatch_size)):
# sample from current batch
minibatch_ind = np.random.choice(batch_size, minibatch_size, replace=False)
minibatch_states = states[minibatch_ind]
minibatch_actions = actions[minibatch_ind]
minibatch_oldlogproba = oldlogproba[minibatch_ind]
minibatch_newlogproba = network.get_logproba(minibatch_states, minibatch_actions)
minibatch_advantages = advantages[minibatch_ind]
minibatch_returns = returns[minibatch_ind]
minibatch_newvalues = network._forward_critic(minibatch_states).flatten()
ratio = torch.exp(minibatch_newlogproba - minibatch_oldlogproba)
surr1 = ratio * minibatch_advantages
surr2 = ratio.clamp(1 - clip_now, 1 + clip_now) * minibatch_advantages
loss_surr = - torch.mean(torch.min(surr1, surr2))
# not sure the value loss should be clipped as well
# clip example: https://github.com/Jiankai-Sun/Proximal-Policy-Optimization-in-Pytorch/blob/master/ppo.py
# however, it does not make sense to clip score-like value by a dimensionless clipping parameter
# moreover, original paper does not mention clipped value
if lossvalue_norm:
minibatch_return_6std = 6 * minibatch_returns.std()
loss_value = torch.mean((minibatch_newvalues - minibatch_returns).pow(2)) / minibatch_return_6std
else:
loss_value = torch.mean((minibatch_newvalues - minibatch_returns).pow(2))
loss_entropy = torch.mean(torch.exp(minibatch_newlogproba) * minibatch_newlogproba)
total_loss = loss_surr + loss_coeff_value * loss_value + loss_coeff_entropy * loss_entropy
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
if schedule_clip == 'linear':
ep_ratio = 1 - (i_episode / num_episode)
clip_now = clip * ep_ratio
if schedule_adam == 'linear':
ep_ratio = 1 - (i_episode / num_episode)
lr_now = lr * ep_ratio
# set learning rate
# ref: https://stackoverflow.com/questions/48324152/
for g in optimizer.param_groups:
g['lr'] = lr_now
eva_reward = get_eva_reward(env, network, state_norm, running_state, max_step,
target_reward, device)
if i_episode % log_num_episode == 0:
print('E: {:4} |R: {:8.3f} EvaR: {:8.2f} |L: {:6.3f} = {:6.3f} + {} * {:6.3f} + {} * {:6.3f}'.format(
i_episode, reward_record[-1]['meanepreward'], eva_reward,
total_loss.data, loss_surr.data,
loss_coeff_value, loss_value.data,
loss_coeff_entropy, loss_entropy.data,
))
if eva_reward > target_reward:
print("########## Solved! ###########")
print('E: {:4} |R: {:8.3f} EvaR: {:8.2f}'.format(
i_episode, reward_record[-1]['meanepreward'], eva_reward, ))
break
except KeyboardInterrupt:
print("KeyboardInterrupt")
except Exception as e:
print(next_state.shape)
print(next_state)
print("Error: {}".format(e))
print('TimeUsed:', int(timer() - start_time))
rs = running_state.rs
np.savez('state_mean_std.npz', (rs.mean, rs.std))
# print("State.mean", repr(rs.mean))
# print("State.std ", repr(rs.std))
torch.save(network.state_dict(), '%s/PPO.pth' % (mod_dir,))
np.save('{}/reward_record.npy'.format(mod_dir), reward_record)
print("Save in Mod_dir:", mod_dir)
reward_record = np.load('{}/reward_record.npy'.format(args.mod_dir), allow_pickle=True)
recorders = np.array([(i['episode'], i['meanepreward'], i['meaneplen'])
for i in reward_record])
draw_plot_ppo(recorders, args.smooth_kernel, args.mod_dir)
# Last Modified: 2018-04-03 20:09:10
# Descption :
# Version : Python 3.6
############################################
from __future__ import division
import argparse
import torch
from torchtext import data
from tqdm import tqdm
import pandas as pd
import random
import os
import time
import logging
import logging.config
torch.set_num_threads(8)
torch.manual_seed(1)
random.seed(1)
# from model.lstm import LSTM
from model.bilstm import LSTM
# from proprecess import data_loader
import proprecess.data_loader_batch as data_loader
import train_batch as train
logging.config.fileConfig('./conf/logging.conf',
disable_existing_loggers=False)
logger = logging.getLogger(__file__)
# import spacy
# spacy_en = spacy.load("en")
# def tokenizer(text):
# return [tok.text for tok in spacy_en.tokenizer(text)]
import argparse
import os
import sys
import time
from datasets.datasets import load_dataset
from models import img_text_composition_models
import numpy as np
from tensorboardX import SummaryWriter
import test_retrieval
import torch
import torch.utils.data
from tqdm import tqdm as tqdm
import git # pip install gitpython
torch.set_num_threads(3)
def parse_opt():
"""Parses the input arguments."""
parser = argparse.ArgumentParser()
parser.add_argument('--exp_name', type=str, default='debug')
parser.add_argument('--comment', type=str, default='test_notebook')
parser.add_argument('--savedir', type=str, default='')
parser.add_argument('--inspect', action="store_true")
parser.add_argument('--dataset', type=str, default='css3d')
parser.add_argument('--dataset_path', type=str, default='')
parser.add_argument('--model', type=str, default='tirg')
parser.add_argument('--embed_dim', type=int, default=512)
"""""""""""""""""""""""""""""""""""""""
"set an optimizer"
optimizer = opt.SGD(our_resnext.parameters(), lr = 0.0001, momentum=0.9) #----- use SGD algorithm for all parameters of our_lenet, by learning rate 0.01 and Momentum is 0.9
# optimizer = opt.Adam(our_resnext.parameters(), lr = 0.001, eps = 1e-08) #----- use Adam algorithm for all parameters of our_classifier
"set a loss function"
# loss_function = nn.MSELoss() #----- here use MSE loss
loss_function = nn.CrossEntropyLoss() #----- here use cross entropy loss
"""""""""""""""""""""""""""
4. Train the ResNeXt34 part
"""""""""""""""""""""""""""
"Train the ResNeXt34"
tc.set_num_threads(10) #----- Sets the number of OpenMP threads used for parallelizing CPU operations
for epoch in range(90):
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
"load input data"
inputs, labels = data
inputs, labels = Variable(inputs), Variable(labels)
"clear all stored gradients if there exist"
optimizer.zero_grad()
"forward prop"
outputs = our_resnext(inputs)
def main(args):
params = ClassFromDict(args)
X = params.X
Y = params.Y
rundate = datetime.datetime.now().strftime('%b_%d_%Y_%H_%M_%S')
# ignore possible warnings from correlation and early stopping calculation
warnings.filterwarnings("ignore", message='An input array is constant')
warnings.filterwarnings("ignore", message='Mean of empty slice')
warnings.filterwarnings("ignore", message=' invalid value encountered in less_equal')
# hardware params
torch.set_num_threads(params.n_threads) # limits CPU usage
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # single GPU
use_cuda = torch.cuda.is_available()
# identifiers for saving
net_preamble = '_'.join([params.model[0], rundate])
for folder in [performance_dir, models_dir, output_dir]:
ensure_subfolder_in_folder(folder=folder, subfolder=params.model[0])
# create structures to hold performance metrics
folds_from_start_to_end = params.end_fold - params.start_fold
range_folds = range(params.start_fold, params.end_fold)
best_test_epochs = dict(zip([f'best_test_epoch_fold_{i}' for i in range_folds],
np.full(folds_from_start_to_end, np.nan)))
stopped_epochs = dict(zip([f'stopped_epoch_fold_{i}' for i in range_folds],
np.full(folds_from_start_to_end, np.nan)))
coords = [range(params.n_epochs), set_names, metrics, params.outcome_names, range_folds]
coords_len = [len(x) for x in coords]
performance = xr.DataArray(np.full(coords_len, np.nan), coords=coords,
dims=['epoch', 'set', 'metrics', 'outcome', 'cv_fold'])
net_ids = [] # aim: track id of each fold's net
for fold in range(params.start_fold, params.end_fold):
print(f'\nTraining fold {fold}')
transformed_data = TransformedData(fold, X, Y)
transformed_data.preprocess_data()
trainset = HCPDataset(transformed_data, mode="train")
testset = HCPDataset(transformed_data, mode="test")
valset = HCPDataset(transformed_data, mode="val")
dataloader_kwargs = dict(shuffle=True, batch_size=8, pin_memory=True)
trainloader = torch.utils.data.DataLoader(trainset, **dataloader_kwargs)
testloader = torch.utils.data.DataLoader(testset, **dataloader_kwargs)
valloader = torch.utils.data.DataLoader(valset, **dataloader_kwargs)
print("\nInit Network",
f'\nTraining data: {", ".join(transformed_data.matrix_labels)}',
f'\nPredicting: {", ".join(transformed_data.outcome_names)}')
def instantiate_net(architecture, train_set):
switcher = dict(kawahara=KawaharaBNCNN,
he_sex=HeSexBNCNN,
pervaiz=PervaizBNCNN,
he_58=He58behaviorsBNCNN)
net = switcher.get(architecture, PervaizBNCNN)
return net(train_set, transformed_data)
net = instantiate_net(params.architecture, trainset.X)
if use_cuda:
net = net.to(device)
assert next(net.parameters()).is_cuda, 'Parameters are not on the GPU !'
cudnn.benchmark = True
assert id(net) not in net_ids, 'No new net was instantiated. Please debug.'
net_ids.append(id(net))
def init_weights_he(m):
""" Weights initialization for the dense layers using He Uniform initialization.
Only applies to linear layers
(He et al., 2015) http://arxiv.org/abs/1502.01852, https://keras.io/initializers/#he_uniform
"""
if type(m) == torch.nn.Linear:
fan_in = net.dense1.in_features
he_lim = np.sqrt(6 / fan_in)
m.weight.data.uniform_(-he_lim, he_lim)
net.apply(init_weights_he)
def get_optimizer():
if params.optimizer == 'sgd':
optimizer = torch.optim.SGD(net.parameters(), lr=params.lr, momentum=params.momentum, nesterov=True,
weight_decay=params.wd)
elif params.optimizer == 'adam':
optimizer = torch.optim.Adam(net.parameters(), lr=params.lr, weight_decay=params.wd)
return optimizer
optimizer = get_optimizer()
def get_criterion():
if Y.multiclass:
if Y.n_classes == 2:
criterion = nn.BCELoss(
weight=torch.Tensor(Y.class_weights).to(device)) # balanced Binary Cross Entropy
elif Y.n_classes > 2:
criterion = nn.CrossEntropyLoss(weight=torch.Tensor(Y.class_weights).to(device))
else:
criterion = torch.nn.MSELoss()
return criterion
criterion = get_criterion()
def train_net(): # training in mini batches
net.train()
running_loss = 0.0
y_pred = []
y_true = []
for batch_idx, (inputs, targets) in enumerate(trainloader):
if use_cuda:
if not Y.multioutcome and not Y.multiclass:
inputs, targets = inputs.to(device), targets.to(device).unsqueeze(
1) # unsqueeze needed for vstack
else:
inputs, targets = inputs.to(device), targets.to(device)
optimizer.zero_grad()
inputs, targets = Variable(inputs), Variable(targets)
outputs = net(inputs)
targets = targets.view(outputs.size())
if Y.multiclass and Y.n_classes > 2:
loss = criterion(input=outputs, target=torch.argmax(targets.data, 1))
else:
loss = criterion(input=outputs, target=targets)
loss.backward()
# prevents a vanishing / exploding gradient problem
torch.nn.utils.clip_grad_norm_(net.parameters(), max_norm=params.max_norm)
for p in net.parameters():
p.data.add_(-params.lr, p.grad.data)
optimizer.step()
running_loss += loss.data.mean(0)
y_pred.append(outputs.data.cpu().numpy())
y_true.append(targets.data.cpu().numpy())
if batch_idx == len(trainloader) - 1: # print loss for final batch
if params.verbose:
print(f'\n train loss: %.6f' % (running_loss / len(trainloader)))
if not Y.multioutcome and not Y.multiclass:
return np.vstack(y_pred), np.vstack(y_true), running_loss / batch_idx
else:
return np.vstack(y_pred), np.vstack(y_true).squeeze(), running_loss / batch_idx
def evaluate_net(set_name):
if set_name == 'test':
dataloader = testloader
elif set_name == 'val':
dataloader = valloader
net.eval()
running_loss = 0.0
y_pred = []
y_true = []
for batch_idx, (inputs, targets) in enumerate(dataloader):
if use_cuda:
if not Y.multioutcome and not Y.multiclass:
inputs, targets = inputs.to(device), targets.to(device).unsqueeze(1)
else:
inputs, targets = inputs.to(device), targets.to(device)
with torch.no_grad():
inputs, targets = Variable(inputs), Variable(targets)
outputs = net(inputs)
targets = targets.view(outputs.size())
if Y.multiclass and Y.n_classes > 2:
loss = criterion(input=outputs, target=torch.argmax(targets.data, 1))
else:
loss = criterion(input=outputs, target=targets)
y_pred.append(outputs.data.cpu().numpy())
y_true.append(targets.data.cpu().numpy())
running_loss += loss.data.mean(0)
if batch_idx == len(dataloader) - 1: # print loss for final batch
if params.verbose:
print(f'\n {set_name} loss: %.6f' % (running_loss / len(dataloader)))
if not Y.multioutcome and not Y.multiclass:
return np.vstack(y_pred), np.vstack(y_true), running_loss / batch_idx
else:
return np.vstack(y_pred), np.vstack(y_true).squeeze(), running_loss / batch_idx
output_names = [[f'{name}p', f'{name}y'] for name in set_names]
output_names = [x for y in output_names for x in y]
epoch_output = dict(zip(output_names, [[] for _ in range(len(output_names))]))
for epoch in range(params.n_epochs):
print("\nEpoch %d" % epoch)
test_mae_kwargs = dict(set='test', metrics='MAE', cv_fold=fold)
test_r_kwargs = dict(set='test', metrics='pearsonr', cv_fold=fold)
test_acc_kwargs = dict(set='test', metrics='accuracy', cv_fold=fold)
def calculate_and_print_performance(pred, true, set_name):
if Y.multiclass:
acc_kwargs = dict(epoch=epoch, metrics=['accuracy'], cv_fold=fold)
pred, true = np.argmax(pred, 1), np.argmax(true, 1)
acc = balanced_accuracy_score(true, pred)
performance.loc[{'set': set_name, **acc_kwargs}] = acc
print(f" {Y.outcome_names}, {set_name} accuracy : {acc:.3}")
else:
save_metrics = ['MAE', 'pearsonr', 'pearsonp', 'spearmanr', 'spearmanp']
metrics_kwargs = dict(epoch=epoch, metrics=save_metrics, cv_fold=fold)
if Y.multioutcome:
mae_all = np.array([mae(true[:, i], pred[:, i]) for i in range(Y.n_outcomes)])
pears_all = np.array([list(pearsonr(true[:, i], pred[:, i])) for i in range(Y.n_outcomes)])
spear_all = np.array([list(spearmanr(true[:, i], pred[:, i])) for i in range(Y.n_outcomes)])
performance.loc[{'set': set_name, **metrics_kwargs}] = [mae_all,
pears_all[:, 0], pears_all[:, 1],
spear_all[:, 0], spear_all[:, 1]]
for i in range(Y.n_outcomes):
print(f" {Y.outcome_names[i]}, {set_name} MAE : {mae_all[i]:.3},"
f" pearson R: {pears_all[i, 0]:.3} (p = {pears_all[i, 1]:.4})")
else:
mae_one = mae(pred, true)
pears_one = pearsonr(pred[:, 0], true[:, 0])
spear_one = spearmanr(pred[:, 0], true[:, 0])
performance.loc[{'set': set_name, **metrics_kwargs}] = np.array([mae_one, pears_one[0],
pears_one[1], spear_one[0],
spear_one[1]])[:, None]
print(f" {Y.outcome_names[0]}, {set_name} MAE : {mae_one:.3}, "
f"pearson R: {pears_one[0]:.3} (p = {pears_one[1]:.4})")
# get model output
for name in set_names:
if name == 'train':
pred, true, loss = train_net()
else:
pred, true, loss = evaluate_net(name)
performance.loc[dict(epoch=epoch, set=name, metrics='loss', cv_fold=fold)] = [loss]
epoch_output[f'{name}p'] = pred
epoch_output[f'{name}y'] = true
calculate_and_print_performance(pred, true, name)
def best_test_epoch():
if Y.multiclass:
best_epoch_yet = performance.loc[test_acc_kwargs].argmax().values
elif Y.multioutcome:
best_epoch_yet = performance.loc[test_mae_kwargs].mean(axis=-1).argmin().values
else:
best_epoch_yet = performance.loc[test_mae_kwargs].argmin().values
return best_epoch_yet
if best_test_epoch() == epoch:
best_epoch = epoch
best_net = net.state_dict()
best_output = epoch_output.copy()
def is_performance_stagnating():
if Y.multiclass:
recent_acc = performance[epoch - params.ep_int:-1].loc[test_acc_kwargs]
current_acc = performance[epoch].loc[test_acc_kwargs]
stagnant = np.nanmean(recent_acc >= current_acc)
else:
recent_mae = performance[epoch - params.ep_int:-1].loc[test_mae_kwargs]
current_mae = performance[epoch].loc[test_mae_kwargs]
recent_r = performance[epoch - params.ep_int:-1].loc[test_r_kwargs]
current_r = performance[epoch].loc[test_r_kwargs]
if Y.multioutcome: # stagnant if model stops learning on at least half of outcomes
majority = int(np.ceil(Y.n_outcomes / 2))
stagnant_mae = (np.nanmean(recent_mae, axis=0) <= current_mae).sum() >= majority
stagnant_abs_r = (np.nanmean(np.abs(recent_r), axis=0) <= np.abs(current_r)).sum() >= majority
else:
stagnant_mae = np.nanmean(recent_mae, axis=0) <= current_mae
stagnant_abs_r = np.nanmean(np.abs(recent_r), axis=0) <= np.abs(current_r)
stagnant = bool(stagnant_mae + stagnant_abs_r)
return stagnant
if params.early and epoch > params.min_train_epochs:
if is_performance_stagnating():
if params.verbose:
print('\bEarly stopping conditions reached, stopping training...')
stopped_epochs[f'stopped_epoch_fold_{fold}'] = epoch - params.ep_int
break
best_test_epochs[f'best_test_epoch_fold_{fold}'] = best_epoch
# saving net weights and output
net_path = os.path.join(models_dir, params.model[0], '_'.join([net_preamble + f'fold{fold}_net.pt']))
torch.save(best_net, net_path)
output_path = os.path.join(output_dir, params.model[0], '_'.join([net_preamble, f'fold{fold}_output.pkl']))
pickle.dump(best_output, open(output_path, "wb"))
training_params = get_training_params(params=params, transformed_data=transformed_data)
training_params.update({'rundate': rundate})
training_params.update(best_test_epochs)
if params.early:
training_params.update(stopped_epochs)
filename_performance = '_'.join([net_preamble, 'performance.nc'])
performance.attrs = training_params
performance.name = filename_performance
performance.to_netcdf(os.path.join(performance_dir, params.model[0], filename_performance)) # saving
def calculate_and_print_val_performance(best_test_epochs):
print(f'\nModel training i/o:'
f'\nmatrix label(s): {", ".join(transformed_data.matrix_labels)}'
f'\noutcome(s): {", ".join(transformed_data.outcome_names)}'
f'\ntransformation: {transformed_data.transformations}'
f'\nconfound(s): {", ".join(list(filter(None, transformed_data.confound_names)))}'
f'\nbest test epoch(s): {list(best_test_epochs.values())}')
print(f'\nValidation set performance on best test epoch (mean across cv_folds):')
for metric in metrics:
best_val_metric_by_fold = [performance.loc[dict(set='val',
metrics=metric,
epoch=best_test_epochs[f'best_test_epoch_fold_{i}'],
cv_fold=i)].values.tolist() for i in range_folds]
best_val_metric_mean = np.nanmean(best_val_metric_by_fold, axis=0)
print(f" {metric}: {best_val_metric_mean}")
calculate_and_print_val_performance(best_test_epochs)
return dict(performance=performance)
import torch from torchtext import data import os import random torch.set_num_threads(1) torch.manual_seed(233) random.seed(233) import numpy as np import argparse import datetime import classification_datasets import classification_datasets_nonstatic import datasets_char import data_five import word_embedding_loader as loader import embedding_modify as loader_modify import Bi_lstm_2DCNN import LSTM import Bi_LSTM import CNN_LSTM import CNN_LSTM_Parallel import multi_CNN import multichannel_CNN import CNN import CNN_char import gru import bi_gru from bnlstm import LSTM_bn, BNLSTMCell import model_bnlstm
parser.add_argument('--seed', type=int, default=1)
args = parser.parse_args()
args.work_dir = osp.dirname(osp.realpath(__file__))
args.data_fp = osp.join(args.work_dir, '..', 'data', args.dataset)
args.out_dir = osp.join(args.work_dir, 'out', args.exp_name)
args.checkpoints_dir = osp.join(args.out_dir, 'checkpoints')
print(args)
utils.makedirs(args.out_dir)
utils.makedirs(args.checkpoints_dir)
writer = writer.Writer(args)
device = torch.device('cuda', args.device_idx) if torch.cuda.is_available() else torch.device('cpu')
torch.set_num_threads(args.n_threads)
# deterministic
torch.manual_seed(args.seed)
cudnn.benchmark = False
cudnn.deterministic = True
# load dataset
template_fp = osp.join(args.data_fp, 'template', 'template.obj')
meshdata = ClsfData(args.data_fp,
template_fp,
split=args.split)
train_loader = DataLoader(meshdata.train_dataset,
batch_size=args.batch_size,
shuffle=True)
test_loader = DataLoader(meshdata.test_dataset, batch_size=args.batch_size)
def start_train(model, train_iter, dev_iter, test_iter):
"""
:function：start train
:param model:
:param train_iter:
:param dev_iter:
:param test_iter:
:return:
"""
if config.predict is not None:
label = train_ALL_CNN.predict(config.predict, model, config.text_field, config.label_field)
print('\n[Text] {}[Label] {}\n'.format(config.predict, label))
elif config.test:
try:
print(test_iter)
train_ALL_CNN.test_eval(test_iter, model, config)
except Exception as e:
print("\nSorry. The test dataset doesn't exist.\n")
else:
print("\n cpu_count \n", mu.cpu_count())
torch.set_num_threads(config.num_threads)
if os.path.exists("./Test_Result.txt"):
os.remove("./Test_Result.txt")
if config.CNN:
print("CNN training start......")
model_count = train_ALL_CNN.train(train_iter, dev_iter, test_iter, model, config)
elif config.DEEP_CNN:
print("DEEP_CNN training start......")
model_count = train_ALL_CNN.train(train_iter, dev_iter, test_iter, model, config)
elif config.LSTM:
print("LSTM training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.GRU:
print("GRU training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.BiLSTM:
print("BiLSTM training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.BiLSTM_1:
print("BiLSTM_1 training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.CNN_LSTM:
print("CNN_LSTM training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.CLSTM:
print("CLSTM training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.CBiLSTM:
print("CBiLSTM training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.CGRU:
print("CGRU training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.CNN_BiLSTM:
print("CNN_BiLSTM training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.BiGRU:
print("BiGRU training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.CNN_BiGRU:
print("CNN_BiGRU training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
elif config.CNN_MUI:
print("CNN_MUI training start......")
model_count = train_ALL_CNN.train(train_iter, dev_iter, test_iter, model, config)
elif config.DEEP_CNN_MUI:
print("DEEP_CNN_MUI training start......")
model_count = train_ALL_CNN.train(train_iter, dev_iter, test_iter, model, config)
elif config.HighWay_CNN is True:
print("HighWay_CNN training start......")
model_count = train_ALL_CNN.train(train_iter, dev_iter, test_iter, model, config)
elif config.HighWay_BiLSTM_1 is True:
print("HighWay_BiLSTM_1 training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, config)
print("Model_count", model_count)
resultlist = []
if os.path.exists("./Test_Result.txt"):
file = open("./Test_Result.txt")
for line in file.readlines():
if line[:10] == "Evaluation":
resultlist.append(float(line[34:41]))
result = sorted(resultlist)
file.close()
file = open("./Test_Result.txt", "a")
file.write("\nThe Best Result is : " + str(result[len(result) - 1]))
file.write("\n")
file.close()
shutil.copy("./Test_Result.txt", "./snapshot/" + config.mulu + "/Test_Result.txt")
def main():
parser = argparse.ArgumentParser(
description='PyTorch Tiny ImageNet testing')
parser.add_argument('-d',
'--data',
default='./',
type=str,
metavar='S',
help='directory of input data (default:./)')
parser.add_argument(
'-m',
'--model',
default="./model",
type=str,
metavar='S',
help='directory of the saved model and weights (default:./model)')
parser.add_argument('-o',
'--output',
default="./output",
type=str,
metavar='S',
help='directory of output images (default:./output)')
parser.add_argument(
'-j',
'--workers',
default=0,
type=int,
metavar='N',
help='number of cup and data loading workers (default: 0)')
parser.add_argument('-bs',
'--batch-size',
type=int,
default=4,
metavar='N',
help='batch size (default: 4)')
args = parser.parse_args()
ensure_dir(args.data)
ensure_dir(args.model)
ensure_dir(args.output)
torch.set_num_threads(args.workers)
state_dir = os.path.join(os.path.expanduser(args.model), 'state.bth')
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
testset = TinyImageNetDataset(root=args.data,
mode='test',
transform=transform)
testloader = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers)
class_names = testset.words
alexnet = myalexnet()
alexnet.classifier = nn.Sequential(
nn.Dropout(),
nn.Linear(256, 256),
nn.ReLU(inplace=True),
nn.Dropout(),
nn.Linear(256, 256),
nn.ReLU(inplace=True),
nn.Linear(256, 200),
)
alexnet.load_state_dict(torch.load(state_dir))
alexnet.eval()
for i, inputs in enumerate(testloader):
with torch.set_grad_enabled(False):
outputs = alexnet(inputs)
preds = torch.argmax(outputs, 1)
images_so_far = 0
num_images = 4
img = inputs.cpu()
for j in range(img.shape[0]):
images_so_far += 1
ax = plt.subplot(num_images // 2, 2, images_so_far)
ax.axis('off')
ax.set_title('predicted: {}'.format(class_names[preds[j]]))
inp = img[j, :, :, :].view(3, 64, 64).numpy().transpose(
(1, 2, 0))
inp = inp * 0.5 + 0.5
plt.imshow(inp)
plt.savefig(
os.path.join(args.output,
'train_image{:03d}.png'.format(i)))
def validation_epoch_end(self, validation_steps_outputs):
self.coco_eval.synchronize_between_processes()
self.coco_eval.accumulate()
self.coco_eval.summarize()
torch.set_num_threads(self.n_threads)
def __init__(
self,
model_type,
model_name,
num_labels=None,
pos_weight=None,
args=None,
use_cuda=True,
cuda_device=-1,
**kwargs,
):
"""
Initializes a MultiLabelClassification model.
Args:
model_type: The type of model (bert, roberta)
model_name: Default Transformer model name or path to a directory containing Transformer model file (pytorch_nodel.bin).
num_labels (optional): The number of labels or classes in the dataset.
pos_weight (optional): A list of length num_labels containing the weights to assign to each label for loss calculation.
args (optional): Default args will be used if this parameter is not provided. If provided, it should be a dict containing the args that should be changed in the default args.
use_cuda (optional): Use GPU if available. Setting to False will force model to use CPU only.
cuda_device (optional): Specific GPU that should be used. Will use the first available GPU by default.
**kwargs (optional): For providing proxies, force_download, resume_download, cache_dir and other options specific to the 'from_pretrained' implementation where this will be supplied.
""" # noqa: ignore flake8"
MODEL_CLASSES = {
"albert": (
AlbertConfig,
AlbertForMultiLabelSequenceClassification,
AlbertTokenizer,
),
"bert": (
BertConfig,
BertForMultiLabelSequenceClassification,
BertTokenizer,
),
"camembert": (
CamembertConfig,
CamembertForMultiLabelSequenceClassification,
CamembertTokenizer,
),
"distilbert": (
DistilBertConfig,
DistilBertForMultiLabelSequenceClassification,
DistilBertTokenizer,
),
"electra":
(ElectraConfig, ElectraForMultiLabelSequenceClassification,
ElectraTokenizer),
"flaubert": (
FlaubertConfig,
FlaubertForMultiLabelSequenceClassification,
FlaubertTokenizer,
),
"longformer": (
LongformerConfig,
LongformerForMultiLabelSequenceClassification,
LongformerTokenizer,
),
"roberta": (
RobertaConfig,
RobertaForMultiLabelSequenceClassification,
RobertaTokenizer,
),
"xlm":
(XLMConfig, XLMForMultiLabelSequenceClassification, XLMTokenizer),
"xlmroberta": (
XLMRobertaConfig,
XLMRobertaForMultiLabelSequenceClassification,
XLMRobertaTokenizer,
),
"xlnet": (
XLNetConfig,
XLNetForMultiLabelSequenceClassification,
XLNetTokenizer,
),
}
self.args = self._load_model_args(model_name)
if isinstance(args, dict):
self.args.update_from_dict(args)
elif isinstance(args, MultiLabelClassificationArgs):
self.args = args
if self.args.thread_count:
torch.set_num_threads(self.args.thread_count)
if "sweep_config" in kwargs:
sweep_config = kwargs.pop("sweep_config")
sweep_values = sweep_config_to_sweep_values(sweep_config)
self.args.update_from_dict(sweep_values)
if self.args.manual_seed:
random.seed(self.args.manual_seed)
np.random.seed(self.args.manual_seed)
torch.manual_seed(self.args.manual_seed)
if self.args.n_gpu > 0:
torch.cuda.manual_seed_all(self.args.manual_seed)
if not use_cuda:
self.args.fp16 = False
config_class, model_class, tokenizer_class = MODEL_CLASSES[model_type]
if num_labels:
self.config = config_class.from_pretrained(model_name,
num_labels=num_labels,
**self.args.config)
self.num_labels = num_labels
else:
self.config = config_class.from_pretrained(model_name,
**self.args.config)
self.num_labels = self.config.num_labels
self.pos_weight = pos_weight
if use_cuda:
if torch.cuda.is_available():
if cuda_device == -1:
self.device = torch.device("cuda")
else:
self.device = torch.device(f"cuda:{cuda_device}")
else:
raise ValueError(
"'use_cuda' set to True when cuda is unavailable."
" Make sure CUDA is available or set use_cuda=False.")
else:
self.device = "cpu"
if not self.args.quantized_model:
if self.pos_weight:
self.model = model_class.from_pretrained(
model_name,
config=self.config,
pos_weight=torch.Tensor(self.pos_weight).to(self.device),
**kwargs)
else:
self.model = model_class.from_pretrained(model_name,
config=self.config,
**kwargs)
else:
quantized_weights = torch.load(
os.path.join(model_name, "pytorch_model.bin"))
if self.pos_weight:
self.model = model_class.from_pretrained(
None,
config=self.config,
state_dict=quantized_weights,
weight=torch.Tensor(self.pos_weight).to(self.device),
)
else:
self.model = model_class.from_pretrained(
None, config=self.config, state_dict=quantized_weights)
if self.args.dynamic_quantize:
self.model = torch.quantization.quantize_dynamic(self.model,
{torch.nn.Linear},
dtype=torch.qint8)
if self.args.quantized_model:
self.model.load_state_dict(quantized_weights)
if self.args.dynamic_quantize:
self.args.quantized_model = True
self.results = {}
self.tokenizer = tokenizer_class.from_pretrained(
model_name, do_lower_case=self.args.do_lower_case, **kwargs)
self.args.model_name = model_name
self.args.model_type = model_type
if self.args.wandb_project and not wandb_available:
warnings.warn(
"wandb_project specified but wandb is not available. Wandb disabled."
)
self.args.wandb_project = None
import math
import time
import torch
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
torch.set_num_threads(torch.get_num_threads())
def start(device, computations, test_int8, test_int16, test_int32, test_int64,
test_float32, test_float64, test_nn):
# create tester object
tester = Tester(device)
if test_int8:
int8_time = tester.test_int8(computations)
print('int8 time: {:7.1f}s'.format(int8_time))
if test_int16:
int16_time = tester.test_int16(computations)
print('int16 time: {:6.1f}s'.format(int16_time))
if test_int32:
int32_time = tester.test_int32(computations)
print('int32 time: {:6.1f}s'.format(int32_time))
if test_int64:
int64_time = tester.test_int64(computations)
print('int64 time: {:6.1f}s'.format(int64_time))
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
gpu = ig_utils.pick_gpu_lowest_memory() if args.gpu == 'auto' else int(args.gpu)
torch.cuda.set_device(gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % gpu)
logging.info("args = %s", args)
#### data
if args.dataset == 'cifar10':
train_transform, valid_transform = ig_utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = ig_utils._data_transforms_cifar100(args)
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = ig_utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data, split='train', download=True, transform=train_transform)
valid_data = dset.SVHN(root=args.data, split='test', download=True, transform=valid_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True)
test_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size,
pin_memory=True)
#### sdarts
if args.perturb_alpha == 'none':
perturb_alpha = None
elif args.perturb_alpha == 'pgd_linf':
perturb_alpha = Linf_PGD_alpha
elif args.perturb_alpha == 'random':
perturb_alpha = Random_alpha
else:
print('ERROR PERTURB_ALPHA TYPE:', args.perturb_alpha); exit(1)
#### model
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
## darts
if args.method in ['darts', 'blank']:
model = DartsNetwork(args.init_channels, n_classes, args.layers, criterion, spaces_dict[args.search_space], args)
## sdarts
elif args.method == 'sdarts':
model = SDartsNetwork(args.init_channels, n_classes, args.layers, criterion, spaces_dict[args.search_space], args)
## projection
elif args.method in ['darts-proj', 'blank-proj']:
model = DartsNetworkProj(args.init_channels, n_classes, args.layers, criterion, spaces_dict[args.search_space], args)
elif args.method in ['sdarts-proj']:
model = SDartsNetworkProj(args.init_channels, n_classes, args.layers, criterion, spaces_dict[args.search_space], args)
else:
print('ERROR: WRONG MODEL:', args.method); exit(0)
model = model.cuda()
architect = Architect(model, args)
logging.info("param size = %fMB", ig_utils.count_parameters_in_MB(model))
#### scheduler
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
model.optimizer, float(args.epochs), eta_min=args.learning_rate_min)
#### resume
start_epoch = 0
if args.resume_epoch > 0:
logging.info('loading checkpoint from {}'.format(expid))
filename = os.path.join(args.save, 'checkpoint_{}.pth.tar'.format(args.resume_epoch))
if os.path.isfile(filename):
logging.info("=> loading checkpoint '{}'".format(filename))
checkpoint = torch.load(filename, map_location='cpu')
resume_epoch = checkpoint['epoch'] # epoch
model.load_state_dict(checkpoint['state_dict']) # model
saved_arch_parameters = checkpoint['alpha'] # arch
model.set_arch_parameters(saved_arch_parameters)
scheduler.load_state_dict(checkpoint['scheduler'])
model.optimizer.load_state_dict(checkpoint['optimizer']) # optimizer
architect.optimizer.load_state_dict(checkpoint['arch_optimizer']) # arch optimizer
start_epoch = args.resume_epoch
logging.info("=> loaded checkpoint '{}' (epoch {})".format(filename, resume_epoch))
else:
logging.info("=> no checkpoint found at '{}'".format(filename))
#### main search
logging.info('starting training at epoch {}'.format(start_epoch))
for epoch in range(start_epoch, args.epochs):
lr = scheduler.get_lr()[0]
## data aug
if args.cutout:
train_transform.transforms[-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr, train_transform.transforms[-1].cutout_prob)
else:
logging.info('epoch %d lr %e', epoch, lr)
## sdarts
if args.perturb_alpha:
epsilon_alpha = 0.03 + (args.epsilon_alpha - 0.03) * epoch / args.epochs
logging.info('epoch %d epsilon_alpha %e', epoch, epsilon_alpha)
## logging
num_params = ig_utils.count_parameters_in_Compact(model)
genotype = model.genotype()
logging.info('param size = %f', num_params)
logging.info('genotype = %s', genotype)
model.printing(logging)
## training
train_acc, train_obj = train(train_queue, valid_queue, model, architect, model.optimizer, lr, epoch,
perturb_alpha, epsilon_alpha)
logging.info('train_acc %f | train_obj %f', train_acc, train_obj)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
## scheduler updates (before saving ckpts)
scheduler.step()
## validation
valid_acc, valid_obj = infer(valid_queue, model, log=False)
logging.info('valid_acc %f | valid_obj %f', valid_acc, valid_obj)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
test_acc, test_obj = infer(test_queue, model, log=False)
logging.info('test_acc %f | test_obj %f', test_acc, test_obj)
writer.add_scalar('Acc/test', test_acc, epoch)
writer.add_scalar('Obj/test', test_obj, epoch)
## saving
if (epoch + 1) % args.ckpt_interval == 0:
save_state_dict = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'alpha': model.arch_parameters(),
'optimizer': model.optimizer.state_dict(),
'arch_optimizer': architect.optimizer.state_dict(),
'scheduler': scheduler.state_dict()
}
ig_utils.save_checkpoint(save_state_dict, False, args.save, per_epoch=True)
#### projection
if args.dev == 'proj':
pt_project(train_queue, valid_queue, model, architect, model.optimizer,
start_epoch, args, infer, perturb_alpha, args.epsilon_alpha)
writer.close()
'''Tracker module'''
from RT_MDNet import RT_MDNet
'''Refine module'''
torch.cuda.set_device(1) # set GPU id
from pytracking.Scale_Estimator_bcm import Scale_Estimator_bcm
from common_path_bcm import *
parser = argparse.ArgumentParser(description='MDNet refine tracking')
parser.add_argument('--dataset', default= dataset_name_, type=str,
help='eval one special dataset')
parser.add_argument('--video', default= video_name_, type=str,
help='eval one special video')
parser.add_argument('--vis', action='store_true',default=False,
help='whether visualzie result')
args = parser.parse_args()
torch.set_num_threads(1)
def bbox_clip(bbox, boundary):
x1, y1, width, height = bbox
cx = x1 + width/2
cy = y1 + height/2
'''clip bbox'''
cx = max(0, min(cx, boundary[1]))
cy = max(0, min(cy, boundary[0]))
width = max(10, min(width, boundary[1]))
height = max(10, min(height, boundary[0]))
'''get new bbox'''
bbox_new = np.array([cx-width/2, cy-height/2, width, height])
return bbox_new
def test_sac(args=get_args()):
torch.set_num_threads(1) # we just need only one thread for NN
env = gym.make(args.task)
if args.task == 'Pendulum-v0':
env.spec.reward_threshold = -250
args.state_shape = env.observation_space.shape or env.observation_space.n
args.action_shape = env.action_space.shape or env.action_space.n
args.max_action = env.action_space.high[0]
# you can also use tianshou.env.SubprocVectorEnv
# train_envs = gym.make(args.task)
train_envs = DummyVectorEnv(
[lambda: gym.make(args.task) for _ in range(args.training_num)])
# test_envs = gym.make(args.task)
test_envs = DummyVectorEnv(
[lambda: gym.make(args.task) for _ in range(args.test_num)])
# seed
np.random.seed(args.seed)
torch.manual_seed(args.seed)
train_envs.seed(args.seed)
test_envs.seed(args.seed)
# model
net = Net(args.layer_num, args.state_shape, device=args.device)
actor = ActorProb(net,
args.action_shape,
args.max_action,
args.device,
unbounded=True).to(args.device)
actor_optim = torch.optim.Adam(actor.parameters(), lr=args.actor_lr)
net_c1 = Net(args.layer_num,
args.state_shape,
args.action_shape,
concat=True,
device=args.device)
critic1 = Critic(net_c1, args.device).to(args.device)
critic1_optim = torch.optim.Adam(critic1.parameters(), lr=args.critic_lr)
net_c2 = Net(args.layer_num,
args.state_shape,
args.action_shape,
concat=True,
device=args.device)
critic2 = Critic(net_c2, args.device).to(args.device)
critic2_optim = torch.optim.Adam(critic2.parameters(), lr=args.critic_lr)
if args.model == 'ODEGBM':
model = ODEGBM(args).to(args.device)
elif args.model == 'PriorGBM':
model = PriorGBM(args).to(args.device)
elif args.model == 'NODAE':
model = NODAE(args).to(args.device)
else:
assert args.model == 'ODENet'
model = ODENet(args).to(args.device)
policy = SSACPolicy(
actor,
actor_optim,
critic1,
critic1_optim,
critic2,
critic2_optim,
model,
args,
action_range=[env.action_space.low[0], env.action_space.high[0]],
tau=args.tau,
gamma=args.gamma,
alpha=args.alpha,
reward_normalization=args.rew_norm,
ignore_done=args.ignore_done,
estimation_step=args.n_step)
# collector
train_collector = Collector(policy, train_envs,
ReplayBuffer(args.buffer_size))
test_collector = Collector(policy, test_envs)
# train_collector.collect(n_step=args.buffer_size)
# log
log_path = os.path.join(args.logdir, args.task, 'sac')
if args.baseline:
if not os.path.exists(log_path + '/baseline/'):
os.makedirs(log_path + '/baseline/')
writer = SummaryWriter(log_path + '/baseline')
else:
writer = SummaryWriter(log_path)
def save_fn(policy):
# torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth'))
pass
def stop_fn(mean_rewards):
# return mean_rewards >= env.spec.reward_threshold
return False
# trainer
result = offpolicy_trainer(policy,
train_collector,
test_collector,
args.epoch,
args.step_per_epoch,
args.collect_per_step,
args.test_num,
args.batch_size,
stop_fn=stop_fn,
save_fn=save_fn,
writer=writer)
# assert stop_fn(result['best_reward'])
if __name__ == '__main__':
pprint.pprint(result)
# Let's watch its performance!
env = gym.make(args.task)
policy.eval()
collector = Collector(policy, env)
result = collector.collect(n_episode=1, render=args.render)
print(f'Final reward: {result["rew"]}, length: {result["len"]}')
def _worker_loop(
data_reader,
batch_queue,
data_queue,
global_done_event,
worker_done_event,
seed,
init_fn,
worker_id,
):
# Intialize C side signal handlers for SIGBUS and SIGSEGV. Python signal
# module's handlers are executed after Python returns from C low-level
# handlers, likely when the same fatal signal happened again already.
# https://docs.python.org/3/library/signal.html Sec. 18.8.1.1
_set_worker_signal_handlers()
torch.set_num_threads(1)
random.seed(seed)
# TODO: numpy doesn't take seed bigger than INT32
# np.random.seed(seed)
torch.manual_seed(seed)
# Do not wait for putting thread to join when this worker exits. Otherwise,
# this worker may always be waiting to put and doesn't check batch_queue
# and global_done_event for termination signal.
data_queue.cancel_join_thread()
if init_fn is not None:
init_fn(worker_id)
watchdog = ManagerWatchdog()
shard = data_reader.get_shard(worker_id)
shard_itr = iter(shard)
shard_done = False
while True:
if shard_done:
# Wait until the main thread acknowledge the WorkerDone message or
# it signals shutdown.
if (
not watchdog.is_alive()
or global_done_event.is_set()
or worker_done_event.wait(0.1)
):
break
continue
try:
idx = batch_queue.get(timeout=MANAGER_STATUS_CHECK_INTERVAL)
except queue.Empty:
if watchdog.is_alive() and not global_done_event.is_set():
continue
else:
break
# use global_done_event so that we can get faster exiting signal even if there
# are still batches in batch_queue
if idx is None or global_done_event.is_set():
break
try:
samples = next(shard_itr)
except StopIteration:
# Signal to the main thread that this worker has run out of data.
# The worker cannot exit immediately because the queue might not be
# flushed immediately.
data_queue.put((idx, WorkerDone(worker_id)))
shard_done = True
except Exception:
data_queue.put((idx, ExceptionWrapper(sys.exc_info())))
else:
data_queue.put((idx, samples))
del samples
def main(dataset_name, net_name, xp_path, data_path, load_config, load_model,
eta, ratio_known_normal, ratio_known_outlier, ratio_pollution, device,
seed, optimizer_name, lr, n_epochs, lr_milestone, batch_size,
weight_decay, pretrain, ae_optimizer_name, ae_lr, ae_n_epochs,
ae_lr_milestone, ae_batch_size, ae_weight_decay, num_threads,
n_jobs_dataloader, normal_class, known_outlier_class,
n_known_outlier_classes):
"""
Deep SAD, a method for deep semi-supervised anomaly detection.
:arg DATASET_NAME: Name of the dataset to load.
:arg NET_NAME: Name of the neural network to use.
:arg XP_PATH: Export path for logging the experiment.
:arg DATA_PATH: Root path of data.
"""
# Get configuration
cfg = Config(locals().copy())
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log_file = xp_path + '/log.txt'
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
# Print paths
logger.info('Log file is %s' % log_file)
logger.info('Data path is %s' % data_path)
logger.info('Export path is %s' % xp_path)
# Print experimental setup
logger.info('Dataset: %s' % dataset_name)
logger.info('Normal class: %d' % normal_class)
logger.info('Ratio of labeled normal train samples: %.2f' %
ratio_known_normal)
logger.info('Ratio of labeled anomalous samples: %.2f' %
ratio_known_outlier)
logger.info('Pollution ratio of unlabeled train data: %.2f' %
ratio_pollution)
if n_known_outlier_classes == 1:
logger.info('Known anomaly class: %d' % known_outlier_class)
else:
logger.info('Number of known anomaly classes: %d' %
n_known_outlier_classes)
logger.info('Network: %s' % net_name)
# If specified, load experiment config from JSON-file
if load_config:
cfg.load_config(import_json=load_config)
logger.info('Loaded configuration from %s.' % load_config)
# Print model configuration
logger.info('Eta-parameter: %.2f' % cfg.settings['eta'])
# Set seed
if cfg.settings['seed'] != -1:
random.seed(cfg.settings['seed'])
np.random.seed(cfg.settings['seed'])
torch.manual_seed(cfg.settings['seed'])
torch.cuda.manual_seed(cfg.settings['seed'])
torch.backends.cudnn.deterministic = True
logger.info('Set seed to %d.' % cfg.settings['seed'])
# Default device to 'cpu' if cuda is not available
if not torch.cuda.is_available():
device = 'cpu'
# Set the number of threads used for parallelizing CPU operations
if num_threads > 0:
torch.set_num_threads(num_threads)
logger.info('Computation device: %s' % device)
logger.info('Number of threads: %d' % num_threads)
logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
# Load data
dataset = load_dataset(dataset_name,
data_path,
normal_class,
known_outlier_class,
n_known_outlier_classes,
ratio_known_normal,
ratio_known_outlier,
ratio_pollution,
random_state=np.random.RandomState(
cfg.settings['seed']))
# Log random sample of known anomaly classes if more than 1 class
if n_known_outlier_classes > 1:
logger.info('Known anomaly classes: %s' %
(dataset.known_outlier_classes, ))
# Initialize DeepSAD model and set neural network phi
deepSAD = DeepSAD(cfg.settings['eta'])
deepSAD.set_network(net_name)
# If specified, load Deep SAD model (center c, network weights, and possibly autoencoder weights)
if load_model:
deepSAD.load_model(model_path=load_model,
load_ae=True,
map_location=device)
logger.info('Loading model from %s.' % load_model)
logger.info('Pretraining: %s' % pretrain)
if pretrain:
# Log pretraining details
logger.info('Pretraining optimizer: %s' %
cfg.settings['ae_optimizer_name'])
logger.info('Pretraining learning rate: %g' % cfg.settings['ae_lr'])
logger.info('Pretraining epochs: %d' % cfg.settings['ae_n_epochs'])
logger.info('Pretraining learning rate scheduler milestones: %s' %
(cfg.settings['ae_lr_milestone'], ))
logger.info('Pretraining batch size: %d' %
cfg.settings['ae_batch_size'])
logger.info('Pretraining weight decay: %g' %
cfg.settings['ae_weight_decay'])
# Pretrain model on dataset (via autoencoder)
deepSAD.pretrain(dataset,
optimizer_name=cfg.settings['ae_optimizer_name'],
lr=cfg.settings['ae_lr'],
n_epochs=cfg.settings['ae_n_epochs'],
lr_milestones=cfg.settings['ae_lr_milestone'],
batch_size=cfg.settings['ae_batch_size'],
weight_decay=cfg.settings['ae_weight_decay'],
device=device,
n_jobs_dataloader=n_jobs_dataloader)
# Save pretraining results
deepSAD.save_ae_results(export_json=xp_path + '/ae_results.json')
# Log training details
logger.info('Training optimizer: %s' % cfg.settings['optimizer_name'])
logger.info('Training learning rate: %g' % cfg.settings['lr'])
logger.info('Training epochs: %d' % cfg.settings['n_epochs'])
logger.info('Training learning rate scheduler milestones: %s' %
(cfg.settings['lr_milestone'], ))
logger.info('Training batch size: %d' % cfg.settings['batch_size'])
logger.info('Training weight decay: %g' % cfg.settings['weight_decay'])
# Train model on dataset
deepSAD.train(dataset,
optimizer_name=cfg.settings['optimizer_name'],
lr=cfg.settings['lr'],
n_epochs=cfg.settings['n_epochs'],
lr_milestones=cfg.settings['lr_milestone'],
batch_size=cfg.settings['batch_size'],
weight_decay=cfg.settings['weight_decay'],
device=device,
n_jobs_dataloader=n_jobs_dataloader)
# Test model
deepSAD.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
# Save results, model, and configuration
deepSAD.save_results(export_json=xp_path + '/results.json')
deepSAD.save_model(export_model=xp_path + '/model.tar')
cfg.save_config(export_json=xp_path + '/config.json')
# Plot most anomalous and most normal test samples
indices, labels, scores = zip(*deepSAD.results['test_scores'])
indices, labels, scores = np.array(indices), np.array(labels), np.array(
scores)
idx_all_sorted = indices[np.argsort(
scores)] # from lowest to highest score
idx_normal_sorted = indices[labels == 0][np.argsort(
scores[labels == 0])] # from lowest to highest score
if dataset_name in ('mnist', 'fmnist', 'cifar10'):
if dataset_name in ('mnist', 'fmnist'):
X_all_low = dataset.test_set.data[idx_all_sorted[:32],
...].unsqueeze(1)
X_all_high = dataset.test_set.data[idx_all_sorted[-32:],
...].unsqueeze(1)
X_normal_low = dataset.test_set.data[idx_normal_sorted[:32],
...].unsqueeze(1)
X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:],
...].unsqueeze(1)
if dataset_name == 'cifar10':
X_all_low = torch.tensor(
np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...],
(0, 3, 1, 2)))
X_all_high = torch.tensor(
np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...],
(0, 3, 1, 2)))
X_normal_low = torch.tensor(
np.transpose(
dataset.test_set.data[idx_normal_sorted[:32], ...],
(0, 3, 1, 2)))
X_normal_high = torch.tensor(
np.transpose(
dataset.test_set.data[idx_normal_sorted[-32:], ...],
(0, 3, 1, 2)))
plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2)
plot_images_grid(X_all_high,
export_img=xp_path + '/all_high',
padding=2)
plot_images_grid(X_normal_low,
export_img=xp_path + '/normals_low',
padding=2)
plot_images_grid(X_normal_high,
export_img=xp_path + '/normals_high',
padding=2)
def set_num_threads(self, num: int) -> None:
"""
Sets number of OpenMP threads to use.
"""
torch.set_num_threads(num)
def main():
args = get_args()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.set_num_threads(1)
device = torch.device("cuda:0" if args.cuda else "cpu")
envs = make_env(args.env_name, args.seed, args.gamma)
model = MujocoModel(envs.observation_space.shape[0],
envs.action_space.shape[0])
model.to(device)
algorithm = PPO(model,
args.clip_param,
args.value_loss_coef,
args.entropy_coef,
initial_lr=args.lr,
eps=args.eps,
max_grad_norm=args.max_grad_norm)
agent = MujocoAgent(algorithm, device)
rollouts = RolloutStorage(args.num_steps, envs.observation_space.shape[0],
envs.action_space.shape[0])
obs = envs.reset()
rollouts.obs[0] = np.copy(obs)
episode_rewards = deque(maxlen=10)
num_updates = int(args.num_env_steps) // args.num_steps
for j in range(num_updates):
if args.use_linear_lr_decay:
# decrease learning rate linearly
utils.update_linear_schedule(algorithm.optimizer, j, num_updates,
args.lr)
for step in range(args.num_steps):
# Sample actions
with torch.no_grad():
value, action, action_log_prob = agent.sample(
rollouts.obs[step]) # why use obs from rollouts???有病吧
# Obser reward and next obs
obs, reward, done, infos = envs.step(action)
for info in infos:
if 'episode' in info.keys():
episode_rewards.append(info['episode']['r'])
# If done then clean the history of observations.
masks = torch.FloatTensor([[0.0] if done_ else [1.0]
for done_ in done])
bad_masks = torch.FloatTensor(
[[0.0] if 'bad_transition' in info.keys() else [1.0]
for info in infos])
rollouts.append(obs, action, action_log_prob, value, reward, masks,
bad_masks)
with torch.no_grad():
next_value = agent.value(rollouts.obs[-1])
value_loss, action_loss, dist_entropy = agent.learn(
next_value, args.gamma, args.gae_lambda, args.ppo_epoch,
args.num_mini_batch, rollouts)
rollouts.after_update()
if j % args.log_interval == 0 and len(episode_rewards) > 1:
total_num_steps = (j + 1) * args.num_steps
print(
"Updates {}, num timesteps {},\n Last {} training episodes: mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}\n"
.format(j, total_num_steps, len(episode_rewards),
np.mean(episode_rewards), np.median(episode_rewards),
np.min(episode_rewards), np.max(episode_rewards),
dist_entropy, value_loss, action_loss))
if (args.eval_interval is not None and len(episode_rewards) > 1
and j % args.eval_interval == 0):
ob_rms = utils.get_vec_normalize(envs).ob_rms
eval_mean_reward = evaluate(agent, ob_rms, args.env_name,
args.seed, device)
exit()
# train or predict
if args.predict is not None:
label = train_ALL_CNN.predict(args.predict, model, text_field, label_field)
print('\n[Text] {}[Label] {}\n'.format(args.predict, label))
elif args.test:
try:
print(test_iter)
train_ALL_CNN.test_eval(test_iter, model, args)
except Exception as e:
print("\nSorry. The test dataset doesn't exist.\n")
else:
print("\n cpu_count \n", mu.cpu_count())
torch.set_num_threads(args.num_threads)
if os.path.exists("./Test_Result.txt"):
os.remove("./Test_Result.txt")
if args.CNN:
print("CNN training start......")
model_count = train_ALL_CNN.train(train_iter, dev_iter, test_iter, model, args)
elif args.DEEP_CNN:
print("DEEP_CNN training start......")
model_count = train_ALL_CNN.train(train_iter, dev_iter, test_iter, model, args)
elif args.LSTM:
print("LSTM training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, args)
elif args.GRU:
print("GRU training start......")
model_count = train_ALL_LSTM.train(train_iter, dev_iter, test_iter, model, args)
elif args.BiLSTM:
cfg.DEBUG = args.debug
if cfg.LABEL_MAP != '':
_, weights = Dataset.load_mapping(cfg.LABEL_MAP)
cfg.NORMALIZE_WEIGHTS = []
for weight in weights:
if weight > 0:
cfg.NORMALIZE_WEIGHTS.append(weight)
cfg.NUM_CLASSES = len(cfg.NORMALIZE_WEIGHTS)
print('Using configs:')
pprint.pprint(cfg)
# ------------------------------------
# gpu
# ------------------------------------
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
torch.set_num_threads(args.num_workers)
# ------------------------------------
# experiment dir
# ------------------------------------
args.output_dir = os.path.join(args.output_dir,
'{}-{}'.format(args.cfg, args.tag) if args.tag is not None
else args.cfg)
os.makedirs(args.output_dir, exist_ok=True)
args.cfg = args.cfg.split('/')[-1]
if not os.path.isfile(os.path.join(args.output_dir, args.cfg + '.yml')):
cfg_to_file(os.path.join(args.output_dir, args.cfg + '.yml'))
# ------------------------------------
# train or eval
# -*- coding: utf-8 -*- import torch from torchtext import data import os import random torch.set_num_threads(2) torch.manual_seed(233) random.seed(233) import numpy as np import argparse import datetime import classification_datasets import datasets_char import data_five import word_embedding_loader as loader import Bi_lstm_2DCNN import LSTM import Bi_LSTM import CNN_LSTM import CNN_LSTM_Parallel import multi_CNN import multichannel_CNN import CNN import CNN_char import train_lstm import train_cnn import train_cnn_char
import torch
import numpy as np
# General torch settings and devices.
torch.set_num_threads(8)
gpu = torch.device('cuda:0')
cpu = torch.device('cpu')
from timeit import default_timer as timer
# This is a flag to tell the R interface which covariance model is being use.
# TODO: Implement a Kernel class to make things cleaner.
KERNEL_FAMILY = "squared exponential"
def compute_cov_pushforward(lambda0,
F,
cells_coords,
device=None,
n_chunks=200,
n_flush=50):
""" Compute the covariance pushforward.
The covariance pushforward is just KF^T, where K is the model
covariance matrix.
Note that the sigam0^2 is not included, and one has to manually add it when
using the covariance pushforward computed here.
Parameters
----------
args.mode = 'atj1_default_param'
test_flag = True
else:
test_flag = False
assert args.mode in [
'atj0_default_param', 'atj1_default_param', 'atj2_default_param',
'atj0_bayes_opt', 'atj1_bayes_opt', 'atj2_bayes_opt',
'atj1_random_bayes_opt', 'atj2_random_bayes_opt', 'lgbm_default_param',
'lgbm_bayes_opt', 'kdd_winner'
], 'invalid mode'
model_func = eval(args.mode)
total_start_time = time.time()
results, stats = [], []
strategy_name = f'{args.preprocess}_{args.mode}' if not test_flag else 'test'
torch.set_num_threads(CONSTANT.JOBS)
for database_path in args.database_paths.split(','):
database_path = os.path.expanduser(database_path)
database_name = os.path.basename(database_path.rstrip(os.path.altsep))
output_path = os.path.join(CONSTANT.ROOT_PATH, database_name,
strategy_name)
print('Output to dir:', output_path)
os.makedirs(output_path, exist_ok=True)
with Logger(output_path, git_log=True) as logger:
database = Database(database_path, output_path)
database.preprocess(args.preprocess)
database.stat(brief=True)
database.save_data()
database.freeze()
def main():
seeding()
# number of parallel agents
number_of_agents = 2
# number of training episodes.
# change this to higher number to experiment. say 30000.
number_of_episodes = 5000
max_t = 1000
batchsize = 128
# amplitude of OU noise
# this slowly decreases to 0
noise = 1
noise_reduction = 0.9999
tau = 1e-3 # soft update factor
gamma = 0.99 # reward discount factor
# how many episodes before update
episode_per_update = 2
model_dir = os.getcwd() + "/model_dir"
os.makedirs(model_dir, exist_ok=True)
# do we need to set multi-thread for this env?
torch.set_num_threads(number_of_agents * 2)
env = TennisEnv()
# keep 5000 episodes worth of replay
buffer = ReplayBuffer(int(1e5))
# initialize policy and critic
maddpg = MADDPG(discount_factor=gamma, tau=tau)
# training loop
scores_window = deque(maxlen=100)
ep_scores = []
# when to save: use a dictionary to track if a model at a given score (key/10) has been saved.
save_on_scores = {
5: False,
6: False,
9: False,
10: False,
11: False,
12: False,
13: False,
14: False,
15: False,
16: False,
17: False,
18: False,
19: False,
20: False
}
agent0_reward = []
agent1_reward = []
for episode in range(0, number_of_episodes):
reward_this_episode = np.zeros((1, number_of_agents))
obs, obs_full, env_info = env.reset()
for agent in maddpg.maddpg_agent:
agent.noise.reset()
for episode_t in range(max_t):
# explore = only explore for a certain number of episodes
# action input needs to be transposed
#print('Obs:', obs)
actions = maddpg.act(torch.tensor(obs, dtype=torch.float),
noise=noise)
#print(actions)
#if noise>0.01:
noise *= noise_reduction
actions_for_env = torch.stack(actions).detach().numpy()
# step forward one frame
next_obs, next_obs_full, rewards, dones, info = env.step(
actions_for_env)
# add data to buffer
buffer.push(obs, obs_full, actions_for_env, rewards, next_obs,
next_obs_full, dones)
reward_this_episode += rewards
obs = np.copy(next_obs)
obs_full = np.copy(next_obs_full)
# update once after every episode_per_update
if len(
buffer
) > batchsize and episode > 0 and episode % episode_per_update == 0:
for a_i in range(number_of_agents):
samples = buffer.sample(batchsize)
maddpg.update(samples, a_i)
if np.any(dones):
break
agent0_reward.append(reward_this_episode[0, 0])
agent1_reward.append(reward_this_episode[0, 1])
avg_rewards = max(reward_this_episode[0, 0], reward_this_episode[0, 1])
scores_window.append(avg_rewards)
cur_score = np.mean(scores_window)
ep_scores.append(cur_score)
print(
'\rEpisode:{}, Rwd:{:.3f} vs. {:.3f}, Average Score:{:.4f}, Noise:{:.4f}'
.format(episode, reward_this_episode[0, 0],
reward_this_episode[0, 1], cur_score, noise))
#saving model
save_dict_list = []
save_info = False
score_code = int(cur_score * 10)
if score_code in save_on_scores.keys():
if not (save_on_scores[score_code]):
save_on_scores[score_code] = True
save_info = True
if save_info:
for i in range(number_of_agents):
save_dict = {
'actor_params':
maddpg.maddpg_agent[i].actor.state_dict(),
'actor_optim_params':
maddpg.maddpg_agent[i].actor_optimizer.state_dict(),
'critic_params':
maddpg.maddpg_agent[i].critic.state_dict(),
'critic_optim_params':
maddpg.maddpg_agent[i].critic_optimizer.state_dict()
}
save_dict_list.append(save_dict)
torch.save(
save_dict_list,
os.path.join(
model_dir,
'episode-{}-{}.pt'.format(episode, score_code)))
np.savez('scores-{}-{}.npz'.format(episode, score_code),
agent0_reward=np.array(agent0_reward),
agent1_reward=np.array(agent1_reward),
avg_max_scores=np.array(ep_scores))
env.close()
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--config", type=str, required=True)
parser.add_argument("--batch-size", type=int, default=1)
parser.add_argument("--steps-plot", type=int, default=50)
parser.add_argument("--threads", type=int, default=10)
parser.add_argument("--lr", type=float)
parser.add_argument("--momentum", type=float)
parser.add_argument("--occupy", dest="occupy", action="store_true")
parser.set_defaults(occupy=False)
params = parser.parse_args()
# Number of threads
torch.set_num_threads(params.threads)
print("Running on GPU {}".format(os.environ["CUDA_VISIBLE_DEVICES"]))
config = ConfigObj(params.config, configspec="config/spec.cfg")
if not config:
raise RuntimeError(
"Could not load the configuration file: '{}'".format(
params.config))
validator = Validator()
ret = config.validate(validator)
if ret is not True:
for k in ret:
if not ret[k]:
print("--- {} is {}".format(k, str(ret[k])))
raise RuntimeError("Errors validating the training configuration file")
def main():
args = get_args()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
if args.cuda and torch.cuda.is_available() and args.cuda_deterministic:
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
log_dir = os.path.expanduser(args.log_dir)
eval_log_dir = log_dir + "_eval"
utils.cleanup_log_dir(log_dir)
utils.cleanup_log_dir(eval_log_dir)
# import pdb; pdb.set_trace()
save_path = os.path.join(args.save_dir, args.algo)
torch.set_num_threads(1)
device = torch.device("cuda:0" if args.cuda else "cpu")
envs = make_vec_envs(args.env_name, args.seed, args.num_processes,
args.gamma, args.log_dir, device, False)
# import pdb; pdb.set_trace()
actor_critic = Policy(envs.observation_space.shape,
envs.action_space,
base_kwargs={'recurrent': args.recurrent_policy})
# transforms = [ hl.transforms.Prune('Constant') ] # Removes Constant nodes from graph.
# graph = hl.build_graph(actor_critic, torch.zeros([1, 1, 64, 64]), transforms=transforms)
# graph.theme = hl.graph.THEMES['blue'].copy()
# graph.save('rnn_hiddenlayer2', format='png')
# print(args.re)
# import pdb; pdb.set_trace()
my_model_state_dict = actor_critic.state_dict()
count = 0
pretrained_weights = torch.load('net_main_4rh_v2_64.pth')
# pretrained_weights = torch.load(os.path.join(save_path, args.env_name + "_ft.pt"))
# pretrained_weights['']
old_names = list(pretrained_weights.items())
pretrained_weights_items = list(pretrained_weights.items())
for key, value in my_model_state_dict.items():
layer_name, weights = pretrained_weights_items[count]
my_model_state_dict[key] = weights
print(count)
print(layer_name)
count += 1
if layer_name == 'enc_dense.bias':
break
# pretrained_weights = torch.load(os.path.join(save_path, args.env_name + "_random.pt"))[1]
actor_critic.load_state_dict(my_model_state_dict)
start_epoch = 0
ka = 0
# for param in actor_critic.parameters():
# ka += 1
# # import pdb; pdb.set_trace()
# param.requires_grad = False
# if ka == 14:
# break
count = 0
# import pdb; pdb.set_trace()n
actor_critic.to(device)
if args.algo == 'a2c':
agent = algo.A2C_ACKTR(actor_critic,
args.value_loss_coef,
args.entropy_coef,
lr=args.lr,
eps=args.eps,
alpha=args.alpha,
max_grad_norm=args.max_grad_norm)
elif args.algo == 'ppo':
agent = algo.PPO(actor_critic,
args.clip_param,
args.ppo_epoch,
args.num_mini_batch,
args.value_loss_coef,
args.entropy_coef,
lr=args.lr,
eps=args.eps,
max_grad_norm=args.max_grad_norm)
elif args.algo == 'acktr':
agent = algo.A2C_ACKTR(actor_critic,
args.value_loss_coef,
args.entropy_coef,
acktr=True)
if args.gail:
assert len(envs.observation_space.shape) == 1
discr = gail.Discriminator(
envs.observation_space.shape[0] + envs.action_space.shape[0], 100,
device)
file_name = os.path.join(
args.gail_experts_dir,
"trajs_{}.pt".format(args.env_name.split('-')[0].lower()))
expert_dataset = gail.ExpertDataset(file_name,
num_trajectories=4,
subsample_frequency=20)
drop_last = len(expert_dataset) > args.gail_batch_size
gail_train_loader = torch.utils.data.DataLoader(
dataset=expert_dataset,
batch_size=args.gail_batch_size,
shuffle=True,
drop_last=drop_last)
rollouts = RolloutStorage(args.num_steps, args.num_processes,
envs.observation_space.shape, envs.action_space,
actor_critic.recurrent_hidden_state_size)
obs = envs.reset()
rollouts.obs[0].copy_(obs)
rollouts.to(device)
episode_rewards = deque(maxlen=10)
start = time.time()
rewards_mean = []
rewards_median = []
val_loss = []
act_loss = []
num_updates = int(
args.num_env_steps) // args.num_steps // args.num_processes
for j in range(start_epoch, num_updates):
if args.use_linear_lr_decay:
# decrease learning rate linearly
utils.update_linear_schedule(
agent.optimizer, j, num_updates,
agent.optimizer.lr if args.algo == "acktr" else args.lr)
for step in range(args.num_steps):
# Sample actions
with torch.no_grad():
value, action, action_log_prob, recurrent_hidden_states = actor_critic.act(
rollouts.obs[step], rollouts.recurrent_hidden_states[step],
rollouts.masks[step])
# Obser reward and next obs
obs, reward, done, infos = envs.step(action)
for info in infos:
if 'episode' in info.keys():
episode_rewards.append(info['episode']['r'])
# If done then clean the history of observations.
masks = torch.FloatTensor([[0.0] if done_ else [1.0]
for done_ in done])
bad_masks = torch.FloatTensor(
[[0.0] if 'bad_transition' in info.keys() else [1.0]
for info in infos])
rollouts.insert(obs, recurrent_hidden_states, action,
action_log_prob, value, reward, masks, bad_masks)
with torch.no_grad():
next_value = actor_critic.get_value(
rollouts.obs[-1], rollouts.recurrent_hidden_states[-1],
rollouts.masks[-1]).detach()
if args.gail:
if j >= 10:
envs.venv.eval()
gail_epoch = args.gail_epoch
if j < 10:
gail_epoch = 100 # Warm up
for _ in range(gail_epoch):
discr.update(gail_train_loader, rollouts,
utils.get_vec_normalize(envs)._obfilt)
for step in range(args.num_steps):
rollouts.rewards[step] = discr.predict_reward(
rollouts.obs[step], rollouts.actions[step], args.gamma,
rollouts.masks[step])
rollouts.compute_returns(next_value, args.use_gae, args.gamma,
args.gae_lambda, args.use_proper_time_limits)
value_loss, action_loss, dist_entropy = agent.update(rollouts)
rollouts.after_update()
# save for every interval-th episode or for the last epoch
if (j % args.save_interval == 0
or j == num_updates - 1) and args.save_dir != "":
save_path = os.path.join(args.save_dir, args.algo)
try:
os.makedirs(save_path)
except OSError:
pass
torch.save([
actor_critic,
actor_critic.state_dict(),
getattr(utils.get_vec_normalize(envs), 'ob_rms', None)
], os.path.join(save_path, args.env_name + "_finetune.pt"))
if j % args.log_interval == 0 and len(episode_rewards) > 1:
total_num_steps = (j + 1) * args.num_processes * args.num_steps
end = time.time()
print(
"Updates {}, num timesteps {}, FPS {} \n Last {} training episodes: mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}\n"
.format(j, total_num_steps,
int(total_num_steps / (end - start)),
len(episode_rewards), np.mean(episode_rewards),
np.median(episode_rewards), np.min(episode_rewards),
np.max(episode_rewards), dist_entropy, value_loss,
action_loss))
rewards_mean.append(np.mean(episode_rewards))
rewards_median.append(np.median(episode_rewards))
val_loss.append(value_loss)
act_loss.append(action_loss)
torch.save(
rewards_mean,
"./plot_data/" + args.env_name + "_avg_rewards_finetune.pt")
torch.save(
rewards_median,
"./plot_data/" + args.env_name + "_median_rewards_finetune.pt")
# torch.save(val_loss, "./plot_data/"+args.env_name+"_val_loss_enc_weights.pt")
# torch.save(act_loss, "./plot_data/"+args.env_name+"_act_loss_enc_weights.pt")
plt.plot(rewards_mean)
# print(plt_points2)
plt.savefig("./imgs/" + args.env_name + "avg_reward_finetune.png")
# plt.show(block = False)
if (args.eval_interval is not None and len(episode_rewards) > 1
and j % args.eval_interval == 0):
ob_rms = utils.get_vec_normalize(envs).ob_rms
evaluate(actor_critic, ob_rms, args.env_name, args.seed,
args.num_processes, eval_log_dir, device)
