def main():
device = torch.device('cpu')
torch.manual_seed(1234)
layer = 2
size = 1024
func = "relu"
model = MyModel(layer, size)
model.load_state_dict(torch.load('model_state_dict_final'))
model.eval()
transforms = T.Compose([T.ToTensor(), T.Normalize((0.5, ), (0.5, ))])
test_dataset = MnistDatasetTest('data', 'test', transforms)
test_dataloader = DataLoader(test_dataset,
batch_size=64,
shuffle=False,
num_workers=4)
result = open("result.txt", "w") # to create a result
with torch.no_grad():
for images, image_name in test_dataloader:
images = images.to(device)
prediction = model(images, layer, func)
for i in range(images.size()[0]):
x = image_name[i] + ' ' + str(int(torch.argmax(prediction[i])))
result.write(x)
result.write("\n")
result.close()
def _load(self, filePath):
checkpoint = torch.load(filePath)
model = MyModel(
device, checkpoint['inputSize'], checkpoint['gatedCnnOutputSize'],
checkpoint['gatedCnnStride1'], checkpoint['gatedCnnStride2'],
checkpoint['gatedCnnKernel1'], checkpoint['gatedCnnKernel2'],
checkpoint['lstmLayer'], checkpoint['lstmHiddenSize'],
checkpoint['fcOutputSize'], checkpoint['dropout'])
model.load_state_dict(checkpoint['stateDict'])
model.eval()
if self.device.type == 'cpu':
model.cpu()
else:
model.cuda(device=self.device)
return model
def run():
use_cuda = torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
dataset = [
{"in" : [0.0, 0.0], "out" : [0.0]},
{"in" : [0.0, 1.0], "out" : [1.0]},
{"in" : [1.0, 0.0], "out" : [1.0]},
{"in" : [1.0, 1.0], "out" : [0.0]}]
data_loader = get_data_loader(dataset=dataset, shuffle=True)
model = MyModel()
model.to(device)
optimizer = optim.Adam(model.parameters())
loss_func = nn.MSELoss()
model.train()
for _ in range(0, 2000):
for output, input in data_loader:
input = input.to(device)
output = output.to(device)
result = model(input)
loss = loss_func(result, output)
optimizer.zero_grad()
loss.backward()
optimizer.step()
test_data_loader = get_data_loader(dataset=dataset, shuffle=False)
model.eval()
with torch.no_grad():
for _, input in test_data_loader:
print(input)
input = input.to(device)
result = model(input)
print(result)
def train(batch_size=16,
pretrain_model_path='',
name='',
model_type='mlp',
dim=1024,
lr=1e-5,
epoch=12,
smoothing=0.05,
sample=False,
open_ad='',
dialog_name='xxx'):
if not pretrain_model_path or not name:
assert 1==-1
# print('\n********** model type:', model_type, '**********')
# print('batch_size:', batch_size)
# load dataset
train_file = '../data/Dataset/my_train.csv'
dev_file = '../data/Dataset/my_dev.csv'
train_num = len(pd.read_csv(train_file).values.tolist())
val_num = len(pd.read_csv(dev_file).values.tolist())
print('train_num: %d, dev_num: %d' % (train_num, val_num))
# 选择模型
if model_type in ['siam', 'esim', 'sbert']:
assert 1==-1
else:
train_iter = MyDataset(file=train_file, is_train=True, sample=sample, pretrain_model_path=pretrain_model_path)
train_iter = get_dataloader(train_iter, batch_size, shuffle=True, drop_last=True)
dev_iter = MyDataset(file=dev_file, is_train=True, sample=sample, pretrain_model_path=pretrain_model_path)
dev_iter = get_dataloader(dev_iter, batch_size, shuffle=False, drop_last=False)
if model_type == 'mlp':
model = MyModel(dim=dim, pretrain_model_path=pretrain_model_path, smoothing=smoothing)
elif model_type == 'cnn':
model = MyTextCNNModel(dim=dim, pretrain_model_path=pretrain_model_path, smoothing=smoothing)
elif model_type == 'rcnn':
model = MyRCNNModel(dim=dim, pretrain_model_path=pretrain_model_path, smoothing=smoothing)
model.to(device)
model_param_num = 0
##### 3.24 muppti-gpu-training
if n_gpu > 1:
model = torch.nn.DataParallel(model)
for p in model.parameters():
if p.requires_grad:
model_param_num += p.nelement()
print('param_num:%d\n' % model_param_num)
# 加入对抗训练，提升泛化能力；但是训练速度明显变慢 (插件式调用)
# 3.12 change to FGM 更快！
if open_ad == 'fgm':
fgm = FGM(model)
elif open_ad == 'pgd':
pgd = PGD(model)
K = 3
# model-store-path
model_path = '../user_data/model_store/' + name + '.pkl' # 输出模型默认存放在当前路径下
state = {}
time0 = time.time()
best_loss = 999
early_stop = 0
for e in range(epoch):
print("*" * 100)
print("Epoch:", e)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=lr,
warmup=0.05,
t_total=len(train_iter)) # 设置优化器
train_loss = 0
train_c = 0
train_right_num = 0
model.train() # 将模型设置成训练模式（Sets the module in training mode）
print('training..., %s, e:%d, lr:%7f' % (name, e, lr))
for batch in tqdm(train_iter): # 每一次返回 batch_size 条数据
optimizer.zero_grad() # 清空梯度
batch = [b.cuda() for b in batch] # cpu -> GPU
# 正常训练
labels = batch[-1].view(-1).cpu().numpy()
loss, bert_enc = model(batch, task='train', epoch=epoch) # 进行前向传播，真正开始训练；计算 loss
right_num = count_right_num(bert_enc, labels)
# multi-gpu training!
if n_gpu > 1:
loss = loss.mean()
loss.backward() # 反向传播计算参数的梯度
if open_ad == 'fgm':
# 对抗训练
fgm.attack() # 在embedding上添加对抗扰动
if model_type == 'multi-task': loss_adv, _, _ = model(batch, task='train')
else: loss_adv, _ = model(batch, task='train')
if n_gpu > 1:
loss_adv = loss_adv.mean()
loss_adv.backward() # 反向传播，并在正常的grad基础上，累加对抗训练的梯度
fgm.restore() # 恢复embedding参数
elif open_ad == 'pgd':
pgd.backup_grad()
# 对抗训练
for t in range(K):
pgd.attack(is_first_attack=(t==0)) # 在embedding上添加对抗扰动, first attack时备份param.data
if t != K-1:
optimizer.zero_grad()
else:
pgd.restore_grad()
if model_type == 'multi-task': loss_adv, _, _ = model(batch, task='train')
else: loss_adv, _ = model(batch, task='train')
if n_gpu > 1:
loss_adv = loss_adv.mean()
loss_adv.backward() # 反向传播，并在正常的grad基础上，累加对抗训练的梯度
pgd.restore() # 恢复embedding参数
optimizer.step() # 更新参数
train_loss += loss.item() # loss 求和
train_c += 1
train_right_num += right_num
val_loss = 0
val_c = 0
val_right_num = 0
model.eval()
print('eval...')
with torch.no_grad(): # 不进行梯度的反向传播
for batch in tqdm(dev_iter): # 每一次返回 batch_size 条数据
batch = [b.cuda() for b in batch]
labels = batch[-1].view(-1).cpu().numpy()
loss, bert_enc = model(batch, task='train', epoch=epoch) # 进行前向传播，真正开始训练；计算 loss
right_num = count_right_num(bert_enc, labels)
if n_gpu > 1:
loss = loss.mean()
val_c += 1
val_loss += loss.item()
val_right_num += right_num
train_acc = train_right_num / train_num
val_acc = val_right_num / val_num
print('train_acc: %.4f, val_acc: %.4f' % (train_acc, val_acc))
print('train_loss: %.4f, val_loss: %.4f, time: %d' % (train_loss/train_c, val_loss/val_c, time.time()-time0))
if val_loss / val_c < best_loss:
early_stop = 0
best_loss = val_loss / val_c
best_acc = val_acc
# 3.24 update 多卡训练时模型保存避坑:
model_to_save = model.module if hasattr(model, 'module') else model
state['model_state'] = model_to_save.state_dict()
state['loss'] = val_loss / val_c
state['acc'] = val_acc
state['e'] = e
state['time'] = time.time() - time0
state['lr'] = lr
torch.save(state, model_path)
best_epoch = e
cost_time = time.time() - time0
tmp_train_acc = train_acc
best_model = model
else:
early_stop += 1
if early_stop == 2:
break
model = best_model
lr = lr * 0.5
print("best_loss:", best_loss)
# 3.12 add 打印显示最终的最优结果
print('-' * 30)
print('best_epoch:', best_epoch, 'best_loss:', best_loss, 'best_acc:', best_acc, 'reach time:', cost_time, '\n')
# model-clean
del model
gc.collect()
# 实验结果写入日志
with open('../user_data/model_dialog/' + dialog_name + '.out', 'w', encoding='utf-8') as f:
f.write('*** model name:' + dialog_name + ' ***\n')
f.write('best dev acc:' + str(best_acc) + '\n')
f.write('best loss:' + str(best_loss) + '\n')
f.write('best_epoch:' + str(best_epoch) + '\n')
f.write('train acc:' + str(tmp_train_acc) + '\n')
f.write('lr:' + str(state['lr']) + '\n')
f.write('time:' + str(cost_time) + '\n')
def infer(PATH_IMG_INFER, PATH_MODEL_PARAMS, PATH_INFER_SAVE, DEPTH,
NUM_CHANNELS, MULT_CHAN, NUM_CLASSES, Z_RANGE, IMG_WIDTH,
IMG_HEIGHT):
### instantiate model and load the model parameters
my_model = MyModel(n_in_channels=NUM_CHANNELS,
mult_chan=MULT_CHAN,
depth=DEPTH)
my_model.load_state_dict(
torch.load(PATH_MODEL_PARAMS, map_location=torch.device('cpu')))
my_model.eval()
### read in the tiff and place into a ZStack object
stack = ZStack(path=PATH_IMG_INFER, zrange=Z_RANGE)
# separate zslices; assign each to its own Slice object, and hold them all in a list
slices = [Slice(x) for x in stack.images]
# split each slice into tiles
tiles = np.array(
[slices[z].tile(IMG_WIDTH, IMG_HEIGHT) for z in range(len(slices))])
# normalize (z-score) the tiles before passing to model
[[tiles[y][x].normalize() for x in range(0, tiles.shape[1])]
for y in range(0, tiles.shape[0])]
# reshape data; h=z_slices, w=numtiles for each z-slice
h, w = np.shape(tiles)
flat_tiles = np.zeros((h * w, 1, IMG_WIDTH, IMG_HEIGHT), dtype=np.float32)
for i in range(h * w):
flat_tiles[i] = tiles.flatten(order='C')[i].reshape(
IMG_WIDTH, IMG_HEIGHT).slice
# load data into pytorch data structs
dataset = FormsDataset(flat_tiles, num_classes=NUM_CLASSES)
hold = []
data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
for i, images in enumerate(data_loader, 1):
images = images.type(torch.FloatTensor)
hold.append(images)
# now lets perform inference on the tiles, store them in 'predictions'
predictions = []
for tile in hold:
predictions.append(my_model(tile))
# and lets time ourselves
print("--- Inference: {} seconds ---".format(time.time() - start_time))
### now we have to stitch our images back together!
unflat_predictions = np.array(predictions).reshape((h, w))
stack_infer = []
for z in unflat_predictions:
stack_infer.append(
stitch(z, IMG_WIDTH=IMG_WIDTH, IMG_HEIGHT=IMG_HEIGHT).toimage())
stack_infer[0].save(PATH_INFER_SAVE,
save_all=True,
append_images=stack_infer[1:])
print("--- Finish: {} seconds ---".format(time.time() - start_time))
watch_dataset(data_loader_train)
for epoch in range(options.args.epoch):
# train
model.train()
for data, label in data_loader_train:
data, label = data.to(device), label.to(device)
optimizer.zero_grad()
output = model(data)
loss = loss_func(output, label)
loss.backward()
optimizer.step()
print("loss: %f" % loss.item())
# test
model.eval()
acc = 0
with torch.no_grad():
for data, label in data_loader_test:
data, label = data.to(device), label.to(device)
output = model(data)
answer = torch.argmax(output, dim=-1)
acc += torch.sum(answer == label).item()
accuracy = acc / len(data_loader_test.dataset)
print("accuracy %.6f" % accuracy)
save_model(model, epoch, options)
def main(args):
# device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# tensorboard writer
writer = SummaryWriter(comment='#layers{}_emb{}_multitask{}'.format(args['num_hidden_layers'], args['embedding_size'], args['multitask_weights']))
# logger
logger = logging.getLogger('#layers{}_emb{}_multitask{}'.format(args['num_hidden_layers'], args['embedding_size'], args['multitask_weights'])) # experiment name
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler = logging.FileHandler("log/training_log.log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
# set random seed for reproducibility
torch.manual_seed(2019)
np.random.seed(2019)
# load data
data = utils.load_dataset(year=args['year'])
# trips -- [(src, dst, cnt)]
train_data = data['train']
valid_data = data['valid']
test_data = data['test']
# in/out flow counts -- [(count)]
train_inflow = data['train_inflow']
train_outflow = data['train_outflow']
# geographical features -- [[features]]
node_feats = data['node_feats']
# census tract adjacency as matrix
ct_adj = data['ct_adjacency_withweight']
# number of nodes
num_nodes = data['num_nodes']
# post-processing
# trip data
train_data = torch.from_numpy(train_data)
trip_od_train = train_data[:, :2].long().to(device)
trip_volume_train = train_data[:, -1].float().to(device)
trip_od_valid = torch.from_numpy(valid_data[:, :2]).long().to(device)
trip_volume_valid = torch.from_numpy(valid_data[:, -1]).float().to(device)
trip_od_test = torch.from_numpy(test_data[:, :2]).long().to(device)
trip_volume_test = torch.from_numpy(test_data[:, -1]).float().to(device)
# in/out flow data for multitask target in/out flow
train_inflow = torch.from_numpy(train_inflow).view(-1, 1).float().to(device)
train_outflow = torch.from_numpy(train_outflow).view(-1, 1).float().to(device)
# construct graph using adjacency matrix
g = utils.build_graph_from_matrix(ct_adj, node_feats.astype(np.float32), device)
g.to(device)
# init model
model = MyModel(g, num_nodes, in_dim = node_feats.shape[1], h_dim = args['embedding_size'], num_hidden_layers=args['num_hidden_layers'], dropout=0, device=device, reg_param=args['reg_param'])
model.to(device)
# training recorder
model_state_file = './models/model_state_layers{}_emb{}_multitask{}.pth'.format(args['num_hidden_layers'], args['embedding_size'], args['multitask_weights'])
best_rmse = 1e6
# create a optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=args['lr'])
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 100, gamma=0.1)
# training process
for epoch in range(args['max_epochs']):
# turn model state
model.train()
# create a mini-batch generator
mini_batch_gen = utils.mini_batch_gen(train_data, mini_batch_size = int(args['mini_batch_size']), num_nodes=num_nodes, negative_sampling_rate = 0)
# SGD from each mini-batch
for mini_batch in mini_batch_gen:
# clear gradients
optimizer.zero_grad()
# get trip od
trip_od = mini_batch[:, :2].long().to(device)
# get trip volume
scaled_trip_volume = utils.scale(mini_batch[:, -1].float()).to(device)
# evaluate loss
loss = model.get_loss(trip_od, scaled_trip_volume, train_inflow, train_outflow, g, multitask_weights=args['multitask_weights'])
# add to tensorboard
writer.add_scalar('mini_loss', loss.item(), global_step = epoch)
# back propagation to get gradients
loss.backward()
# clip to make stable
torch.nn.utils.clip_grad_norm_(model.parameters(), args['grad_norm'])
# update weights by optimizer
optimizer.step()
# scheduler update learning rate
scheduler.step()
# report training epoch
if logger.level == logging.DEBUG:
model.eval()
# loss function
with torch.no_grad():
loss = model.get_loss(trip_od_train, utils.scale(trip_volume_train), train_inflow, train_outflow, g)
# metric on train dataset
rmse, mae, mape, cpc, cpl = utils.evaluate(model, g, trip_od_train, trip_volume_train)
# report
logger.debug("Evaluation on train dataset:")
logger.debug("Epoch {:04d} | Loss = {:.4f}".format(epoch, loss))
logger.debug("RMSE {:.4f} | MAE {:.4f} | MAPE {:.4f} | CPC {:.4f} | CPL {:.4f} |".format(rmse, mae, mape, cpc, cpl))
writer.add_scalar('overall-loss', loss.item(), epoch)
writer.add_scalar('RMSE', rmse, epoch)
writer.add_scalar('MAE', mae, epoch)
writer.add_scalar('MAPE', mape, epoch)
writer.add_scalar('CPC', cpc, epoch)
writer.add_scalar('CPL', cpl, epoch)
# validation part
if epoch % args['evaluate_every'] == 0:
# turn model state to eval
model.eval()
# loss function
with torch.no_grad():
loss = model.get_loss(trip_od_valid, utils.scale(trip_volume_valid), train_inflow, train_outflow, g)
# evaluate on validation set
rmse, mae, mape, cpc, cpl = utils.evaluate(model, g, trip_od_valid, trip_volume_valid)
# report
logger.info("-----------------------------------------")
logger.info("Evaluation on Validation:")
logger.info("Epoch {:04d} | Loss = {:.4f}".format(epoch, loss))
logger.info("RMSE {:.4f} | MAE {:.4f} | MAPE {:.4f} | CPC {:.4f} | CPL {:.4f} |".format(rmse, mae, mape, cpc, cpl))
# save best model
if rmse < best_rmse:
# update indicator
best_rmse = rmse
# save model
torch.save({'state_dict': model.state_dict(), 'epoch': epoch, 'rmse': rmse, 'mae': mae, 'mape': mape, 'cpc': cpc, 'cpl': cpl}, model_state_file)
# save embeddings
src_embedding = model(g).detach().cpu().numpy() # get embeddings
dst_embedding = model.forward2(g).detach().cpu().numpy() # get embeddings
emb_fp = "./embeddings/censustract_embeddings_year{}_layers{}_emb{}_multitask{}.npz".format(args['year'], args['num_hidden_layers'] , args['embedding_size'], args['multitask_weights'])
np.savez(emb_fp, src_embedding, dst_embedding)
# report
logger.info('Best RMSE found on epoch {}'.format(epoch))
logger.info("-----------------------------------------")
logger.histo_summary(tag,
value.data.cpu().numpy(),
epoch * total_step + i)
logger.histo_summary(tag + '/grad',
value.grad.data.cpu().numpy(),
epoch * total_step + i)
# 3. Log training images (image summary)
# info = {'images': joint.view(-1, 24, 2)[:10].cpu().numpy()}
# for tag, images in info.items():
# logger.image_summary(tag, images, step+1)
loss = 0
if epoch % 1 == 0:
# Test the model
model.eval(
) # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
with torch.no_grad():
for joint, pose in test_loader:
joint = joint.to(device)
pose = pose.to(device)
outputs = model(joint)
loss += criterion(outputs, pose)
loss = loss / len(test_loader)
print(
'Test Loss of the model on the {} test images: {:.4f} '.format(
total_step * batch_size, loss))
logger.scalar_summary('test_loss', loss.item(), epoch)
if loss < last_loss:
last_loss = loss
# Save the model checkpoint
torch.save(model.state_dict(),
def predict(dim,
names,
weight,
batch_size,
pretrain_model_path,
model_types=None):
print('-' * 100)
print('multi-models begin predicting ...')
print('-' * 100)
# read test data
test_file = '../data/Dataset/test.csv'
# data
test_df = pd.read_csv(test_file)
test_ids = test_df['id'].values.tolist()
result_prob_tmp = torch.zeros((len(test_ids), 2))
# load model
for i, name in enumerate(names):
# 3.17 add
weight_ = weight[i]
model_path = '../user_data/model_store/' + name + '.pkl'
state = torch.load(model_path)
# 3.10 add
model_type = model_types[i]
if model_type == 'mlp':
test_iter = MyDataset(file=test_file, is_train=False, pretrain_model_path=pretrain_model_path[i])
test_iter = get_dataloader(test_iter, batch_size, shuffle=False, drop_last=False)
model = MyModel(dim=dim[i], pretrain_model_path=pretrain_model_path[i])
elif model_type == 'cnn':
test_iter = MyDataset(file=test_file, is_train=False, pretrain_model_path=pretrain_model_path[i])
test_iter = get_dataloader(test_iter, batch_size, shuffle=False, drop_last=False)
model = MyTextCNNModel(dim=dim[i], pretrain_model_path=pretrain_model_path[i])
elif model_type == 'rcnn':
test_iter = MyDataset(file=test_file, is_train=False, pretrain_model_path=pretrain_model_path[i])
test_iter = get_dataloader(test_iter, batch_size, shuffle=False, drop_last=False)
model = MyRCNNModel(dim=dim[i], pretrain_model_path=pretrain_model_path[i])
model.to(device)
model.load_state_dict(state['model_state'])
model.eval()
print('-'*20, 'model', i, '-'*20)
print('load model:%s, loss:%.4f, e:%d, lr:%.7f, time:%d' %
(name, state['loss'], state['e'], state['lr'], state['time']))
# predict
with torch.no_grad():
j = 0
for batch in tqdm(test_iter):
batch = [b.cuda() for b in batch]
out = model(batch, task='eval')
out = out.cpu() # gpu -> cpu
if j == 0:
tmp = out # 初始化 tmp
else:
tmp = torch.cat([tmp, out], dim=0) # 将之后的预测结果拼接到 tmp 中
j += 1
# 当前 模型预测完成
print('model', i, 'predict finished!\n')
# 3.17 按权重融合
result_prob_tmp += (weight_ / len(names)) * tmp
# 删除模型
del model
gc.collect()
time.sleep(1)
# 3.10 当前融合策略：prob 简单的取 avg
_, result = torch.max(result_prob_tmp, dim=-1)
result = result.numpy()
# 3.16 update: label 0的prob 大于 3，就认为是 label=0
# with open('tmp.txt', 'w', encoding='utf-8') as f:
# for r in result_prob_tmp:
# f.write(str(r) + '\n')
# save result
df = pd.DataFrame()
df['id'] = test_ids
df['label'] = result
df.to_csv(("../prediction_result/result_"+datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + ".csv"), encoding='utf-8', index=False)
def extract_cache_features(cfg, epoch, loader, dataset, test_batch_size, uuids,
nls):
# extract img fts first to save time
if not os.path.exists('cache'):
# shutil.rmtree('cache')
os.mkdir('cache')
if not os.path.exists(f'cache/{epoch}'):
os.mkdir(f'cache/{epoch}')
model = MyModel(cfg,
len(dataset.nl),
dataset.nl.word_to_idx['<PAD>'],
nn.BatchNorm2d,
num_colors=len(CityFlowNLDataset.colors),
num_types=len(CityFlowNLDataset.vehicle_type) -
2).cuda()
saved_dict = torch.load(f'save/{epoch}.pth')
n = {}
for k, v in saved_dict.items():
n[k.replace('module.', '')] = v
model.load_state_dict(n, False)
model.eval()
with torch.no_grad():
for idx, (id, frames, _, _, _, labels) in enumerate(tqdm(loader)):
frames = frames.squeeze(0).cuda()
labels = labels.squeeze(0).cuda()
# b = frames.shape[0]
# cache = []
# version 3
# if b <= test_batch_size:
# cache = model.cnn(frames)
# torch.save(cache, f'cache/{epoch}/{idx}_0.pth')
# else:
# cache = []
for i, (f, l) in enumerate(
zip(frames.split(test_batch_size),
labels.split(test_batch_size))):
cache = model(None, f, l)
img_ft = cache[0]
color = F.softmax(cache[1].mean(dim=0),
dim=0).cpu() #.numpy()
typ = F.softmax(cache[2].mean(dim=0),
dim=0).cpu() #.numpy()
# color = np.argmax(color)
# typ = np.argmax(typ)
cache = [img_ft, color, typ]
torch.save(cache, f'cache/{epoch}/{idx}_{i}.pth')
break
print('saving language features..')
for uuid, query_nl in zip(uuids, nls):
nls_list = []
query_nl, vehicle_type = CityFlowNLDataset.type_replacer(
query_nl)
query_nl, vehicle_color = CityFlowNLDataset.color_replacer(
query_nl)
# max_len = max([len(dataset.nl.do_clean(nl)) for nl in query_nl])
for nl in query_nl:
nl = torch.tensor(
dataset.nl.sentence_to_index(nl,
is_train=False)).cuda()
# nls.append(nl.unsqueeze(0).transpose(1, 0))
nl = nl.unsqueeze(0).transpose(1, 0)
# bs, len, dim
nl = model.rnn(nl)
nls_list.append(nl)
saved_nls = {
'nls': nls_list,
'type': vehicle_type,
'color': vehicle_color
}
torch.save(saved_nls, f'cache/{epoch}/{uuid}.pth')
# model = model.cpu()
del model, saved_dict, n, nls_list, img_ft
torch.cuda.empty_cache()
def test(rank, cfg, loader, dataset, epoch, uuids, nls, scene_threshold,
total_threshold):
dataset.load_frame = False
dist.init_process_group(backend="nccl",
rank=rank,
world_size=cfg.num_gpu,
init_method="env://")
torch.cuda.set_device(rank)
cudnn.benchmark = True
model = MyModel(cfg,
len(dataset.nl),
dataset.nl.word_to_idx['<PAD>'],
norm_layer=nn.BatchNorm2d,
num_colors=len(CityFlowNLDataset.colors),
num_types=len(CityFlowNLDataset.vehicle_type) - 2).cuda()
model = DistributedDataParallel(model,
device_ids=[rank],
output_device=rank,
broadcast_buffers=cfg.num_gpu > 1,
find_unused_parameters=False)
saved_dict = torch.load(f'save/{epoch}.pth',
map_location=torch.device(f'cuda:{rank}'))
model.load_state_dict(saved_dict, True)
model.eval()
final_results = {}
a = len(nls) // cfg.num_gpu
start = a * rank
end = None if (rank + 1) == cfg.num_gpu else a * (rank + 1)
end_str = 'end' if end == None else end
print(f'process number: {rank}, {start}:{end_str}')
for nlidx, (uuid,
query_nl) in enumerate(zip(uuids[start:end], nls[start:end])):
nlidx = nlidx + start
print(f'{nlidx} / {len(nls)}')
cache_nl = torch.load(f'cache/{epoch}/{uuid}.pth',
map_location=torch.device(f'cuda:{rank}'))
cache_nl, vehicle_type, vehicle_color = cache_nl['nls'], cache_nl[
'type'], cache_nl['color']
# nls = []
# for nl in query_nl:
# nl = torch.tensor(dataset.nl.sentence_to_index(nl, is_train=False)).cuda()
# nls.append(nl.unsqueeze(0).transpose(1, 0))
uuids_per_nl = []
prob_per_nl = []
for idx, (id, frames, boxes, paths, rois, labels) in enumerate(loader):
# print(f'{nlidx}_{idx}')
with torch.no_grad():
boxes = boxes.squeeze(0).numpy()
rois = rois.squeeze(0).numpy()
# print(rois)
frames = frames.squeeze(0)
# print(frames.shape)
# b = frames.shape[0]
labels = labels.squeeze(0)
labels = labels.cuda()
cache = torch.load(f'cache/{epoch}/{idx}_0.pth',
map_location=torch.device(f'cuda:{rank}'))
# print(cache)
frame, cs, vs = cache
# if vehicle_type != -1 and vs != vehicle_type:
# continue
# if vehicle_color != -1 and cs != vehicle_color:
# continue
# print(frame.device)
# results = []
nl1 = cache_nl[0]
nl2 = cache_nl[1]
nl3 = cache_nl[2]
bs = frame.shape[0]
# cache = cache[:num_of_vehicles]
# print(cache.shape)
if nl1.shape[0] != bs:
nl1 = cache_nl[0].expand(bs, -1, -1).cuda()
nl2 = cache_nl[1].expand(bs, -1, -1).cuda()
nl3 = cache_nl[2].expand(bs, -1, -1).cuda()
am1 = model(nl1, frame, labels)
am2 = model(nl2, frame, labels)
am3 = model(nl3, frame, labels)
# am1, c1, v1 = model(nl1, frame, labels)
# am2, c2, v2 = model(nl2, frame, labels)
# am3, c3, v3 = model(nl3, frame, labels)
activation_aggregation = (am1 + am2 + am3) / 3
# c_aggregation = (c1 + c2 + c3) / 3
# v_aggregation = (v1 + v2 + v3) / 3
# activation_aggregation = model(nl1, frame) +\
# model(nl2, frame) +\
# model(nl3, frame)
# activation_aggregation = activation_aggregation / 3
# results.append(activation_aggregation)
# cs.append(c_aggregation)
# vs.append(v_aggregation)
results = activation_aggregation.cpu().numpy()
# cs = c_aggregation.mean(dim=0).cpu().numpy()
# vs = v_aggregation.mean(dim=0).cpu().numpy()
# version 1
# cache = torch.load(f'cache/{epoch}/{idx}.pth')
# results = []
# for batch_idx in range(cache.shape[0]):
# output = model(cache_nl[0], cache[batch_idx:batch_idx+1]).sigmoid()
# results.append(output.squeeze(0).cpu().detach().numpy())
prob = compute_probability_of_activations(
results, rois, scene_threshold)
# if vehicle_type != -1 and np.argmax(vs) != vehicle_type:
# prob = 0.
# if vehicle_color != -1 and np.argmax(cs) != vehicle_color:
# prob = 0.
###### visualization
# if not os.path.exists('results/' + query_nl[0]):
# os.mkdir('results/' + query_nl[0])
# # cs = np.argmax(cs)
# cs = cs.item()
# vs = vs.item()
# cs = CityFlowNLDataset.colors[cs]
# # vs = np.argmax(vs)
# vs = CityFlowNLDataset.vehicle_type[vs]
# if prob > total_threshold:
# print(f'color: {cs}, type: {vs}')
# save_img(np.squeeze(results[0], axis=0) * 255, cv2.imread(paths[0][0]), boxes[0], f"results/{query_nl[0]}/{idx}_{prob}.png")
###### end visualization
# for submission
uuids_per_nl.append(id[0])
prob_per_nl.append(prob)
final_results['uuids_order'] = uuids_per_nl
final_results[uuid] = prob_per_nl
# uuids_per_nl = np.array(uuids_per_nl)
# # print(uuids_per_nl.shape)
# prob_per_nl = np.array(prob_per_nl)
# prob_per_nl_arg = (-prob_per_nl).argsort(axis=0)
# sorted_uuids_per_nl = uuids_per_nl[prob_per_nl_arg]
# # print(prob_per_nl[prob_per_nl_arg])
# final_results[uuid] = sorted_uuids_per_nl.tolist()
# print(len(final_results.keys()))
with open(f'results/submit_{epoch}_{start}_{end_str}.json', 'w') as fp:
json.dump(final_results, fp)
class MultiHeadTester:
def __init__(
self,
dataset_path='./drive/MyDrive/datasets/car classification/train_data',
batch_size=1,
model_name='tf_efficientnet_b3_ns',
test_csv='./train_labels.csv',
unique_csv='./train_labels.csv',
output_dir='../drive/MyDrive/ckpt/grapheme/submission.csv',
ckpt='../drive/MyDrive/ckpt/grapheme/20.pth'):
# initialize attributes
self.dataset_path = dataset_path
self.batch_size = batch_size
self.model_name = model_name
self.test_csv = test_csv
self.unique_csv = unique_csv
self.output_dir = output_dir
self.ckpt = ckpt
if model_name == 'tf_efficientnet_b0_ns':
self.input_size = (224, 224)
elif model_name == 'tf_efficientnet_b3_ns':
self.input_size = (300, 300)
elif model_name == 'tf_efficientnet_b4_ns':
scaleelf.input_size = (380, 380)
elif model_name == 'tf_efficientnet_b6_ns':
self.input_size = (528, 528)
else:
raise Exception('non-valid model name')
# Compose transforms
transform = []
transform += [transforms.Resize(self.input_size)]
self.transform = transforms.Compose(transform)
self.test_dataset = BengaliDataset(self.test_csv,
self.unique_csv,
self.dataset_path,
self.transform,
cache=True)
self.names = self.test_dataset.names
self.test_dataloader = DataLoader(self.test_dataset,
batch_size=self.batch_size,
num_workers=0,
shuffle=False)
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.model_root = MyModel(self.input_size,
self.model_name,
168,
pretrained=True,
dropout=0).to('cuda')
self.model_consonant = MyModel(self.input_size,
self.model_name,
11,
pretrained=True,
dropout=0).to('cuda')
self.model_vowel = MyModel(self.input_size,
self.model_name,
18,
pretrained=True,
dropout=0).to('cuda')
self.model_multihead = MultiHeadModel(self.input_size,
self.model_name,
pretrained=True,
dropout=0).to('cuda')
ckpt = torch.load(self.ckpt)
self.model_root.load_state_dict(ckpt['model_root_state_dict'])
self.model_consonant.load_state_dict(
ckpt['model_consonant_state_dict'])
self.model_vowel.load_state_dict(ckpt['model_vowel_state_dict'])
self.model_multihead.load_state_dict(
ckpt['model_multihead_state_dict'])
def test(self):
pbar = tqdm.tqdm(self.test_dataloader)
pbar.set_description('testing process')
self.model_root.eval()
self.model_consonant.eval()
self.model_vowel.eval()
self.model_multihead.eval()
output_roots = []
output_consonants = []
output_vowels = []
count = 0
with torch.no_grad():
for it, data in enumerate(pbar):
inputs = data[0].to(self.device)
inputs = inputs.repeat(1, 3, 1, 1)
roots = data[1].to(self.device).long()
consonants = data[2].to(self.device).long()
vowels = data[3].to(self.device).long()
uniques = data[4].to(self.device).long()
root_preds, root_preds_2 = self.model_root(inputs, roots)
self.model_root.zero_grad()
consonant_preds, consonant_preds_2 = self.model_consonant(
inputs, consonants)
self.model_consonant.zero_grad()
vowel_preds, vowel_preds_2 = self.model_vowel(inputs, vowels)
self.model_vowel.zero_grad()
root, consonant, vowel, unique, root2, consonant2, vowel2, unique2 = self.model_multihead(
inputs, roots, consonants, vowels, uniques)
self.model_multihead.zero_grad()
unique_prob = F.softmax(unique, dim=1)
for index in range(inputs.shape[0]):
#print('unique:', unique_prob.max(-1).values[index])
if unique_prob.max(-1).values[index] > 0.5:
output_roots.append(root.argmax(-1)[index].item())
output_consonants.append(
consonant.argmax(-1)[index].item())
output_vowels.append(vowel.argmax(-1)[index].item())
else:
output_roots.append(
root_preds.argmax(-1)[index].item())
output_consonants.append(
consonant_preds.argmax(-1)[index].item())
output_vowels.append(
vowel_preds.argmax(-1)[index].item())
row_id, target = [], []
for iid, r, v, c in zip(self.names, output_roots, output_consonants,
output_vowels):
row_id.append(iid + '_grapheme_root')
target.append(int(r))
row_id.append(iid + '_vowel_diacritic')
target.append(int(v))
row_id.append(iid + '_consonant_diacritic')
target.append(int(c))
count += 1
sub_fn = self.output_dir
sub = pd.DataFrame({'row_id': row_id, 'target': target})
sub.to_csv(sub_fn, index=False)
print(f'Done wrote to {sub_fn}')
# udpate tensorboardX
writer.add_scalar('train_loss', n_iter)
# compute computation time and *compute_efficiency*
process_time = start_time - time.time() - prepare_time
compute_efficiency = process_time / (process_time + prepare_time)
pbar.set_description(
f'Compute efficiency: {compute_efficiency:.2f}, '
f'loss: {loss.item():.2f}, epoch: {epoch}/{opt.epochs}')
start_time = time.time()
# maybe do a test pass every N=1 epochs
if epoch % 1 == 0:
# bring models to evaluation mode
net.eval()
correct = 0
total = 0
pbar = tqdm(enumerate(test_data_loader),
total=len(test_data_loader))
with torch.no_grad():
for i, data in pbar:
# data preparation
img, label = data
if use_cuda:
img = img.cuda()
label = label.cuda()
out = net(img)
def train(num_epochs, batch_size):
current_time = time.strftime('%Y_%m_%d_%H_%M', time.localtime(time.time()))
logger = utils.Logger(os.path.join('out', 'log_' + current_time + '.txt'))
# logger.write(f'{current_time}logger_batch_size{batch_size}')
vocab_size = ge.vocab_size
answer_size = ge.answer_size
train_set = GQA(root='data', split='train', transform=transform)
val_set = GQA(root='data', split='val', transform=transform)
train_loader = DataLoader(train_set,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_data,
drop_last=True)
val_loader = DataLoader(val_set,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_data,
drop_last=True)
# add a tqdm bar
d = iter(train_loader)
pbar = tqdm(d)
net = MyModel(out_features=answer_size).to(device)
optimizer = torch.optim.Adam(net.parameters(), lr=1e-4)
loss = nn.CrossEntropyLoss()
def _validation(val_loader):
acc_sum, n = 0.0, 0
for img, q, loq, a in val_loader:
img, q, a = (
img.to(device),
q.to(device),
a.to(device),
)
a_hat = net(img, q)
acc_sum += (a_hat.detach().argmax(dim=1) == a).sum().item()
n += a.shape[0]
if n <= 0:
print('validation loader does not work')
return 0.0
return acc_sum / n
for i in range(num_epochs):
loss_sum = 0.0
acc_sum = 0.0
n = 0
start = time.time()
for img, q, loq, a in train_loader:
img, q, a = (
img.to(device),
q.to(device),
a.to(device),
)
net.train()
net.zero_grad()
a_hat = net(img, q)
l = loss(a_hat, a)
l.backward()
optimizer.step()
loss_sum += l.item()
acc_sum += (a_hat.detach().argmax(dim=1) == a).sum().item()
n += a.shape[0]
if n < 100:
print('time for one batch:', time.time() - start)
net.eval()
with torch.no_grad():
vali_acc = _validation(val_loader)
logger.write(
'Epoch #%d, Loss:%.3f, Train Acc: %.4f, Validation Acc:%.4f' %
(i, loss_sum, acc_sum / n, vali_acc))
print('Epoch #%d, Loss:%.3f, Train Acc: %.4f, Validation Acc:%.4f' %
(i, loss_sum, acc_sum / n, vali_acc))
try:
torch.save(net.state_dict(),
'checkpoint/checkpoint_{}.pth'.format(i))
except:
print('can not save checkpoint.')
torch.save(net.state_dict(), 'checkpoint/model_2.pth')
TEST_INTERVAL = 200
PRINT_INTERVAL = 50
LEARNING_RATE = 1e-5
if USE_MEMORY:
LEARNING_RATE = (4e-4 if DOUBLE_Q else 3e-3)
RENDER_INTERVAL = 20
ENV_NAME = 'CartPole-v0'
env = gym.make(ENV_NAME)
state_shape = len(env.reset())
n_actions = env.action_space.n
model = MyModel(state_shape, n_actions).to(device)
target = MyModel(state_shape, n_actions).to(device)
target.load_state_dict(model.state_dict())
target.eval()
optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)
loss_function = torch.nn.MSELoss(reduction='mean')
memory = ReplayBuffer()
def choose_action(state, test_mode=False):
if random.random() < EPS_EXPLORATION:
return env.action_space.sample()
return model.select_action(torch.from_numpy(state)).item()
# Given a tuple (s_t, a_t, s_{t+1}, r_t, done_t) update your model weights
def optimize_model(state, action, next_state, reward, done):
# For double Q, use the other Q function for getting values of the next state.
nextStateQ = target if DOUBLE_Q else model
