def __init__(self, features, labels, policy):
self.features = features
self.labels = labels
# init label rewards (adapt freely)
m = np.zeros((len(self.labels), max(self.labels)+1))
for i in range(0, len(self.labels)):
m[i][self.labels[i]] = 1
# label frame
self.y = pd.DataFrame(m)
# frame of features
self.X = pd.DataFrame(features)
self.X = self.X.reset_index().drop(columns=['index'])
self.cBandit = ContextualBandit(self.X, self.y)
self.myAgent = KerasAgent(lr=0.001,
a_size=self.cBandit.num_actions,
n_states=self.cBandit.num_state_features)
self.policy = policy
self.policy.setBandit(self.cBandit)
self.policy.setAgent(self.myAgent)
def train_model(args, vocab1, vocab2, device):
print(args)
print("generating config")
config1 = Config1(
vocab_size=len(vocab1),
embedding_dim=args.embedding_dim,
LSTM_layers=args.lstm_layer_1,
LSTM_hidden_units=args.hidden,
train_embed=args.train_embed,
# pretrained_embedding=vocab1.embedding,
word2id=vocab1.word_to_index,
id2word=vocab1.index_to_word,
dropout=args.dropout)
config2 = Config2(
vocab_size=len(vocab2),
embedding_dim=args.embedding_dim,
LSTM_layers=args.lstm_layer_2,
LSTM_hidden_units=args.hidden,
train_embed=args.train_embed,
# pretrained_embedding=vocab2.embedding,
word2id=vocab2.word_to_index,
id2word=vocab2.index_to_word,
dropout=args.dropout,
decode_type=args.decode_type)
model_name_1 = ".".join(
(args.model_file_1, str(args.rl_baseline_method), args.sampling_method,
"gamma", str(args.gamma), "beta", str(args.beta), "batch",
str(args.train_batch), "learning_rate", str(args.lr_1), "bsz",
str(args.batch_size), "data", args.data_dir.split('/')[0], "emb",
str(config1.embedding_dim), "dropout", str(args.dropout), "max_num",
str(args.max_num_of_ans), "train_embed", str(args.train_embed),
'd2s'))
# model_name_2 = ".".join((args.model_file_2,
# "gamma",str(args.gamma),
# "beta",str(args.beta),
# "batch",str(args.train_batch),
# "learning_rate",str(args.lr_2),
# "data", args.data_dir.split('/')[0],
# "emb", str(config2.embedding_dim),
# "dropout", str(args.dropout),
# 'decode_type',str(args.decode_type),
# 'd2s'))
log_name = ".".join(
("log/model", str(args.rl_baseline_method), args.sampling_method,
"gamma", str(args.gamma), "beta", str(args.beta), "batch",
str(args.train_batch), "lr_1", str(args.lr_1), "lr_2", str(args.lr_1),
args.sampling_method, "bsz", str(args.batch_size), "data",
args.data_dir.split('/')[0], "emb1", str(config1.embedding_dim),
"emb2", str(config2.embedding_dim), "dropout", str(args.dropout),
'decode_type', str(args.decode_type), "train_embed",
str(args.train_embed), 'd2s'))
print("initialising data loader and RL learner")
data_loader = PickleReader(args.data_dir)
data = args.data_dir.split('/')[0]
num_data = 3398
# init statistics
reward_list = []
loss_list1 = []
loss_list2 = []
best_eval_reward = 0.
model_save_name_1 = model_name_1
# model_save_name_2 = model_name_2
bandit = ContextualBandit(b=args.batch_size,
rl_baseline_method=args.rl_baseline_method,
vocab=vocab2,
sample_method=args.sampling_method,
device=device)
print("Loaded the Bandit")
model1 = model.Bandit(config1).to(device)
# model2 = model.Generator(config2).to(device)
print("Loaded the models")
if args.load_ext:
model_name_1 = args.model_file_1
# model_name_2 = args.model_file_2
model_save_name_1 = model_name_1
# model_save_name_2 = model_name_2
print("loading existing models:1->%s" % model_name_1)
# print("loading existing models:2->%s" % model_name_2)
model1 = torch.load(model_name_1,
map_location=lambda storage, loc: storage)
model1.to(device)
# model2 = torch.load(model_name_2, map_location=lambda storage, loc: storage)
# model2.to(device)
log_name = 'log/' + model_name_1.split('/')[-1]
print("finish loading and evaluate models:")
# evaluate.ext_model_eval(extract_net, vocab, args, eval_data="test")
best_eval_reward = evaluate.ext_model_eval(model1, None, vocab2, args,
"val", device)
logging.basicConfig(filename='%s.log' % log_name,
level=logging.DEBUG,
format='%(asctime)s %(levelname)-10s %(message)s')
logging.info("prev best eval reward:%.4f" % (best_eval_reward))
# Loss and Optimizer
optimizer1 = torch.optim.Adam([
param for param in model1.parameters() if param.requires_grad == True
],
lr=args.lr_1,
betas=(args.beta, 0.999),
weight_decay=1e-6)
# optimizer2 = torch.optim.Adam([param for param in model2.parameters() if param.requires_grad == True ], lr=args.lr_2, betas=(args.beta, 0.999),weight_decay=1e-6)
# if args.lr_sch ==1:
# scheduler = ReduceLROnPlateau(optimizer_ans, 'max',verbose=1,factor=0.9,patience=3,cooldown=3,min_lr=9e-5,epsilon=1e-6)
# if best_eval_reward:
# scheduler.step(best_eval_reward,0)
# print("init_scheduler")
# elif args.lr_sch ==2:
# scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer_ans,args.lr, args.lr_2, step_size_up=3*int(num_data/args.train_batch), step_size_down=3*int(num_data/args.train_batch), mode='exp_range', gamma=0.98,cycle_momentum=False)
print("starting training")
start_time = time.time()
n_step = 100
gamma = args.gamma
n_val = int(num_data / (7 * args.train_batch))
supervised_loss = torch.nn.BCELoss()
regression_loss = torch.nn.MSELoss()
with torch.autograd.set_detect_anomaly(True):
for epoch in tqdm(range(args.epochs_ext), desc="epoch:"):
train_iter = data_loader.chunked_data_reader(
"train", data_quota=args.train_example_quota) #-1
step_in_epoch = 0
for dataset in train_iter:
for step, contexts in tqdm(
enumerate(
BatchDataLoader(dataset,
batch_size=args.train_batch,
shuffle=True))):
try:
model1.train()
# model2.train()
step_in_epoch += 1
loss = 0.
reward = 0.
for context in contexts:
records = context.records
target = context.summary
records = torch.autograd.Variable(
torch.LongTensor(records)).to(device)
# target = torch.autograd.Variable(torch.LongTensor(target)).to(device)
# target_len = len(target)
prob, num_r = model1(records)
num_of_records = int(num_r.item() * 100)
sample_content, greedy_cp = bandit.sample(
prob, context, num_of_records)
# # apply data_parallel after this step
# sample_content.append((greedy_cp,0))
# gen_summaries = []
# total_loss = 0.
# for cp in [(greedy_cp.data,0)]:
# gen_input = torch.autograd.Variable(r_cs[cp[0]].data).to(device)
# e_k,prev_hidden, prev_emb = model2(gen_input,vocab2)
# z_k = torch.autograd.Variable(records[cp[0]][:,0].data).to(device)
# prev_t =0
# loss=0.
# gen_summary =[]
# ## perform bptt here
# for y_t in range(target_len):
# p_out, prev_hidden = model2.forward_step(prev_emb,prev_hidden,gen_input,e_k,z_k)
# topv,topi = p_out.topk(1)
# gen_summary.append(topi)
# prev_emb = model2.get_embedding(topi)
# loss += decode_loss(p_out,target[y_t].unsqueeze(0))
# if (y_t-prev_t)==50:
# prev_t = y_t
# loss.backward(retain_graph=True)
# loss.detach()
# if prev_t < target_len:
# loss.backward()
# loss.detach()
# gen_summaries.append((gen_summary,cp[1]))
# loss/=float(target_len)
# total_loss+=loss
# optimizer2.step()
# optimizer2.zero_grad()
# total_loss/=len(sample_content)
bandit_loss, reward_b = bandit.calculate_loss(
sample_content, context.gold_index, greedy_cp)
true_numr = context.num_of_records / 100.
r_loss = regression_loss(
num_r,
torch.tensor(true_numr).type(
torch.float).to(device))
#greedy_cp,bandit_loss = greedy_sample(prob,num_of_records+1,device)
#reward_b = generate_reward(None,None,gold_cp=context.gold_index,cp=greedy_cp)
labels = np.zeros(len(prob))
labels[context.gold_index] = 1.0
ml_loss = supervised_loss(
prob.view(-1),
torch.tensor(labels).type(
torch.float).to(device))
loss_e = (gamma * (bandit_loss + r_loss)) + (
(1 - gamma) * ml_loss)
loss_e.backward()
reward += reward_b
loss += loss_e.item()
optimizer1.step()
optimizer1.zero_grad()
loss /= args.train_batch
reward /= args.train_batch
reward_list.append(reward)
loss_list1.append(loss)
# loss_list2.append(total_loss)
# if args.lr_sch==2:
# scheduler.step()
# logging.info('Epoch %d Step %d Reward %.4f Loss1 %.4f Loss2 %.4f' % (epoch, step_in_epoch, reward,bandit_loss,total_loss))
logging.info(
'Epoch %d Step %d Reward %.4f Loss1 %.4f' %
(epoch, step_in_epoch, reward, loss))
except Exception as e:
print(e)
traceback.print_exc()
if (step_in_epoch) % n_step == 0 and step_in_epoch != 0:
# logging.info('Epoch ' + str(epoch) + ' Step ' + str(step_in_epoch) +
# ' reward: ' + str(np.mean(reward_list))+' loss1: ' + str(np.mean(loss_list1))+' loss2: ' + str(np.mean(loss_list2)))
logging.info('Epoch ' + str(epoch) + ' Step ' +
str(step_in_epoch) + ' reward: ' +
str(np.mean(reward_list)) + ' loss1: ' +
str(np.mean(loss_list1)))
reward_list = []
loss_list1 = []
# loss_list2=[]
if (step_in_epoch) % n_val == 0 and step_in_epoch != 0:
print("doing evaluation")
model1.eval()
# model2.eval()
#eval_reward = evaluate.ext_model_eval(mcan_cb, vocab, args, "test")
eval_reward = evaluate.ext_model_eval(
model1, None, vocab2, args, "val", device)
if eval_reward > best_eval_reward:
best_eval_reward = eval_reward
print(
"saving models %s : with eval_reward:" %
model_save_name_1, eval_reward)
logging.debug("saving models" +
str(model_save_name_1) + " " +
"with eval_reward:" +
str(eval_reward))
torch.save(model1, model_save_name_1)
# torch.save(model2,model_save_name_2)
print('epoch ' + str(epoch) +
' reward in validation: ' + str(eval_reward))
logging.debug('epoch ' + str(epoch) +
' reward in validation: ' +
str(eval_reward))
logging.debug('time elapsed:' +
str(time.time() - start_time))
# if args.lr_sch ==1:
# mcan_cb.eval()
# eval_reward = evaluate.ext_model_eval(mcan_cb, vocab, args, "val")
# #eval_reward = evaluate.ext_model_eval(mcan_cb, vocab, args, "test")
# scheduler.step(eval_reward[0],epoch)
return model1
def train_model(args):
print(args)
print("generating config")
config = Config(
input_dim=args.input_dim,
dropout=args.dropout,
highway=args.highway,
nn_layers=args.nn_layers,
)
model_name = ".".join(
(args.model_file, str(args.rl_baseline_method), args.sampling_method,
"gamma", str(args.gamma), "beta", str(args.beta), "batch",
str(args.train_batch),
"learning_rate", str(args.lr) + "-" + str(args.lr_sch), "bsz",
str(args.batch_size), "data", args.data_dir.split('/')[0],
args.eval_data, "input_dim", str(config.input_dim), "max_num",
str(args.max_num_of_ans), "reward", str(args.reward_type), "dropout",
str(args.dropout) + "-" + str(args.clip_grad), "highway",
str(args.highway), "nn-" + str(args.nn_layers), 'ans'))
log_name = ".".join(
("log_bert/model", str(args.rl_baseline_method), args.sampling_method,
"gamma", str(args.gamma), "beta", str(args.beta), "batch",
str(args.train_batch), "lr", str(args.lr) + "-" + str(args.lr_sch),
"bsz", str(args.batch_size), "data", args.data_dir.split('/')[0],
args.eval_data, "input_dim", str(config.input_dim), "max_num",
str(args.max_num_of_ans), "reward", str(args.reward_type), "dropout",
str(args.dropout) + "-" + str(args.clip_grad), "highway",
str(args.highway), "nn-" + str(args.nn_layers), 'ans'))
print("initialising data loader and RL learner")
data_loader = PickleReader(args.data_dir)
data = args.data_dir.split('/')[0]
num_data = 0
if data == "wiki_qa":
num_data = 873
elif data == "trec_qa":
num_data = 1229
else:
assert (1 == 2)
# init statistics
reward_list = []
loss_list = []
best_eval_reward = 0.
model_save_name = model_name
bandit = ContextualBandit(b=args.batch_size,
rl_baseline_method=args.rl_baseline_method,
sample_method=args.sampling_method)
print("Loaded the Bandit")
bert_cb = model2.BERT_CB(config)
print("Loaded the model")
bert_cb.cuda()
vocab = "vocab"
if args.load_ext:
model_name = args.model_file
print("loading existing model%s" % model_name)
bert_cb = torch.load(model_name,
map_location=lambda storage, loc: storage)
bert_cb.cuda()
model_save_name = model_name
log_name = "/".join(("log_bert", model_name.split("/")[1]))
print("finish loading and evaluate model %s" % model_name)
# evaluate.ext_model_eval(extract_net, vocab, args, eval_data="test")
best_eval_reward = evaluate.ext_model_eval(bert_cb, vocab, args,
args.eval_data)[0]
logging.basicConfig(filename='%s.log' % log_name,
level=logging.DEBUG,
format='%(asctime)s %(levelname)-10s %(message)s')
# Loss and Optimizer
optimizer_ans = torch.optim.Adam([
param for param in bert_cb.parameters() if param.requires_grad == True
],
lr=args.lr,
betas=(args.beta, 0.999),
weight_decay=1e-6)
if args.lr_sch == 1:
scheduler = ReduceLROnPlateau(optimizer_ans,
'max',
verbose=1,
factor=0.9,
patience=3,
cooldown=3,
min_lr=9e-5,
epsilon=1e-6)
if best_eval_reward:
scheduler.step(best_eval_reward, 0)
print("init_scheduler")
elif args.lr_sch == 2:
scheduler = torch.optim.lr_scheduler.CyclicLR(
optimizer_ans,
args.lr,
args.lr_2,
step_size_up=3 * int(num_data / args.train_batch),
step_size_down=3 * int(num_data / args.train_batch),
mode='exp_range',
gamma=0.98,
cycle_momentum=False)
print("starting training")
start_time = time.time()
n_step = 100
gamma = args.gamma
#vocab = "vocab"
if num_data < 2000:
n_val = int(num_data / (5 * args.train_batch))
else:
n_val = int(num_data / (7 * args.train_batch))
with torch.autograd.set_detect_anomaly(True):
for epoch in tqdm(range(args.epochs_ext), desc="epoch:"):
train_iter = data_loader.chunked_data_reader(
"train", data_quota=args.train_example_quota) #-1
step_in_epoch = 0
for dataset in train_iter:
for step, contexts in tqdm(
enumerate(
BatchDataLoader(dataset,
batch_size=args.train_batch,
shuffle=True))):
try:
bert_cb.train()
step_in_epoch += 1
loss = 0.
reward = 0.
for context in contexts:
# q_a = torch.autograd.Variable(torch.from_numpy(context.features)).cuda()
pre_processed, a_len, sorted_id = model2.bert_preprocess(
context.answers)
q_a = torch.autograd.Variable(
pre_processed.type(torch.float))
a_len = torch.autograd.Variable(a_len)
outputs = bert_cb(q_a, a_len)
context.labels = np.array(
context.labels)[sorted_id]
if args.prt_inf and np.random.randint(0, 100) == 0:
prt = True
else:
prt = False
loss_t, reward_t = bandit.train(
outputs,
context,
max_num_of_ans=args.max_num_of_ans,
reward_type=args.reward_type,
prt=prt)
#print(str(loss_t)+' '+str(len(a_len)))
# loss_t = loss_t.view(-1)
true_labels = np.zeros(len(context.labels))
gold_labels = np.array(context.labels)
true_labels[gold_labels > 0] = 1.0
# ml_loss = F.binary_cross_entropy(outputs.view(-1),torch.tensor(true_labels).type(torch.float).cuda())
ml_loss = F.binary_cross_entropy(
outputs.view(-1),
torch.tensor(true_labels).type(
torch.float).cuda())
loss_e = ((gamma * loss_t) +
((1 - gamma) * ml_loss))
loss_e.backward()
loss += loss_e.item()
reward += reward_t
loss = loss / args.train_batch
reward = reward / args.train_batch
if prt:
print('Probabilities: ',
outputs.squeeze().data.cpu().numpy())
print('-' * 80)
reward_list.append(reward)
loss_list.append(loss)
#if isinstance(loss, Variable):
# loss.backward()
if step % 1 == 0:
if args.clip_grad:
torch.nn.utils.clip_grad_norm_(
bert_cb.parameters(),
args.clip_grad) # gradient clipping
optimizer_ans.step()
optimizer_ans.zero_grad()
if args.lr_sch == 2:
scheduler.step()
logging.info('Epoch %d Step %d Reward %.4f Loss %.4f' %
(epoch, step_in_epoch, reward, loss))
except Exception as e:
print(e)
#print(loss)
#print(loss_e)
traceback.print_exc()
if (step_in_epoch) % n_step == 0 and step_in_epoch != 0:
logging.info('Epoch ' + str(epoch) + ' Step ' +
str(step_in_epoch) + ' reward: ' +
str(np.mean(reward_list)) + ' loss: ' +
str(np.mean(loss_list)))
reward_list = []
loss_list = []
if (step_in_epoch) % n_val == 0 and step_in_epoch != 0:
print("doing evaluation")
bert_cb.eval()
eval_reward = evaluate.ext_model_eval(
bert_cb, vocab, args, args.eval_data)
if eval_reward[0] > best_eval_reward:
best_eval_reward = eval_reward[0]
print(
"saving model %s with eval_reward:" %
model_save_name, eval_reward)
logging.debug("saving model" +
str(model_save_name) +
"with eval_reward:" +
str(eval_reward))
torch.save(bert_cb, model_name)
print('epoch ' + str(epoch) +
' reward in validation: ' + str(eval_reward))
logging.debug('epoch ' + str(epoch) +
' reward in validation: ' +
str(eval_reward))
logging.debug('time elapsed:' +
str(time.time() - start_time))
if args.lr_sch == 1:
bert_cb.eval()
eval_reward = evaluate.ext_model_eval(bert_cb, vocab, args,
args.eval_data)
scheduler.step(eval_reward[0], epoch)
return bert_cb
class Environment(object):
def __init__(self, features, labels, policy):
self.features = features
self.labels = labels
# init label rewards (adapt freely)
m = np.zeros((len(self.labels), max(self.labels)+1))
for i in range(0, len(self.labels)):
m[i][self.labels[i]] = 1
# label frame
self.y = pd.DataFrame(m)
# frame of features
self.X = pd.DataFrame(features)
self.X = self.X.reset_index().drop(columns=['index'])
self.cBandit = ContextualBandit(self.X, self.y)
self.myAgent = KerasAgent(lr=0.001,
a_size=self.cBandit.num_actions,
n_states=self.cBandit.num_state_features)
self.policy = policy
self.policy.setBandit(self.cBandit)
self.policy.setAgent(self.myAgent)
def iter(self):
# classical bandit interaction
# a) get state, b) perform action, c) get reward and update
s, t = self.cBandit.getInputState()
action = self.policy.select(s)
reward = self.cBandit.pullArm(action)
# Update the network.
y = self.policy.qval[:]
y[0][action] = reward
self.myAgent.model.fit(s, y, batch_size=1, epochs=1, verbose=0)
return t, action, reward
def experiment(self, total_rounds=1000000):
i = 0
pbar = tqdm(total=total_rounds)
while i < total_rounds:
t, action, reward = self.iter()
i += 1
pbar.update(1)
pbar.close()
inputs = self.myAgent.model.predict(self.cBandit.X)
probas = inputs.reshape(self.cBandit.num_samples, -1)
predictions = np.argmax(probas, axis=1)
accuracy = Mean_Log_Loss(predictions=predictions, labels=self.labels)
self.output(accuracy, predictions)
return predictions
def output(self, accuracy, predictions):
print("baseline accuracy: ", accuracy)
print("predicitons: ", predictions)
high_order_knockout, index = High_Order_Iterative_Knockout(
features_knockout=np.array(self.features),
model=self.myAgent.model,
baseline=accuracy,
labels=self.labels)
print("high-order knockout accuracy change: ")
Z = [(y, x) for y, x in sorted(zip(high_order_knockout, index),
reverse=True, key=lambda l:(l[0], -len(l[1])))]
for z in Z:
print(z)
import pandas as pd
## Set random seed
np.random.seed(123)
## Hyperparameters for the contextual bandit model
k = 2 # number of arms
p = 30 # covariate dimension
# p = 100 # covariate dimension
n = 1000 # number of data
## Hyperparameters for the bandit agent
h = 5
## Initialize bandit model
bandit = ContextualBandit(n, p, k, diversity=True, reward_type=4)
print("True params:", )
X = bandit.covariates
rewards = bandit.rewards
betas = bandit.betas
## Initialize agent: Uncomment the lines that correspond to agents in use
agentList = []
# agentList.append(Agent_OLS(n=n, h=h, k=k, greedy_only=True, name= "Greedy_OLS"))
# agentList.append(Agent_OLS(n=n, h=h, k=k, greedy_only=False, name= "OLS"))
# agentList.append(Agent_OLS(n=n, h=h, k=k, p=p, greedy_only=False, basis_expansion=True, name= "OLS_BE"))
# agentList.append(Agent_LASSO(n=n, h=h, k=k, greedy_only=False, lam= 0.05, name= "LASSO"))
agentList.append(
Agent_LASSO(n=n,
h=h,