def evaluate(model, valid_dataset, batch_size,
evaluate_object_list: typing.List[Evaluator], train_loss_function,
desc, label_preprocess_fn):
model.eval()
for o in evaluate_object_list:
o.clear_result()
train_total_loss = to_cuda(torch.Tensor([0]))
steps = to_cuda(torch.Tensor([0]))
with tqdm(total=math.ceil(len(valid_dataset) / batch_size),
leave=False) as pbar:
for batch_data in data_loader(valid_dataset,
batch_size=batch_size,
is_shuffle=False,
drop_last=False):
model.zero_grad()
predict_logit = model.forward(batch_data)
target = label_preprocess_fn(batch_data)
train_loss = train_loss_function(predict_logit, target)
for evaluator in evaluate_object_list:
evaluator.add_result(predict_logit,
target,
batch_data=batch_data)
train_total_loss += train_loss.data
steps += 1
pbar.update(1)
return evaluate_object_list, train_total_loss / steps
def evaluate(model, dataset, batch_size, loss_function):
total_loss = torch.Tensor([0]).cuda(GPU_INDEX)
steps = torch.Tensor([0]).cuda(GPU_INDEX)
for batch_data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True):
identifier_scope_mask, is_identifier, lengths, target, terminal_mask, tokens, update_mask, has_identifier\
= parse_batch_data(
batch_data)
log_probs = model.forward(tokens, identifier_scope_mask, is_identifier,
update_mask, terminal_mask, lengths,
has_identifier)
batch_log_probs = log_probs.contiguous().view(
-1,
list(log_probs.size())[-1])
target, idx_unsort = torch_util.pack_padded_sequence(
autograd.Variable(torch.LongTensor(target)).cuda(GPU_INDEX),
lengths,
batch_firse=True,
GPU_INDEX=GPU_INDEX)
target, _ = torch_util.pad_packed_sequence(target,
idx_unsort,
pad_value=PAD_TOKEN,
batch_firse=True,
GPU_INDEX=GPU_INDEX)
loss = loss_function(batch_log_probs, target.view(-1))
total_loss += loss.data
steps += torch.sum(lengths.float().cuda(GPU_INDEX).data)
return total_loss / steps
def evaluate(model, dataset, batch_size, loss_function):
total_loss = torch.Tensor([0])
steps = torch.Tensor([0])
for batch_data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True):
target = batch_data["target"]
del batch_data["target"]
batch_data = {
k: autograd.Variable(torch.LongTensor(v))
for k, v in batch_data.items()
}
log_probs = model.forward(**batch_data)
batch_log_probs = log_probs.view(-1, list(log_probs.size())[-1])
target, idx_unsort = torch_util.pack_padded_sequence(
autograd.Variable(torch.LongTensor(target)).cuda(GPU_INDEX),
batch_data['length'],
batch_firse=True,
GPU_INDEX=GPU_INDEX)
target, _ = torch_util.pad_packed_sequence(target,
idx_unsort,
pad_value=PAD_TOKEN,
batch_firse=True,
GPU_INDEX=GPU_INDEX)
loss = loss_function(batch_log_probs, target.view(-1))
total_loss += loss.data.cpu()
steps += torch.sum(batch_data['length'].data.cpu())
return total_loss / steps
def train(model,
dataset,
batch_size,
loss_function,
optimizer):
total_loss = torch.Tensor([0])
steps = torch.Tensor([0])
for batch_data in data_loader(dataset, batch_size=batch_size, is_shuffle=True, drop_last=True, epoch_ratio=0.5):
# print(batch_data['terminal_mask'])
# print('batch_data size: ', len(batch_data['terminal_mask'][0]), len(batch_data['terminal_mask'][0][0]))
# res = list(more_itertools.collapse(batch_data['terminal_mask']))
# print('res len: ', len(res))
# res = util.padded(batch_data['terminal_mask'], deepcopy=True, fill_value=0)
# print('batch_data size: ', len(res[0]), len(res[0][0]))
# res = list(more_itertools.collapse(res))
# print('res len: ', len(res))
target = batch_data["target"]
del batch_data["target"]
model.zero_grad()
batch_data = {k: autograd.Variable(torch.LongTensor(v)) for k, v in batch_data.items()}
log_probs = model.forward(**batch_data)
# log_probs.register_hook(create_hook_fn("log_probs"))
batch_log_probs = log_probs.view(-1, list(log_probs.size())[-1])
target, idx_unsort = torch_util.pack_padded_sequence(
autograd.Variable(torch.LongTensor(target)).cuda(GPU_INDEX),
batch_data['length'], batch_firse=True, GPU_INDEX=GPU_INDEX)
target, _ = torch_util.pad_packed_sequence(target, idx_unsort, pad_value=PAD_TOKEN, batch_firse=True,
GPU_INDEX=GPU_INDEX)
loss = loss_function(batch_log_probs, target.view(-1))
# loss.register_hook(create_hook_fn("loss"))
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 10)
# print()
# print("The loss is nan:{}".format(is_nan(loss.detach())))
# print("The loss grad is nan:{}".format(is_nan(loss.grad)))
# print("The log_probs is nan:{}".format(is_nan(log_probs.detach())))
# print("The log_probs grad is nan:{}".format(is_nan(log_probs.grad)))
# for name, param in model.named_parameters():
# print("name of {}: has nan:{}".format(name, is_nan(param.detach())))
# print("the gradient of {}: has nan:{}".format(name, is_nan(param.grad)))
# if HAS_NAN:
# for k, v in batch_data.items():
# print("{}:{}".format(k, show_tensor(v)))
# print("{}:{}".format("target", show_tensor(target)))
# print()
optimizer.step()
total_loss += loss.data.cpu()
steps += torch.sum(batch_data['length'].data.cpu())
return total_loss/steps
def train(model, dataset, batch_size, loss_function, optimizer):
total_loss = torch.Tensor([0])
steps = torch.Tensor([0])
for batch_data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True,
epoch_ratio=0.25):
# print(batch_data['terminal_mask'])
# print('batch_data size: ', len(batch_data['terminal_mask'][0]), len(batch_data['terminal_mask'][0][0]))
# res = list(more_itertools.collapse(batch_data['terminal_mask']))
# print('res len: ', len(res))
# res = util.padded(batch_data['terminal_mask'], deepcopy=True, fill_value=0)
# print('batch_data size: ', len(res[0]), len(res[0][0]))
# res = list(more_itertools.collapse(res))
# print('res len: ', len(res))
tokens, target, productions, productions_target, predict_index = transform_samples_to_tensor(
**batch_data)
model.zero_grad()
token_predict, production_predict = model.forward(
tokens=tokens,
productions=productions,
predict_index=predict_index)
# log_probs.register_hook(create_hook_fn("log_probs"))
token_loss = loss_function(token_predict, to_cuda(target.view(-1)))
# register_hook(token_loss, "token_loss")
production_loss = loss_function(production_predict,
to_cuda(productions_target.view(-1)))
# register_hook(production_loss, "production_loss")
loss = token_loss + production_loss
# loss.register_hook(create_hook_fn("loss"))
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 10)
# print()
# print("The loss is nan:{}".format(is_nan(loss.detach())))
# print("The loss grad is nan:{}".format(is_nan(loss.grad)))
# print("The log_probs is nan:{}".format(is_nan(log_probs.detach())))
# print("The log_probs grad is nan:{}".format(is_nan(log_probs.grad)))
# for name, param in model.named_parameters():
# print("name of {}: has nan:{}".format(name, is_nan(param.detach())))
# print("the gradient of {}: has nan:{}".format(name, is_nan(param.grad)))
# if HAS_NAN:
# for k, v in batch_data.items():
# print("{}:{}".format(k, show_tensor(v)))
# print("{}:{}".format("target", show_tensor(target)))
# print()
optimizer.step()
total_loss += token_loss.data.cpu()
steps += torch.sum(torch.FloatTensor([len(t) for t in tokens]))
return total_loss / steps
def reset(self):
with tqdm(total=len(self.dataset) * self.data_radio) as pbar:
for batch_data in data_loader(self.dataset,
batch_size=self.batch_size,
drop_last=False,
epoch_ratio=self.data_radio):
self.continue_list = [True for _ in range(self.batch_size)]
self.result_list = [True for _ in range(self.batch_size)]
self.last_sample = [None for _ in range(self.batch_size)]
self.step_action_list = []
self.ac_batch_data = batch_data.copy()
yield batch_data
pbar.update(self.batch_size)
def accuracy_evaluate(
model,
dataset,
batch_size,
loss_function,
):
total_loss = torch.Tensor([0]).cuda(GPU_INDEX)
steps = torch.Tensor([0]).cuda(GPU_INDEX)
accuracy_dict = None
for batch_data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True):
identifier_scope_mask, is_identifier, lengths, target, tokens, update_mask\
= parse_batch_data(
batch_data)
log_probs = model.forward(tokens, identifier_scope_mask, is_identifier,
update_mask, lengths)
batch_log_probs = log_probs.contiguous().view(
-1,
list(log_probs.size())[-1])
target, idx_unsort = torch_util.pack_padded_sequence(
autograd.Variable(torch.LongTensor(target)).cuda(GPU_INDEX),
lengths,
batch_firse=True,
GPU_INDEX=GPU_INDEX)
target, _ = torch_util.pad_packed_sequence(target,
idx_unsort,
pad_value=PAD_TOKEN,
batch_firse=True,
GPU_INDEX=GPU_INDEX)
loss = loss_function(batch_log_probs, target.view(-1))
total_loss += loss.data
steps += torch.sum(lengths.data)
# print("target size:{}".format(target.size()))
# print("batch log size:{}".format(log_probs.size()))
topk_accuracy = calculate_accuracy_of_code_completion(
log_probs, target, ignore_token=PAD_TOKEN, gpu_index=GPU_INDEX)
if accuracy_dict is None:
accuracy_dict = topk_accuracy
else:
for k, v in topk_accuracy.items():
accuracy_dict[k] += topk_accuracy[k]
accuracy_dict = {
k: float(v) / steps.item()
for k, v in accuracy_dict.items()
}
return total_loss / steps, accuracy_dict
def train(model, dataset, batch_size, loss_function, optimizer, clip_norm,
epoch_ratio, parse_input_batch_data_fn, parse_target_batch_data_fn,
create_output_ids_fn, evaluate_obj_list):
total_loss = to_cuda(torch.Tensor([0]))
steps = 0
for o in evaluate_obj_list:
o.clear_result()
model.train()
with tqdm(total=(len(dataset) * epoch_ratio)) as pbar:
for batch_data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True,
epoch_ratio=epoch_ratio):
model.zero_grad()
model_input = parse_input_batch_data_fn(batch_data,
do_sample=False)
model_output = model.forward(*model_input)
model_target = parse_target_batch_data_fn(batch_data)
loss = loss_function(*model_output, *model_target)
loss.backward()
optimizer.step()
output_ids = create_output_ids_fn(model_output, model_input, False)
for evaluator in evaluate_obj_list:
evaluator.add_result(output_ids,
model_output,
model_target,
model_input,
batch_data=batch_data)
total_loss += loss.data
step_output = 'in train step {} loss: {}'.format(
steps, loss.data.item())
# print(step_output)
info(step_output)
steps += 1
pbar.update(batch_size)
return evaluate_obj_list, (total_loss / steps).item()
def train(model, dataset, batch_size, loss_function, optimizer):
print('in train')
total_loss = torch.Tensor([0])
count = torch.Tensor([0])
steps = 0
model.train()
for batch_data in data_loader(dataset, batch_size=batch_size, is_shuffle=True, drop_last=True):
# with torch.autograd.profiler.profile() as prof:
error_tokens = trans_to_cuda(torch.LongTensor(PaddedList(batch_data['error_tokens'])))
error_length = trans_to_cuda(torch.LongTensor(batch_data['error_length']))
ac_tokens_input = trans_to_cuda(torch.LongTensor(PaddedList(batch_data['ac_tokens_input'])))
ac_tokens_length = trans_to_cuda(torch.LongTensor(batch_data['ac_length']))
target_tokens = trans_to_cuda(torch.LongTensor(PaddedList(batch_data['target_tokens'], fill_value=TARGET_PAD_TOKEN)))
del batch_data["error_tokens"], batch_data["error_length"], batch_data["ac_tokens_input"], batch_data["ac_length"], batch_data["target_tokens"]
model.zero_grad()
log_probs = model.ibm_forward(error_tokens, error_length, ac_tokens_input, ac_tokens_length)
print('finish one step train')
loss = loss_function(log_probs.view(log_probs.shape[0]*log_probs.shape[1], -1), target_tokens.view(-1))
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 10)
optimizer.step()
print('finish optimizer step train')
cur_target_count = (torch.sum(ac_tokens_length.data.cpu()) - batch_size).float()
total_loss += (loss.data.cpu() * cur_target_count)
count += cur_target_count
steps += 1
print('step {} loss: {}'.format(steps, loss))
# print(prof)
sys.stdout.flush()
sys.stderr.flush()
return (total_loss/count).data[0]
def combine_train(p_model,
s_model,
seq_model,
dataset,
batch_size,
loss_fn,
p_optimizer,
s_optimizer,
delay_reward_fn,
baseline_fn,
delay_loss_fn,
vocab,
train_type=None,
predict_type='first',
include_error_reward=-10000,
pretrain=False,
random_action=None):
if train_type == 'p_model':
change_model_state([p_model], [s_model, seq_model])
policy_train = True
elif train_type == 's_model':
change_model_state([s_model, seq_model], [p_model])
policy_train = False
else:
change_model_state([], [p_model, s_model, seq_model])
policy_train = False
begin_tensor = s_model.begin_token
end_tensor = s_model.end_token
gap_tensor = s_model.gap_token
begin_len = 1
begin_token = vocab.word_to_id(vocab.begin_tokens[0])
end_token = vocab.word_to_id(vocab.end_tokens[0])
gap_token = vocab.word_to_id(vocab.addition_tokens[0])
step = 0
select_count = torch.LongTensor([0])
seq_count = torch.LongTensor([0])
decoder_input_count = torch.LongTensor([0])
total_seq_loss = torch.Tensor([0])
total_p_loss = torch.Tensor([0])
total_s_accuracy_top_k = {}
for data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True):
p_model.zero_grad()
s_model.zero_grad()
seq_model.zero_grad()
error_tokens = transform_to_cuda(
torch.LongTensor(PaddedList(data['error_tokens'])))
error_length = transform_to_cuda(torch.LongTensor(
data['error_length']))
error_action_masks = transform_to_cuda(
torch.ByteTensor(PaddedList(data['error_mask'], fill_value=0)))
max_len = torch.max(error_length)
error_token_masks = create_sequence_length_mask(
error_length, max_len=max_len.data.item(), gpu_index=gpu_index)
# add full code context information to each position word using BiRNN.
context_input, context_hidden = s_model.do_context_rnn(error_tokens)
# sample the action by interaction between policy model(p_model) and structed model(s_model)
if not pretrain:
action_probs_records_list, action_records_list, output_records_list, hidden = create_policy_action_batch(
p_model, s_model, context_input, policy_train=policy_train)
else:
action_probs_records_list, action_records_list, output_records_list, hidden = create_policy_action_batch(
p_model,
s_model,
context_input,
policy_train=True,
random_action=[0.8, 0.2])
action_probs_records = torch.stack(action_probs_records_list, dim=1)
action_records = torch.stack(action_records_list, dim=1)
output_records = torch.cat(output_records_list, dim=1)
masked_action_records = action_records.data.masked_fill_(
~error_token_masks, 0)
if pretrain:
masked_action_records = error_action_masks.byte(
) | masked_action_records.byte()
include_all_error = check_action_include_all_error(
masked_action_records, error_action_masks)
contain_all_error_count = torch.sum(include_all_error)
tokens_tensor, token_length, part_ac_tokens_list, ac_token_length = combine_spilt_tokens_batch_with_tensor(
output_records,
data['ac_tokens'],
masked_action_records,
data['token_map'],
gap_tensor,
begin_tensor,
end_tensor,
gap_token,
begin_token,
end_token,
gpu_index=gpu_index)
if predict_type == 'start':
decoder_input = [tokens[:-1] for tokens in part_ac_tokens_list]
decoder_length = [len(inp) for inp in decoder_input]
target_output = [tokens[1:] for tokens in part_ac_tokens_list]
elif predict_type == 'first':
decoder_input = [
tokens[begin_len:-1] for tokens in part_ac_tokens_list
]
decoder_length = [len(inp) for inp in decoder_input]
target_output = [
tokens[begin_len + 1:] for tokens in part_ac_tokens_list
]
token_length_tensor = transform_to_cuda(torch.LongTensor(token_length))
ac_token_tensor = transform_to_cuda(
torch.LongTensor(PaddedList(decoder_input, fill_value=0)))
ac_token_length_tensor = transform_to_cuda(
torch.LongTensor(decoder_length))
log_probs = seq_model.forward(tokens_tensor, token_length_tensor,
ac_token_tensor, ac_token_length_tensor)
target_output_tensor = transform_to_cuda(
torch.LongTensor(
PaddedList(target_output, fill_value=TARGET_PAD_TOKEN)))
s_loss = loss_fn(log_probs.view(-1, vocab.vocabulary_size),
target_output_tensor.view(-1))
remain_batch = torch.sum(masked_action_records, dim=1)
add_batch = torch.eq(remain_batch, 0).long()
remain_batch = remain_batch + add_batch
total_batch = torch.sum(error_token_masks, dim=1)
force_error_rewards = (
~include_all_error).float() * include_error_reward
delay_reward = delay_reward_fn(log_probs, target_output_tensor,
total_batch, remain_batch,
force_error_rewards)
delay_reward = torch.unsqueeze(delay_reward, dim=1).expand(-1, max_len)
delay_reward = delay_reward * error_token_masks.float()
if baseline_fn is not None:
baseline_reward = baseline_fn(delay_reward, error_token_masks)
total_reward = delay_reward - baseline_reward
else:
total_reward = delay_reward
# force_error_rewards = torch.unsqueeze(~include_all_error, dim=1).float() * error_token_masks.float() * include_error_reward
force_error_rewards = torch.unsqueeze(
~include_all_error, dim=1).float() * error_token_masks.float() * 0
p_loss = delay_loss_fn(action_probs_records, total_reward,
error_token_masks, force_error_rewards)
if math.isnan(p_loss):
print('p_loss is nan')
continue
# iterate record variable
step += 1
one_decoder_input_count = torch.sum(ac_token_length_tensor)
decoder_input_count += one_decoder_input_count.data.cpu()
total_seq_loss += s_loss.cpu().data.item(
) * one_decoder_input_count.float().cpu()
one_seq_count = torch.sum(error_length)
seq_count += one_seq_count.cpu()
total_p_loss += p_loss.cpu().data.item() * one_seq_count.float().cpu()
s_accuracy_top_k = calculate_accuracy_of_code_completion(
log_probs,
target_output_tensor,
ignore_token=TARGET_PAD_TOKEN,
topk_range=(1, 5),
gpu_index=gpu_index)
for key, value in s_accuracy_top_k.items():
total_s_accuracy_top_k[key] = s_accuracy_top_k.get(key, 0) + value
select_count_each_batch = torch.sum(masked_action_records, dim=1)
select_count = select_count + torch.sum(
select_count_each_batch).data.cpu()
print(
'train_type: {} step {} sequence model loss: {}, policy model loss: {}, contain all error count: {}, select of each batch: {}, total of each batch: {}, total decoder_input_cout: {}, topk: {}, '
.format(train_type, step, s_loss, p_loss, contain_all_error_count,
select_count_each_batch.data.tolist(),
error_length.data.tolist(),
one_decoder_input_count.data.item(), s_accuracy_top_k))
sys.stdout.flush()
sys.stderr.flush()
if train_type != 'p_model':
p_model.zero_grad()
if train_type != 's_model':
s_model.zero_grad()
seq_model.zero_grad()
if train_type == 'p_model':
torch.nn.utils.clip_grad_norm_(p_model.parameters(), 0.5)
p_loss.backward()
p_optimizer.step()
elif train_type == 's_model':
torch.nn.utils.clip_grad_norm_(s_model.parameters(), 8)
torch.nn.utils.clip_grad_norm_(seq_model.parameters(), 8)
s_loss.backward()
s_optimizer.step()
for key, value in total_s_accuracy_top_k.items():
total_s_accuracy_top_k[key] = total_s_accuracy_top_k.get(
key, 0) / decoder_input_count.data.item()
return (total_seq_loss / decoder_input_count.float()).data.item(), (
total_p_loss / seq_count.float()).data.item(), (
select_count.float() /
seq_count.float()).data.item(), total_s_accuracy_top_k
def evaluate(model,
dataset,
batch_size,
loss_fn,
id_to_word_fn,
file_path,
gap_token,
begin_tokens,
end_tokens,
predict_type,
use_force_train=False):
print('in evaluate')
model.train()
total_loss_in_train = torch.Tensor([0])
count = torch.Tensor([0])
count_in_train = torch.Tensor([0])
steps = 0
begin_len = len(begin_tokens) if begin_tokens is not None else 0
end_len = len(end_tokens) if end_tokens is not None else 0
for data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True):
error_tokens = transform_to_cuda(
torch.LongTensor(PaddedList(data['error_tokens'])))
error_length = transform_to_cuda(torch.LongTensor(
data['error_length']))
ac_tokens_input = transform_to_cuda(
torch.LongTensor(PaddedList(data['ac_tokens'])))
ac_tokens_length = transform_to_cuda(
torch.LongTensor(data['ac_length']))
token_maps = transform_to_cuda(
torch.LongTensor(
PaddedList(data['token_map'], fill_value=TARGET_PAD_TOKEN)))
# get split of error list. replace it to rl model
stay_label_list = choose_token_random_batch(data['error_length'],
data['error_mask'],
random_value=0.2)
part_tokens, part_ac_tokens = combine_spilt_tokens_batch(
data['error_tokens'], data['ac_tokens'], stay_label_list,
data['token_map'], gap_token, begin_tokens, end_tokens)
encoder_input = part_tokens
encoder_length = [len(inp) for inp in encoder_input]
if use_force_train:
if predict_type == 'start':
decoder_input = [tokens[:-1] for tokens in part_ac_tokens]
decoder_length = [len(inp) for inp in decoder_input]
target_output = [tokens[1:] for tokens in part_ac_tokens]
elif predict_type == 'first':
decoder_input = [
tokens[begin_len:-1] for tokens in part_ac_tokens
]
decoder_length = [len(inp) for inp in decoder_input]
target_output = [
tokens[begin_len + 1:] for tokens in part_ac_tokens
]
encoder_input = transform_to_cuda(
torch.LongTensor(PaddedList(encoder_input)))
encoder_length = transform_to_cuda(
torch.LongTensor(encoder_length))
decoder_input = transform_to_cuda(
torch.LongTensor(PaddedList(decoder_input)))
decoder_length = transform_to_cuda(
torch.LongTensor(decoder_length))
target_output = PaddedList(target_output,
fill_value=TARGET_PAD_TOKEN)
log_probs = model.forward(encoder_input, encoder_length,
decoder_input, decoder_length)
loss = loss_fn(
log_probs.view(-1, log_probs.shape[-1]),
transform_to_cuda(torch.LongTensor(target_output)).view(-1))
cur_target_count = torch.sum(decoder_length.data.cpu()).float()
total_loss_in_train += (loss.data.cpu() * cur_target_count)
count_in_train += cur_target_count
steps += 1
return (total_loss_in_train / count_in_train).data.item()
def train(model,
dataset,
batch_size,
loss_fn,
optimizer,
gap_token,
begin_tokens,
end_tokens,
predict_type='start'):
print('in train')
model.train()
total_loss = torch.Tensor([0])
count = torch.Tensor([0])
steps = 0
begin_len = len(begin_tokens) if begin_tokens is not None else 0
end_len = len(end_tokens) if end_tokens is not None else 0
for data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True):
error_tokens = transform_to_cuda(
torch.LongTensor(PaddedList(data['error_tokens'])))
error_length = transform_to_cuda(torch.LongTensor(
data['error_length']))
ac_tokens_input = transform_to_cuda(
torch.LongTensor(PaddedList(data['ac_tokens'])))
ac_tokens_length = transform_to_cuda(
torch.LongTensor(data['ac_length']))
token_maps = transform_to_cuda(
torch.LongTensor(
PaddedList(data['token_map'], fill_value=TARGET_PAD_TOKEN)))
model.zero_grad()
# get split of error list. replace it to rl model
stay_label_list = choose_token_random_batch(data['error_length'],
data['error_mask'],
random_value=0.2)
part_tokens, part_ac_tokens = combine_spilt_tokens_batch(
data['error_tokens'], data['ac_tokens'], stay_label_list,
data['token_map'], gap_token, begin_tokens, end_tokens)
print('part_tokens: length: {}/{},{}/{}'.format(
len(part_tokens[0]), len(data['error_tokens'][0]),
len(part_tokens[1]), len(data['error_tokens'][1])))
if predict_type == 'start':
encoder_input = part_tokens
encoder_length = [len(inp) for inp in encoder_input]
decoder_input = [tokens[:-1] for tokens in part_ac_tokens]
decoder_length = [len(inp) for inp in decoder_input]
target_output = [tokens[1:] for tokens in part_ac_tokens]
elif predict_type == 'first':
encoder_input = part_tokens
encoder_length = [len(inp) for inp in encoder_input]
decoder_input = [tokens[begin_len:-1] for tokens in part_ac_tokens]
decoder_length = [len(inp) for inp in decoder_input]
target_output = [
tokens[begin_len + 1:] for tokens in part_ac_tokens
]
encoder_input = transform_to_cuda(
torch.LongTensor(PaddedList(encoder_input)))
encoder_length = transform_to_cuda(torch.LongTensor(encoder_length))
decoder_input = transform_to_cuda(
torch.LongTensor(PaddedList(decoder_input)))
decoder_length = transform_to_cuda(torch.LongTensor(decoder_length))
target_output = PaddedList(target_output, fill_value=TARGET_PAD_TOKEN)
log_probs = model.forward(encoder_input, encoder_length, decoder_input,
decoder_length)
loss = loss_fn(
log_probs.view(-1, log_probs.shape[-1]),
transform_to_cuda(torch.LongTensor(target_output)).view(-1))
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 40)
optimizer.step()
cur_target_count = torch.sum(decoder_length.data.cpu()).float()
total_loss += (loss.data.cpu() * cur_target_count)
count += cur_target_count
steps += 1
return (total_loss / count).data.item()
def sample_and_save(model,
dataset,
batch_size,
loss_function,
parse_input_batch_data_fn,
parse_target_batch_data_fn,
do_sample=False,
print_output=False,
create_output_ids_fn=None,
evaluate_obj_list=[],
expand_output_and_target_fn=None,
add_data_record_fn=None,
db_path='',
table_name=''):
# total_loss = to_cuda(torch.Tensor([0]))
total_batch = to_cuda(torch.Tensor([0]))
saved_count = 0
steps = 1
for o in evaluate_obj_list:
o.clear_result()
model.eval()
total_saved_list = []
with tqdm(total=len(dataset)) as pbar:
with torch.no_grad():
for batch_data in data_loader(dataset,
batch_size=batch_size,
drop_last=True):
model.zero_grad()
# model_input = parse_input_batch_data(batch_data)
model_input = parse_input_batch_data_fn(batch_data,
do_sample=do_sample)
# model_output = model.forward(*model_input, test=do_sample)
if do_sample:
model_output = model.forward(*model_input, do_sample=True)
model_target = parse_target_batch_data_fn(batch_data)
model_output, model_target = expand_output_and_target_fn(
model_output, model_target)
else:
model_output = model.forward(*model_input)
model_target = parse_target_batch_data_fn(batch_data)
# loss = loss_function(*model_output, *model_target)
output_ids = create_output_ids_fn(model_output, model_input)
# total_loss += loss.data
total_batch += batch_size
# step_output = 'in evaluate step {} loss: {}, '.format(steps, loss.data.item())
step_output = 'in evaluate step {} '.format(steps)
for evaluator in evaluate_obj_list:
res = evaluator.add_result(output_ids,
model_output,
model_target,
model_input,
batch_data=batch_data)
step_output += res
# print(step_output)
info(step_output)
saved_list = add_data_record_fn(output_ids, model_output,
batch_data)
total_saved_list += saved_list
if steps % 100 == 0:
create_table(db_path, table_name)
insert_items(db_path, table_name, total_saved_list)
saved_count += len(total_saved_list)
print('saved {} record in total {}. '.format(
saved_count, total_batch.item()))
total_saved_list = []
if print_output and steps % 100 == 0:
pass
# output_ids = output_ids.tolist()
# target_ids = batch_data['ac_tokens']
# is_copy = (is_copy > 0.5).tolist()
# target_is_copy = target_is_copy.tolist()
# value_output = torch.squeeze(torch.topk(F.softmax(value_output, dim=-1), k=1, dim=-1)[1], dim=-1)
# value_output = value_output.tolist()
# target_ac_tokens = target_ac_tokens.tolist()
# pointer_output = torch.squeeze(torch.topk(F.softmax(pointer_output, dim=-1), k=1, dim=-1)[1], dim=-1)
# pointer_output = pointer_output.tolist()
# target_pointer_output = target_pointer_output.tolist()
# target_length = torch.sum(output_mask, dim=-1)
# target_length = target_length.tolist()
# for out, tar, cop, tar_cop, val, tar_val, poi, tar_poi, tar_len in zip(output_ids, target_ids, is_copy,
# target_is_copy, value_output,
# target_ac_tokens,
# pointer_output,
# target_pointer_output, target_length):
# # for out, tar, in zip(output_ids, target_ids):
# out_code, end_pos = convert_one_token_ids_to_code(out, id_to_word_fn=vocab.id_to_word, start=start_id,
# end=end_id, unk=unk_id)
# tar_code, tar_end_pos = convert_one_token_ids_to_code(tar[1:], id_to_word_fn=vocab.id_to_word, start=start_id,
# end=end_id, unk=unk_id)
# info('-------------- step {} ------------------------'.format(steps))
# info('output: {}'.format(out_code))
# info('target: {}'.format(tar_code))
# cop = [str(c) for c in cop]
# tar_cop = [str(int(c)) for c in tar_cop]
# poi = [str(c) for c in poi]
# tar_poi = [str(c) for c in tar_poi]
# info('copy output: {}'.format(' '.join(cop[:tar_len])))
# info('copy target: {}'.format(' '.join(tar_cop[:tar_len])))
# info('pointer output: {}'.format(' '.join(poi[:tar_len])))
# info('pointer target: {}'.format(' '.join(tar_poi[:tar_len])))
#
# value_list = []
# target_list = []
# for c, v, t in zip(tar_cop, val, tar_val):
# if c == '1':
# value_list += ['<COPY>']
# target_list += ['<COPY>']
# else:
# value_list += [vocab.id_to_word(int(v))]
# target_list += [vocab.id_to_word(int(t))]
# info('value output: {}'.format(' '.join(value_list[:tar_len])))
# info('value target: {}'.format(' '.join(target_list[:tar_len])))
steps += 1
pbar.update(batch_size)
create_table(db_path, table_name)
insert_items(db_path, table_name, total_saved_list)
saved_count += len(total_saved_list)
print('saved {} record in total {}. '.format(saved_count,
total_batch.item()))
return evaluate_obj_list
def multi_step_evaluate(model,
dataset,
batch_size,
parse_input_batch_data_fn,
parse_target_batch_data_fn,
do_sample=False,
print_output=False,
create_output_ids_fn=None,
evaluate_obj_list=[],
expand_output_and_target_fn=None,
max_step_times=0,
vocabulary=None,
file_path='',
create_multi_step_next_input_batch_fn=None,
extract_includes_fn=lambda x: x['includes'],
print_output_fn=None,
do_beam_search=False,
target_file_path='main.out',
log_file_path='main.log',
do_save_data=False,
max_save_distance=None,
save_records_to_database=False,
db_path='',
table_name='',
change_output_records_to_batch_fn=None,
create_save_database_records_fn=None,
error_stop_type='normal'):
total_loss = to_cuda(torch.Tensor([0]))
total_batch = to_cuda(torch.Tensor([0]))
steps = 0
compile_evaluator = CompileResultEvaluate()
compile_evaluator.clear_result()
for o in evaluate_obj_list:
o.clear_result()
model.eval()
from common.pycparser_util import tokenize_by_clex_fn
tokenize_fn = tokenize_by_clex_fn()
save_data_dict = {}
save_records_list = []
# file_path = add_pid_to_file_path(file_path)
# target_file_path = add_pid_to_file_path(target_file_path)
with tqdm(total=len(dataset)) as pbar:
with torch.no_grad():
for batch_data in data_loader(dataset,
batch_size=batch_size,
drop_last=False):
model.zero_grad()
input_data = batch_data.copy()
final_output_list = []
output_records_list = []
continue_list = [True for _ in range(batch_size)]
result_list = [False for _ in range(batch_size)]
result_records_list = []
sample_steps = [-1 for _ in range(batch_size)]
error_count_list = batch_data['error_count']
for i in range(max_step_times):
model_input = parse_input_batch_data_fn(input_data,
do_sample=True)
model_output = model.forward(*model_input,
do_sample=True,
do_beam_search=do_beam_search)
input_data, final_output, output_records, final_output_name_list, continue_list = create_multi_step_next_input_batch_fn(
input_data, model_input, model_output, continue_list,
do_beam_search)
final_output_list += [final_output]
output_records_list += [output_records]
continue_list, result_list, cur_error_count_list = compile_code_ids_list(
final_output_name_list,
continue_list,
result_list,
vocabulary=vocabulary,
includes_list=extract_includes_fn(input_data),
file_path=file_path,
target_file_path=target_file_path,
log_file_path=log_file_path,
do_compile_pool=True,
need_transform=False)
if error_stop_type == 'oracle':
reject_list = [
True if c and n > o else False
for c, o, n in zip(continue_list, error_count_list,
cur_error_count_list)
]
elif error_stop_type == 'normal':
reject_list = [False for _ in range(batch_size)]
error_count_list = [
n if n < o and n >= 0 else o
for o, n in zip(error_count_list, cur_error_count_list)
]
for i_f, rej in enumerate(reject_list):
if rej:
# use last output
final_output_name_list[i_f] = input_data[
'last_input_seq_name'][i_f]
continue_list[i_f] = False
sample_steps = [
i + 1 if s == -1 and not c and not r else s for s, c, r
in zip(sample_steps, continue_list, reject_list)
]
sample_steps = [
i if s == -1 and not c and r else s for s, c, r in zip(
sample_steps, continue_list, reject_list)
]
result_records_list += [result_list]
if sum(continue_list) == 0:
break
sample_steps = [
max_step_times if s == -1 else s for s in sample_steps
]
if do_save_data:
batch_data['input_seq_name'] = batch_data[
'final_output_name']
save_res_dict = save_addition_data(
original_states=batch_data,
states=input_data,
tokenize_fn=tokenize_fn,
batch_size=batch_size,
file_path=file_path,
target_file_path=target_file_path,
vocabulary=vocabulary,
max_distande=max_save_distance,
only_error=True)
for k, v in save_res_dict.items():
save_data_dict[k] = save_data_dict.get(k, []) + v
if save_records_to_database:
batch_output_records = change_output_records_to_batch_fn(
output_records_list, sample_steps)
records_list = create_save_database_records_fn(
batch_data, sample_steps, final_output_name_list,
result_list, batch_output_records, input_data)
save_records_list += records_list
step_output = 'in evaluate step {}: '.format(steps)
res = compile_evaluator.add_result(result_list)
step_output += res
for evaluator in evaluate_obj_list:
# customer evaluator interface
res = evaluator.add_result(result_list,
batch_data=batch_data)
step_output += res
# print(step_output)
info(step_output)
if print_output and steps % 1 == 0:
print_output_fn(output_records=output_records_list,
final_output=final_output_list,
batch_data=batch_data,
step_i=steps,
vocabulary=vocabulary,
compile_result_list=result_records_list)
steps += 1
pbar.update(batch_size)
evaluate_obj_list = [compile_evaluator] + evaluate_obj_list
if save_records_to_database:
create_table(db_path,
DATA_RECORDS_DEEPFIX,
replace_table_name=table_name)
run_sql_statment(db_path,
DATA_RECORDS_DEEPFIX,
'insert_ignore',
save_records_list,
replace_table_name=table_name)
if steps == 0:
t_loss = 0
else:
t_loss = (total_loss / steps).item()
return evaluate_obj_list, t_loss, save_data_dict
def evaluate(model,
dataset,
batch_size,
loss_function,
parse_input_batch_data_fn,
parse_target_batch_data_fn,
do_sample=False,
print_output=False,
create_output_ids_fn=None,
evaluate_obj_list=[],
expand_output_and_target_fn=None):
total_loss = to_cuda(torch.Tensor([0]))
total_batch = to_cuda(torch.Tensor([0]))
steps = 0
for o in evaluate_obj_list:
o.clear_result()
model.eval()
with tqdm(total=len(dataset)) as pbar:
with torch.no_grad():
for batch_data in data_loader(dataset,
batch_size=batch_size,
drop_last=True):
model.zero_grad()
# model_input = parse_input_batch_data(batch_data)
model_input = parse_input_batch_data_fn(batch_data,
do_sample=do_sample)
# model_output = model.forward(*model_input, test=do_sample)
if do_sample:
model_output = model.forward(*model_input, do_sample=True)
model_target = parse_target_batch_data_fn(batch_data)
model_output, model_target = expand_output_and_target_fn(
model_output, model_target)
else:
model_output = model.forward(*model_input)
model_target = parse_target_batch_data_fn(batch_data)
loss = loss_function(*model_output, *model_target)
output_ids = create_output_ids_fn(model_output, model_input,
do_sample)
total_loss += loss.data
total_batch += batch_size
step_output = 'in evaluate step {} loss: {}, '.format(
steps, loss.data.item())
for evaluator in evaluate_obj_list:
res = evaluator.add_result(output_ids,
model_output,
model_target,
model_input,
batch_data=batch_data)
step_output += res
# print(step_output)
info(step_output)
if print_output and steps % 10 == 0:
pass
steps += 1
pbar.update(batch_size)
return evaluate_obj_list, (total_loss / steps).item()
def accuracy_evaluate(model, dataset, batch_size, loss_function, vocabulary):
total_loss = torch.Tensor([0]).cuda(GPU_INDEX)
steps = torch.Tensor([0]).cuda(GPU_INDEX)
accuracy_dict = None
sample_predict = []
sample_target = []
sample_prob = []
for batch_data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True):
identifier_scope_mask, is_identifier, lengths, target, terminal_mask, tokens, update_mask, has_identifier\
= parse_batch_data(
batch_data)
log_probs = model.forward(tokens, identifier_scope_mask, is_identifier,
update_mask, terminal_mask, lengths,
has_identifier)
batch_log_probs = log_probs.contiguous().view(
-1,
list(log_probs.size())[-1])
ori_target = target
target, idx_unsort = torch_util.pack_padded_sequence(
autograd.Variable(torch.LongTensor(target)).cuda(GPU_INDEX),
lengths,
batch_firse=True,
GPU_INDEX=GPU_INDEX)
target, _ = torch_util.pad_packed_sequence(target,
idx_unsort,
pad_value=PAD_TOKEN,
batch_firse=True,
GPU_INDEX=GPU_INDEX)
loss = loss_function(batch_log_probs, target.view(-1))
total_loss += loss.data
steps += torch.sum(lengths.data)
# print("target size:{}".format(target.size()))
# print("batch log size:{}".format(log_probs.size()))
topk_accuracy = calculate_accuracy_of_code_completion(
log_probs, target, ignore_token=PAD_TOKEN, gpu_index=GPU_INDEX)
if accuracy_dict is None:
accuracy_dict = topk_accuracy
else:
for k, v in topk_accuracy.items():
accuracy_dict[k] += topk_accuracy[k]
for l, p, t in zip(lengths, log_probs, ori_target):
p = p[:l]
p = torch.unsqueeze(p, dim=0)
a, b, pro = get_predict_and_target_tokens(p, [t[:l]],
vocabulary.id_to_word,
k=5)
sample_predict.append(a)
sample_target.append(b)
sample_prob.append(pro)
accuracy_dict = {
k: float(v) / steps.item()
for k, v in accuracy_dict.items()
}
sample_predict = more_itertools.flatten(sample_predict)
sample_target = more_itertools.flatten(sample_target)
sample_prob = more_itertools.flatten(sample_prob)
return total_loss / steps, accuracy_dict, sample_predict, sample_target, sample_prob, steps
def train(model, dataset, batch_size, loss_function, optimizer, clip_norm,
epoch_ratio, evaluate_object_list, desc, label_preprocess_fn):
total_loss = to_cuda(torch.Tensor([0]))
steps = to_cuda(torch.Tensor([0]))
# previous_char_max = 0
# previous_word_max = 0
for o in evaluate_object_list:
o.clear_result()
model.train()
with tqdm(total=len(dataset) // batch_size, desc=desc,
leave=False) as pbar:
for batch_data in data_loader(dataset,
batch_size=batch_size,
is_shuffle=True,
drop_last=True,
epoch_ratio=epoch_ratio):
# print(batch_data['terminal_mask'])
# print('batch_data size: ', len(batch_data['terminal_mask'][0]), len(batch_data['terminal_mask'][0][0]))
# res = list(more_itertools.collapse(batch_data['terminal_mask']))
# print('res len: ', len(res))
# res = util.padded(batch_data['terminal_mask'], deepcopy=True, fill_value=0)
# print('batch_data size: ', len(res[0]), len(res[0][0]))
# res = list(more_itertools.collapse(res))
# print('res len: ', len(res))
# previous_char_max = max(previous_char_max, max(batch_data['q1_char_length']), max(batch_data['q2_char_length']))
# previous_word_max = max(previous_word_max, max(batch_data['q1_word_length']), max(batch_data['q2_word_length']))
# print('max q1_length:{},{}'.format(max(batch_data['q1_char_length']), max(batch_data['q1_word_length'])))
# print("max q2_length:{},{}".format(max(batch_data['q2_char_length']), max(batch_data['q2_word_length'])))
# print("previous_char_max:{}, previous_word_max:{}".format(previous_char_max, previous_word_max))
model.zero_grad()
log_probs = model.forward(batch_data)
# log_probs.register_hook(create_hook_fn("log_probs"))
# print("log_probs sizze:{}".format(log_probs.size()))
label = label_preprocess_fn(batch_data)
loss = loss_function(log_probs, label)
# loss.register_hook(create_hook_fn("loss"))
loss.backward()
if clip_norm is not None:
torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
# print()
# print("The loss is nan:{}".format(is_nan(loss.detach())))
# print("The loss grad is nan:{}".format(is_nan(loss.grad)))
# print("The log_probs is nan:{}".format(is_nan(log_probs.detach())))
# print("The log_probs grad is nan:{}".format(is_nan(log_probs.grad)))
# for name, param in model.named_parameters():
# print("name of {}: has nan:{}".format(name, is_nan(param.detach())))
# print("the gradient of {}: has nan:{}".format(name, is_nan(param.grad)))
# if HAS_NAN:
# for k, v in batch_data.items():
# print("{}:{}".format(k, show_tensor(v)))
# print("{}:{}".format("target", show_tensor(target)))
# print()
optimizer.step()
# print("loss：{}".format(loss.data))
total_loss += loss.data
steps += 1
for evaluator in evaluate_object_list:
evaluator.add_result(log_probs, label, batch_data=batch_data)
pbar.update(1)
return evaluate_object_list, total_loss / steps
def evaluate(model, dataset, loss_function, batch_size, start, end, unk, id_to_word_fn, file_path='test.c', use_force_train=False):
print('in evaluate')
global print_count
steps = 0
success = 0
total = 0
total_loss = torch.Tensor([0])
count = torch.Tensor([0])
total_correct = torch.Tensor([0])
total_compare_correct = torch.Tensor([0])
total_loss_in_train = torch.Tensor([0])
count_in_train = torch.Tensor([0])
model.eval()
for batch_data in data_loader(dataset, batch_size=batch_size, is_shuffle=True, drop_last=True):
error_tokens = trans_to_cuda(torch.LongTensor(PaddedList(batch_data['error_tokens'])))
error_length = trans_to_cuda(torch.LongTensor(PaddedList(batch_data['error_length'])))
ac_tokens_input = trans_to_cuda(torch.LongTensor(PaddedList(batch_data['ac_tokens_input'])))
ac_tokens_length = trans_to_cuda(torch.LongTensor(PaddedList(batch_data['ac_length'])))
target_tokens = trans_to_cuda(torch.LongTensor(PaddedList(batch_data['target_tokens'], fill_value=TARGET_PAD_TOKEN)))
target_tokens_padded = padded_tensor_one_dim_to_length(target_tokens.float(), dim=1,
padded_length=MAX_LENGTH,
is_cuda=available_cuda, gpu_index=GPU_INDEX, fill_value=TARGET_PAD_TOKEN).long()
del batch_data["error_tokens"], batch_data["error_length"], batch_data["ac_tokens_input"], batch_data[
"ac_length"], batch_data["target_tokens"]
includes = batch_data['includes']
loss_in_train = None
# calculate loss like train
if use_force_train:
log_probs = model.ibm_forward(error_tokens, error_length, ac_tokens_input, ac_tokens_length)
print('finish one step train')
loss_in_train = loss_function(log_probs.view(log_probs.shape[0] * log_probs.shape[1], -1), target_tokens.view(-1))
cur_target_count = (torch.sum(ac_tokens_length.data.cpu()) - batch_size).float()
total_loss_in_train += (loss_in_train.data.cpu() * cur_target_count)
count_in_train += cur_target_count
else:
log_probs = model._ibm_test_forward(error_tokens, error_length)
# do evaluate
cur_batch_len = len(batch_data['includes'])
predict_log_probs = torch.transpose(log_probs, 0, 1)
target_label = torch.transpose(target_tokens_padded, 0, 1)
cur_loss = torch.Tensor([0])
cur_step = torch.Tensor([0])
cur_correct = torch.Tensor([0])
is_compare_success = torch.Tensor([1] * batch_size)
for i, step_output in enumerate(predict_log_probs):
step_target = target_label[i, :].view(batch_size)
batch_loss = loss_function(step_output.view(batch_size, -1), step_target)
batch_predict_label = step_output.view(batch_size, -1).topk(1)[1].view(batch_size)
in_step_count = step_target.ne(TARGET_PAD_TOKEN).sum().float()
cur_loss += (batch_loss.data.cpu() * in_step_count.cpu())
cur_step += in_step_count.data.cpu()
batch_correct = (step_target.ne(TARGET_PAD_TOKEN) & step_target.eq(batch_predict_label)).sum().cpu().float()
batch_error = step_target.ne(TARGET_PAD_TOKEN) & step_target.ne(batch_predict_label)
is_compare_success[batch_error.cpu()] = 0
if batch_correct > 16:
print(batch_correct)
# batch_correct = (step_target.ne(TARGET_PAD_TOKEN) & step_target.eq(step_target)).sum().cpu().float()
cur_correct += batch_correct
total_loss += cur_loss
total_correct += cur_correct
count += cur_step
total_compare_correct += is_compare_success.sum().float()
_, output_tokens = torch.max(log_probs, dim=2)
cur_success = 0
for token_ids, include, ac_token_ids in zip(output_tokens, includes, ac_tokens_input):
if print_count % 100 == 0:
code = convert_one_token_ids_to_code(token_ids.tolist(), id_to_word_fn, start, end, unk, include)
ac_code = convert_one_token_ids_to_code(ac_token_ids.tolist(), id_to_word_fn, start, end, unk, include)
print(code)
print(ac_code)
# res = compile_c_code_by_gcc(code, file_path)
res = False
if res:
cur_success += 1
success += cur_success
steps += 1
total += cur_batch_len
print_count += 1
print('step {} accuracy: {}, loss: {}, correct: {}, compare correct: {}, loss according train: {}'.format(steps, cur_success/cur_batch_len, (cur_loss/cur_step).data[0], (cur_correct/cur_step).data[0], (is_compare_success.sum()/cur_batch_len).data[0], loss_in_train))
sys.stdout.flush()
sys.stderr.flush()
return (total_loss/count).data[0], float(success/total), (total_correct/count).data[0], (total_compare_correct/total).data[0], (total_loss_in_train/count_in_train).data[0]
def multi_step_evaluate(model,
dataset,
batch_size,
parse_input_batch_data_fn,
parse_target_batch_data_fn,
do_sample=False,
print_output=False,
create_output_ids_fn=None,
evaluate_obj_list=[],
expand_output_and_target_fn=None,
max_step_times=0,
vocabulary=None,
file_path='',
create_multi_step_next_input_batch_fn=None,
extract_includes_fn=lambda x: x['includes'],
print_output_fn=None,
do_beam_search=False,
target_file_path='main.out',
do_save_data=False,
max_save_distance=None,
save_records_to_database=False,
db_path='',
table_name='',
change_output_records_to_batch_fn=None,
create_save_database_records_fn=None):
total_loss = to_cuda(torch.Tensor([0]))
total_batch = to_cuda(torch.Tensor([0]))
steps = 0
compile_evaluator = CompileResultEvaluate()
compile_evaluator.clear_result()
for o in evaluate_object_list:
o.clear_result()
model.eval()
from common.pycparser_util import tokenize_by_clex_fn
tokenize_fn = tokenize_by_clex_fn()
# file_path = add_pid_to_file_path(file_path)
# target_file_path = add_pid_to_file_path(target_file_path)
with tqdm(total=len(dataset)) as pbar:
with torch.no_grad():
for batch_data in data_loader(dataset,
batch_size=batch_size,
drop_last=False):
model.zero_grad()
input_data = batch_data.copy()
final_output_list = []
output_records_list = []
continue_list = [True for _ in range(batch_size)]
result_list = [False for _ in range(batch_size)]
result_records_list = []
sample_steps = [-1 for _ in range(batch_size)]
for i in range(max_step_times):
model_input = parse_input_batch_data_fn(input_data,
do_sample=True)
model_output = model.forward(*model_input,
do_sample=True,
do_beam_search=do_beam_search)
input_data, final_output, output_records, final_output_name_list, continue_list = create_multi_step_next_input_batch_fn(
input_data, model_input, model_output, continue_list,
do_beam_search)
final_output_list += [final_output]
output_records_list += [output_records]
continue_list, result_list = compile_code_ids_list(
final_output_name_list,
continue_list,
result_list,
vocabulary=vocabulary,
includes_list=extract_includes_fn(input_data),
file_path=file_path,
target_file_path=target_file_path,
do_compile_pool=True,
need_transform=False)
sample_steps = [
i + 1 if s == -1 and not c else s
for s, c in zip(sample_steps, continue_list)
]
result_records_list += [result_list]
if sum(continue_list) == 0:
break
sample_steps = [
max_step_times if s == -1 else s for s in sample_steps
]
step_output = 'in evaluate step {}: '.format(steps)
res = compile_evaluator.add_result(result_list)
step_output += res
for evaluator in evaluate_obj_list:
# customer evaluator interface
res = evaluator.add_result(result_list,
batch_data=batch_data)
step_output += res
# print(step_output)
info(step_output)
if print_output and steps % 1 == 0:
print_output_fn(output_records=output_records_list,
final_output=final_output_list,
batch_data=batch_data,
step_i=steps,
vocabulary=vocabulary,
compile_result_list=result_records_list)
steps += 1
pbar.update(batch_size)
evaluate_obj_list = [compile_evaluator] + evaluate_obj_list
t_loss = (total_loss / steps).item() if steps != 0 else 0
return evaluate_obj_list, t_loss
