def prep_for_training(num_train_optimization_steps, _config):
tokenizer = BertTokenizer.from_pretrained(
_config["bert_model"], do_lower_case=_config["do_lower_case"])
# TODO:Change model here
model = MultimodalBertForSequenceClassification.multimodal_from_pretrained(
_config["bert_model"],
newly_added_config=_config,
cache_dir=_config["cache_dir"],
num_labels=_config["num_labels"])
model.to(_config["device"])
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=_config["learning_rate"],
warmup=_config["warmup_proportion"],
t_total=num_train_optimization_steps)
return model, optimizer, tokenizer
model = BertModel.from_pretrained(_config["bert_model"])
with open(os.path.join(_config["dataset_location"],'humor_splitdata_sdk.pkl'), 'rb') as handle:
all_data = pickle.load(handle)
train_data = all_data["train"]
dev_data= all_data["dev"]
test_data=all_data["test"]
#humor_data=train_data+dev_data+test_data
humor_data=train_data
tokenizer = BertTokenizer.from_pretrained(_config["bert_model"], do_lower_case=_config["do_lower_case"])
output_mode = _config["output_mode"]
humor_dataset=get_appropriate_dataset(humor_data,tokenizer, output_mode,_config)
humor_dataloader = DataLoader(humor_dataset, batch_size=1,shuffle=_config["shuffle"], num_workers=_config["num_workers"])
humor_bert_embeddings=[]
model.eval()
with torch.no_grad():
for step, batch in enumerate(tqdm(humor_dataloader, desc="Iteration")):
batch = tuple(t.to(_config["device"]) for t in batch)
input_ids, visual,acoustic,input_mask, segment_ids, label_ids = batch
encoder_output, pooled_output=model(input_ids.view(-1,_config["max_seq_length"]), segment_ids.view(-1,_config["max_seq_length"]), input_mask.view(-1,_config["max_seq_length"]), output_all_encoded_layers=False)
def set_up_data_loader(_config):
# #MUST remove it
# train_examples = None
# num_train_optimization_steps = None
# if args.do_train:
# train_examples = processor.get_train_examples(args.data_dir)
# #print("Train examples:",train_examples)
# #assert False
# num_train_optimization_steps = int(
# len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs
# if args.local_rank != -1:
# num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size()
with open(os.path.join(_config["dataset_location"], 'all_mod_data.pickle'),
'rb') as handle:
all_data = pickle.load(handle)
train_data = all_data["train"]
dev_data = all_data["dev"]
test_data = all_data["test"]
if (_config["prototype"]):
train_data = train_data[:100]
dev_data = dev_data[:100]
test_data = test_data[:100]
tokenizer = BertTokenizer.from_pretrained(
_config["bert_model"], do_lower_case=_config["do_lower_case"])
output_mode = _config["output_mode"]
train_dataset = get_appropriate_dataset(train_data, tokenizer, output_mode,
_config)
dev_dataset = get_appropriate_dataset(dev_data, tokenizer, output_mode,
_config)
test_dataset = get_appropriate_dataset(test_data, tokenizer, output_mode,
_config)
#print("train_dataset:",train_dataset)
#print(len(train_dataset),_config["train_batch_size"],_config["gradient_accumulation_steps"], _config["num_train_epochs"])
num_train_optimization_steps = int(
len(train_dataset) / _config["train_batch_size"] /
_config["gradient_accumulation_steps"]) * _config["num_train_epochs"]
#print("num_tr_opt_st:",num_train_optimization_steps)
#print("Train len:",len(train_dataset)," dev:",len(dev_dataset)," test:",len(test_dataset))
train_dataloader = DataLoader(train_dataset,
batch_size=_config["train_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
dev_dataloader = DataLoader(dev_dataset,
batch_size=_config["dev_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
test_dataloader = DataLoader(test_dataset,
batch_size=_config["test_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
#print(train_X.shape,train_Y.shape,dev_X.shape,dev_Y.shape,test_X.shape,test_Y.shape)
#data_loader = test_data_loader(train_X,train_Y,_config)
return train_dataloader, dev_dataloader, test_dataloader, num_train_optimization_steps
def set_up_data_loader(_config):
dataset_id_file = os.path.join(_config["dataset_location"],
"revised_id_list.pkl")
dataset_id = load_pickle(dataset_id_file)
train = dataset_id['train']
dev = dataset_id['dev']
test = dataset_id['test']
#print("real sizes:",len(train),len(dev),len(test))
if (_config["prototype"]):
train_num = _config["prot_train"]
dev_num = _config["prot_dev"]
test_num = _config["prot_test"]
#dev=dataset_id['train']
train = train[:train_num]
dev = dev[:dev_num]
test = test[:test_num]
#print("train:",train)
#print("dev:",dev)
data_path = _config["dataset_location"]
facet_file = os.path.join(data_path, 'revised_facet.pkl')
covarep_file = os.path.join(data_path, "covarep.pkl")
word_vec_file = os.path.join(data_path, "glove_index.pkl")
y_labels = os.path.join(data_path, "video_labels.pkl")
id_2_word_file = os.path.join(data_path, "ets_word_list.pkl")
word_aligned_facet_sdk = load_pickle(facet_file)
word_aligned_covarep_sdk = load_pickle(covarep_file)
word_embedding_idx_sdk = load_pickle(word_vec_file)
y_labels_sdk = load_pickle(y_labels)
id_2_word = load_pickle(id_2_word_file)['data']
#print(id_2_word)
all_data = (word_aligned_facet_sdk, word_aligned_covarep_sdk,
word_embedding_idx_sdk, y_labels_sdk, id_2_word)
tokenizer = BertTokenizer.from_pretrained(
_config["bert_model"], do_lower_case=_config["do_lower_case"])
training_set = ETSDataset(train, _config, all_data, tokenizer)
dev_set = ETSDataset(dev, _config, all_data, tokenizer)
test_set = ETSDataset(test, _config, all_data, tokenizer)
#print("dataset init")
#print("In train dataloader:",_config["train_batch_size"])
train_dataloader = DataLoader(training_set,
batch_size=_config["train_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
dev_dataloader = DataLoader(dev_set,
batch_size=_config["dev_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
test_dataloader = DataLoader(test_set,
batch_size=_config["test_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
num_train_optimization_steps = int(
len(training_set) / _config["train_batch_size"] /
_config["gradient_accumulation_steps"]) * _config["num_train_epochs"]
print("num_t:{0}".format(num_train_optimization_steps))
#print("data loader prepared")
#my_logger.debug(train_X.shape,train_Y.shape,dev_X.shape,dev_Y.shape,test_X.shape,test_Y.shape)
#data_loader = test_data_loader(train_X,train_Y,_config)
return train_dataloader, dev_dataloader, test_dataloader, num_train_optimization_steps
# with open(os.path.join(_config["dataset_location"],'all_mod_data.pickle'), 'rb') as handle:
# all_data = pickle.load(handle)
# train_data = all_data["train"]
# dev_data=all_data["dev"]
# test_data=all_data["test"]
# if(_config["prototype"]):
# train_data=train_data[:100]
# dev_data=dev_data[:100]
# test_data=test_data[:100]
tokenizer = BertTokenizer.from_pretrained(
_config["bert_model"], do_lower_case=_config["do_lower_case"])
output_mode = _config["output_mode"]
train_dataset = get_appropriate_dataset(train_data, tokenizer, output_mode,
_config)
dev_dataset = get_appropriate_dataset(dev_data, tokenizer, output_mode,
_config)
test_dataset = get_appropriate_dataset(test_data, tokenizer, output_mode,
_config)
#print("train_dataset:",train_dataset)
#print(len(train_dataset),_config["train_batch_size"],_config["gradient_accumulation_steps"], _config["num_train_epochs"])
num_train_optimization_steps = int(
len(train_dataset) / _config["train_batch_size"] /
_config["gradient_accumulation_steps"]) * _config["num_train_epochs"]
#print("num_tr_opt_st:",num_train_optimization_steps)
#print("Train len:",len(train_dataset)," dev:",len(dev_dataset)," test:",len(test_dataset))
train_dataloader = DataLoader(train_dataset,
batch_size=_config["train_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
dev_dataloader = DataLoader(dev_dataset,
batch_size=_config["dev_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
test_dataloader = DataLoader(test_dataset,
batch_size=_config["test_batch_size"],
shuffle=_config["shuffle"],
num_workers=_config["num_workers"])
#print(train_X.shape,train_Y.shape,dev_X.shape,dev_Y.shape,test_X.shape,test_Y.shape)
#data_loader = test_data_loader(train_X,train_Y,_config)
return train_dataloader, dev_dataloader, test_dataloader, num_train_optimization_steps