def test():
# 配置文件
cf = Config('./config.yaml')
# 有GPU用GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# 测试数据
test_data = NewsDataset("./data/cnews_final_test.txt", cf.max_seq_len)
test_dataloader = DataLoader(test_data,
batch_size=cf.batch_size,
shuffle=True)
# 模型
config = BertConfig("./output/pytorch_bert_config.json")
model = BertForSequenceClassification(config, num_labels=cf.num_labels)
model.load_state_dict(torch.load("./output/pytorch_model.bin"))
# 把模型放到指定设备
model.to(device)
# 让模型并行化运算
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
# 训练
start_time = time.time()
data_len = len(test_dataloader)
model.eval()
y_pred = np.array([])
y_test = np.array([])
# for step,batch in enumerate(tqdm(test_dataloader,"batch",total=len(test_dataloader))):
for step, batch in enumerate(test_dataloader):
label_id = batch['label_id'].squeeze(1).to(device)
word_ids = batch['word_ids'].to(device)
segment_ids = batch['segment_ids'].to(device)
word_mask = batch['word_mask'].to(device)
loss = model(word_ids, segment_ids, word_mask, label_id)
with torch.no_grad():
pred = get_model_labels(model, word_ids, segment_ids, word_mask)
y_pred = np.hstack((y_pred, pred))
y_test = np.hstack((y_test, label_id.to("cpu").numpy()))
# 评估
print("Precision, Recall and F1-Score...")
print(
metrics.classification_report(y_test,
y_pred,
target_names=get_labels('./data/label')))
# 混淆矩阵
print("Confusion Matrix...")
cm = metrics.confusion_matrix(y_test, y_pred)
print(cm)
def main():
test_df = pd.read_csv(TEST_PATH)
with timer('preprocessing text'):
test_df['comment_text'] = test_df['comment_text'].astype(str)
test_df = test_df.fillna(0)
with timer('load embedding'):
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH,
cache_dir=None,
do_lower_case=True)
X_text = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"),
max_len, tokenizer)
with timer('train'):
model = BertForSequenceClassification(bert_config, num_labels=n_labels)
model.load_state_dict(torch.load(model_path))
model = model.to(device)
test_dataset = torch.utils.data.TensorDataset(
torch.tensor(X_text, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=batch_size * 2,
shuffle=False)
test_pred = inference(model, test_loader, device, n_labels)
del model
gc.collect()
torch.cuda.empty_cache()
submission = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_pred.reshape(-1)
})
submission.to_csv('submission.csv', index=False)
LOGGER.info(submission.head())
def get_trained_model(fine_tuned="bert_pytorch.bin",
device=torch.device('cuda')):
model = None
y_columns = [
'toxic', "severe_toxic", "obscene", "threat", "insult", "identity_hate"
]
pretrain_data_folder = PRETRAIND_PICKLE_AND_MORE
if not os.path.exists(pretrain_data_folder + "/" + fine_tuned):
pretrain_data_folder = '/home/working'
if os.path.exists(pretrain_data_folder + "/" + fine_tuned):
output_model_file = pretrain_data_folder + "/" + fine_tuned
bert_config = BertConfig.from_json_file(pretrain_data_folder +
"/bert_config.json")
# Run validation
# The following 2 lines are not needed but show how to download the model for prediction
model = BertForSequenceClassification(bert_config,
num_labels=len(y_columns))
model.load_state_dict(torch.load(output_model_file))
model.to(device)
return model
type=str, required=True, help="the test_file.")
parser.add_argument('--vocab_file', default='../data/chinese_L-12_H-768_A-12/vocab.txt',
type=str, help="the vocab file for bert")
parser.add_argument('--max_length', default=256,
type=int, help="the max length of a sentence")
parser.add_argument('--output_path', default='./rank_output.json',
type=str, required=True, help="the output file path")
args = parser.parse_args()
label_list = ["0", "1"]
device = torch.device('cuda')
tokenizer = BertTokenizer(args.vocab_file)
config = BertConfig(args.config_file)
model = BertForSequenceClassification(config, num_labels=2)
model.load_state_dict(torch.load(args.model_path, map_location=device))
def get_datas(path):
datasets = []
with open(path, 'r', encoding='utf-8') as reader:
for lidx, line in enumerate(tqdm(reader)):
dataset = []
sample = json.loads(line.strip())
question = sample['question']
for i, doc in enumerate(sample['documents']):
q_id = str(sample['question_id'])
for para in doc['paragraphs']:
dataset.append([q_id, question, para])
datasets.append(dataset)
return datasets
def main():
# train_df = pd.read_csv(TRAIN_PATH).sample(frac=1.0, random_state=seed)
# train_size = int(len(train_df) * 0.9)
train_df = pd.read_csv(TRAIN_PATH).sample(train_size + valid_size, random_state=seed)
LOGGER.info(f'data_size is {len(train_df)}')
LOGGER.info(f'train_size is {train_size}')
y = np.where(train_df['target'] >= 0.5, 1, 0)
y_aux = train_df[AUX_COLUMNS].values
identity_columns_new = []
for column in identity_columns + ['target']:
train_df[column + "_bin"] = np.where(train_df[column] >= 0.5, True, False)
if column != "target":
identity_columns_new.append(column + "_bin")
sample_weights = np.ones(len(train_df), dtype=np.float32)
sample_weights += train_df[identity_columns_new].sum(axis=1)
sample_weights += train_df['target_bin'] * (~train_df[identity_columns_new]).sum(axis=1)
sample_weights += (~train_df['target_bin']) * train_df[identity_columns_new].sum(axis=1) * 5
sample_weights /= sample_weights.mean()
with timer('preprocessing text'):
# df["comment_text"] = [analyzer_embed(text) for text in df["comment_text"]]
train_df['comment_text'] = train_df['comment_text'].astype(str)
train_df = train_df.fillna(0)
with timer('load embedding'):
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None, do_lower_case=True)
X_text = convert_lines(train_df["comment_text"].fillna("DUMMY_VALUE"), max_len, tokenizer)
test_df = train_df[train_size:]
with timer('train'):
X_train, y_train, y_aux_train, w_train = X_text[:train_size], y[:train_size], y_aux[
:train_size], sample_weights[
:train_size]
X_val, y_val, y_aux_val, w_val = X_text[train_size:], y[train_size:], y_aux[train_size:], sample_weights[
train_size:]
model = BertForSequenceClassification(bert_config, num_labels=n_labels)
model.load_state_dict(torch.load(model_path))
model.zero_grad()
model = model.to(device)
train_dataset = torch.utils.data.TensorDataset(torch.tensor(X_train, dtype=torch.long),
torch.tensor(y_train, dtype=torch.float))
valid = torch.utils.data.TensorDataset(torch.tensor(X_val, dtype=torch.long),
torch.tensor(y_val, dtype=torch.float))
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size * 2, shuffle=False)
sample_weight_train = [w_train.values, np.ones_like(w_train)]
sample_weight_val = [w_val.values, np.ones_like(w_val)]
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
num_train_optimization_steps = int(epochs * train_size / batch_size / accumulation_steps)
total_step = int(epochs * train_size / batch_size)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=2e-5*gamma,
warmup=0.05,
t_total=num_train_optimization_steps)
model, optimizer = amp.initialize(model, optimizer, opt_level="O1", verbosity=0)
criterion = torch.nn.BCEWithLogitsLoss().to(device)
LOGGER.info(f"Starting 1 epoch...")
tr_loss, train_losses = train_one_epoch(model, train_loader, criterion, optimizer, device,
accumulation_steps, total_step, n_labels)
LOGGER.info(f'Mean train loss: {round(tr_loss,5)}')
torch.save(model.state_dict(), '{}_dic'.format(exp))
valid_loss, oof_pred = validate(model, valid_loader, criterion, device, n_labels)
del model
gc.collect()
torch.cuda.empty_cache()
test_df["pred"] = oof_pred.reshape(-1)
test_df = convert_dataframe_to_bool(test_df)
bias_metrics_df = compute_bias_metrics_for_model(test_df, identity_columns)
LOGGER.info(bias_metrics_df)
score = get_final_metric(bias_metrics_df, calculate_overall_auc(test_df))
LOGGER.info(f'final score is {score}')
test_df.to_csv("oof.csv", index=False)
xs = list(range(1, len(train_losses) + 1))
plt.plot(xs, train_losses, label='Train loss');
plt.legend();
plt.xticks(xs);
plt.xlabel('Iter')
plt.savefig("loss.png")
class TransformersClassifierHandler(BaseHandler, ABC):
"""
Transformers text classifier handler class. This handler takes a text (string) and
as input and returns the classification text based on the serialized transformers checkpoint.
"""
def __init__(self):
super(TransformersClassifierHandler, self).__init__()
self.initialized = False
def initialize(self, ctx):
properties = ctx.system_properties
MODEL_DIR = properties.get("model_dir")
self.device = torch.device("cuda:" +
str(properties.get("gpu_id")) if torch.cuda.
is_available() else "cpu")
self.labelencoder = preprocessing.LabelEncoder()
self.labelencoder.classes_ = np.load(
os.path.join(MODEL_DIR, 'classes.npy'))
config = BertConfig(os.path.join(MODEL_DIR, 'bert_config.json'))
self.model = BertForSequenceClassification(
config, num_labels=len(self.labelencoder.classes_))
self.model.load_state_dict(
torch.load(os.path.join(MODEL_DIR, 'pytorch_model.bin'),
map_location="cpu"))
self.model.to(self.device)
self.model.eval()
self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
self.softmax = torch.nn.Softmax(dim=-1)
# self.batch_size = batch_size
logger.debug(
'Transformer model from path {0} loaded successfully'.format(
MODEL_DIR))
self.manifest = ctx.manifest
self.initialized = True
def preprocess(self, data):
ids = []
segment_ids = []
input_masks = []
MAX_LEN = 128
for sen in data:
text_tokens = self.tokenizer.tokenize(sen)
tokens = ["[CLS]"] + text_tokens + ["[SEP]"]
temp_ids = self.tokenizer.convert_tokens_to_ids(tokens)
input_mask = [1] * len(temp_ids)
segment_id = [0] * len(temp_ids)
padding = [0] * (MAX_LEN - len(temp_ids))
temp_ids += padding
input_mask += padding
segment_id += padding
ids.append(temp_ids)
input_masks.append(input_mask)
segment_ids.append(segment_id)
## Convert input list to Torch Tensors
ids = torch.tensor(ids)
segment_ids = torch.tensor(segment_ids)
input_masks = torch.tensor(input_masks)
validation_data = TensorDataset(ids, input_masks, segment_ids)
validation_sampler = SequentialSampler(validation_data)
validation_dataloader = DataLoader(
validation_data,
sampler=validation_sampler,
batch_size=len(data),
num_workers=self.dataloader_num_workers)
return validation_dataloader
def inference(self, validation_dataloader):
"""
Predict the class of a text using a trained transformer model.
"""
# NOTE: This makes the assumption that your model expects text to be tokenized
# with "input_ids" and "token_type_ids" - which is true for some popular transformer models, e.g. bert.
# If your transformer model expects different tokenization, adapt this code to suit
# its expected input format.
responses = []
for batch in validation_dataloader:
# Add batch to GPU
batch = tuple(t.to(self.device) for t in batch)
# Unpack the inputs from our dataloader
b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad():
# Forward pass, calculate logit predictions
logits = self.model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
for i in range(logits.size(0)):
label_idx = [
self.softmax(
logits[i]).detach().cpu().numpy().argmax()
]
label_str = self.labelencoder.inverse_transform(
label_idx)[0]
responses.append(label_str)
return responses
def postprocess(self, inference_output):
# TODO: Add any needed post-processing of the model predictions here
return inference_output
x_test = test_df["comment_text"].apply(lambda x: content_preprocessing(x))
## Tokenize and padding
BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH)
x_test = convert_lines(x_test,MAX_LEN,tokenizer)
x_test_cuda = torch.tensor(x_test, dtype=torch.long).cuda()
test_data = torch.utils.data.TensorDataset(x_test_cuda)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=False)
## load fine-tuned model
bert_config = BertConfig('../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/bert_config.json')
net = BertForSequenceClassification(bert_config,num_labels=6)
net.load_state_dict(torch.load("../input/bert-model3/bert_pytorch_v3.pt"))
net.cuda()
## inference
net.eval()
result_1 = list()
with torch.no_grad():
for (x_batch,) in test_loader:
y_pred = net(x_batch)
y_pred = torch.sigmoid(y_pred.cpu()).numpy()[:,0]
result_1.extend(y_pred)
result_1 = np.array(result_1)
net = BertForSequenceClassification(bert_config,num_labels=6)
optimizer.step() # Now we can do an optimizer step
optimizer.zero_grad()
if lossf:
lossf = 0.98*lossf+0.02*loss.item()
else:
lossf = loss.item()
tk0.set_postfix(loss = lossf)
avg_loss += loss.item() / len(train_loader)
avg_accuracy += torch.mean(((torch.sigmoid(y_pred[:,0])>0.5) == (y_batch[:,0]>0.5).to(device)).to(torch.float) ).item()/len(train_loader)
tq.set_postfix(avg_loss=avg_loss,avg_accuracy=avg_accuracy)
torch.save(model.state_dict(), output_model_file)
# Run validation
# The following 2 lines are not needed but show how to download the model for prediction
model = BertForSequenceClassification(bert_config,num_labels=len(y_columns))
model.load_state_dict(torch.load(output_model_file ))
model.to(device)
for param in model.parameters():
param.requires_grad=False
model.eval()
valid_preds = np.zeros((len(X_val)))
valid = torch.utils.data.TensorDataset(torch.tensor(X_val,dtype=torch.long))
valid_loader = torch.utils.data.DataLoader(valid, batch_size=32, shuffle=False)
tk0 = tqdm_notebook(valid_loader)
for i,(x_batch,) in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch>0).to(device), labels=None)
valid_preds[i*32:(i+1)*32]=pred[:,0].detach().cpu().squeeze().numpy()
batch_size = 1
# preprocessing
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH,
cache_dir=None,
do_lower_case=True)
test_all = pd.read_csv("data/test.csv")
test_all = test_all.iloc[:10, :]
test_all['comment_text'] = test_all['comment_text'].astype(str)
test_X = convert_lines(test_all["comment_text"].fillna("DUMMY_VALUE"),
MAX_SEQUENCE_LENGTH, tokenizer)
test_all = test_all.fillna(0)
# load model and perform inference
model = BertForSequenceClassification(bert_config, num_labels=1)
model.load_state_dict(torch.load(model_file))
model.to(device)
for param in model.parameters():
param.requires_grad = False
model.eval()
test_preds = np.zeros((len(test_X)))
test = torch.utils.data.TensorDataset(torch.tensor(test_X, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test,
batch_size=batch_size,
shuffle=False)
tk0 = tqdm(test_loader)
for i, (x_batch, ) in enumerate(tk0):
pred = model(x_batch.to(device),
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
bert_config = BertConfig('../input/bert-inference/bert/bert_config.json')
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH,
cache_dir=None,
do_lower_case=True)
test_df = pd.read_csv(
"../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
test_df['comment_text'] = test_df['comment_text'].astype(str)
X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"),
MAX_SEQUENCE_LENGTH, tokenizer)
model = BertForSequenceClassification(bert_config, num_labels=1)
model.load_state_dict(
torch.load("../input/bert-inference/bert/bert_pytorch.bin"))
model.to(device)
for param in model.parameters():
param.requires_grad = False
model.eval()
test_preds = np.zeros((len(X_test)))
test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
tk0 = tqdm(test_loader)
for i, (x_batch, ) in enumerate(tk0):
pred = model(x_batch.to(device),
attention_mask=(x_batch > 0).to(device),
labels=None)
test_preds[i * 32:(i + 1) * 32] = pred[:,
0].detach().cpu().squeeze().numpy()
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path
parser.add_argument("--output_model_path",
default="./models/classifier_model.bin",
type=str,
help="Path of the output model.")
parser.add_argument("--output_lossfig_path",
default="./models/loss.png",
type=str,
help="Path of the output model.")
# Model options.
parser.add_argument("--batch_size",
type=int,
default=32,
help="Batch size.")
parser.add_argument("--seq_length",
type=int,
default=128,
help="Sequence length.")
# Optimizer options.
parser.add_argument("--learning_rate",
type=float,
default=2e-5,
help="Learning rate.")
parser.add_argument("--warmup",
type=float,
default=0.1,
help="Warm up value.")
# Training options.
parser.add_argument("--dropout", type=float, default=0.5, help="Dropout.")
parser.add_argument("--epochs_num",
type=int,
default=5,
help="Number of epochs.")
parser.add_argument("--report_steps",
type=int,
default=100,
help="Specific steps to print prompt.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
parser.add_argument("--device",
type=str,
default='cpu',
help="Device use.")
args = parser.parse_args()
def set_seed(seed=7):
random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
set_seed(args.seed)
# 读取数据
train = pd.read_csv('../data5k/train.tsv', encoding='utf-8', sep='\t')
dev = pd.read_csv('../data5k/dev.tsv', encoding='utf-8', sep='\t')
test = pd.read_csv('../data5k/test.tsv', encoding='utf-8', sep='\t')
# Load bert vocabulary and tokenizer
bert_config = BertConfig('bert_model/bert_config.json')
BERT_MODEL_PATH = 'bert_model'
bert_tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH,
cache_dir=None,
do_lower_case=False)
# 产生输入数据
processor = DataPrecessForSingleSentence(bert_tokenizer=bert_tokenizer)
# train dataset
seqs, seq_masks, seq_segments = processor.get_input(
sentences=train['text_a'].tolist(), max_seq_len=args.seq_length)
labels = train['label'].tolist()
t_seqs = torch.tensor(seqs, dtype=torch.long)
t_seq_masks = torch.tensor(seq_masks, dtype=torch.long)
t_seq_segments = torch.tensor(seq_segments, dtype=torch.long)
t_labels = torch.tensor(labels, dtype=torch.long)
train_data = TensorDataset(t_seqs, t_seq_masks, t_seq_segments, t_labels)
train_sampler = RandomSampler(train_data)
train_dataloder = DataLoader(dataset=train_data,
sampler=train_sampler,
batch_size=args.batch_size)
# dev dataset
seqs, seq_masks, seq_segments = processor.get_input(
sentences=dev['text_a'].tolist(), max_seq_len=args.seq_length)
labels = dev['label'].tolist()
t_seqs = torch.tensor(seqs, dtype=torch.long)
t_seq_masks = torch.tensor(seq_masks, dtype=torch.long)
t_seq_segments = torch.tensor(seq_segments, dtype=torch.long)
t_labels = torch.tensor(labels, dtype=torch.long)
dev_data = TensorDataset(t_seqs, t_seq_masks, t_seq_segments, t_labels)
dev_sampler = RandomSampler(dev_data)
dev_dataloder = DataLoader(dataset=dev_data,
sampler=dev_sampler,
batch_size=args.batch_size)
# test dataset
seqs, seq_masks, seq_segments = processor.get_input(
sentences=test['text_a'].tolist(), max_seq_len=args.seq_length)
labels = test['label'].tolist()
t_seqs = torch.tensor(seqs, dtype=torch.long)
t_seq_masks = torch.tensor(seq_masks, dtype=torch.long)
t_seq_segments = torch.tensor(seq_segments, dtype=torch.long)
t_labels = torch.tensor(labels, dtype=torch.long)
test_data = TensorDataset(t_seqs, t_seq_masks, t_seq_segments, t_labels)
test_sampler = RandomSampler(test_data)
test_dataloder = DataLoader(dataset=test_data,
sampler=test_sampler,
batch_size=args.batch_size)
# build classification model
model = BertForSequenceClassification(bert_config, 2)
# For simplicity, we use DataParallel wrapper to use multiple GPUs.
if args.device == 'cpu':
device = torch.device("cpu")
else:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = nn.DataParallel(model)
model = model.to(device)
# evaluation function
def evaluate(args, is_test, metrics='Acc'):
if is_test:
dataset = test_dataloder
instances_num = test.shape[0]
print("The number of evaluation instances: ", instances_num)
else:
dataset = dev_dataloder
instances_num = dev.shape[0]
print("The number of evaluation instances: ", instances_num)
correct = 0
model.eval()
# Confusion matrix.
confusion = torch.zeros(2, 2, dtype=torch.long)
for i, batch_data in enumerate(dataset):
batch_data = tuple(t.to(device) for t in batch_data)
batch_seqs, batch_seq_masks, batch_seq_segments, batch_labels = batch_data
with torch.no_grad():
logits = model(batch_seqs,
batch_seq_masks,
batch_seq_segments,
labels=None)
pred = logits.softmax(dim=1).argmax(dim=1)
gold = batch_labels
for j in range(pred.size()[0]):
confusion[pred[j], gold[j]] += 1
correct += torch.sum(pred == gold).item()
if is_test:
print("Confusion matrix:")
print(confusion)
print("Report precision, recall, and f1:")
for i in range(confusion.size()[0]):
p = confusion[i, i].item() / confusion[i, :].sum().item()
r = confusion[i, i].item() / confusion[:, i].sum().item()
f1 = 2 * p * r / (p + r)
if i == 1:
label_1_f1 = f1
print("Label {}: {:.3f}, {:.3f}, {:.3f}".format(i, p, r, f1))
print("Acc. (Correct/Total): {:.4f} ({}/{}) ".format(
correct / instances_num, correct, instances_num))
if metrics == 'Acc':
return correct / instances_num
elif metrics == 'f1':
return label_1_f1
else:
return correct / instances_num
# training phase
print("Start training.")
instances_num = train.shape[0]
batch_size = args.batch_size
train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
# 待优化的参数
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup,
t_total=train_steps)
# 存储每一个batch的loss
all_loss = []
all_acc = []
total_loss = 0.0
result = 0.0
best_result = 0.0
for epoch in range(1, args.epochs_num + 1):
model.train()
for step, batch_data in enumerate(train_dataloder):
batch_data = tuple(t.to(device) for t in batch_data)
batch_seqs, batch_seq_masks, batch_seq_segments, batch_labels = batch_data
# 对标签进行onehot编码
one_hot = torch.zeros(batch_labels.size(0), 2).long()
'''one_hot_batch_labels = one_hot.scatter_(
dim=1,
index=torch.unsqueeze(batch_labels, dim=1),
src=torch.ones(batch_labels.size(0), 2).long())
logits = model(
batch_seqs, batch_seq_masks, batch_seq_segments, labels=None)
logits = logits.softmax(dim=1)
loss_function = CrossEntropyLoss()
loss = loss_function(logits, batch_labels)'''
loss = model(batch_seqs, batch_seq_masks, batch_seq_segments,
batch_labels)
loss.backward()
total_loss += loss.item()
if (step + 1) % 100 == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, step + 1, total_loss / 100))
sys.stdout.flush()
total_loss = 0.
#print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".format(epoch, step+1, loss))
optimizer.step()
optimizer.zero_grad()
all_loss.append(total_loss)
total_loss = 0.
print("Start evaluation on dev dataset.")
result = evaluate(args, False)
all_acc.append(result)
if result > best_result:
best_result = result
torch.save(model, open(args.output_model_path, "wb"))
#save_model(model, args.output_model_path)
else:
continue
print("Start evaluation on test dataset.")
evaluate(args, True)
print('all_loss:', all_loss)
print('all_acc:', all_acc)
# Evaluation phase.
print("Final evaluation on the test dataset.")
model.load_state_dict(torch.load(args.output_model_path))
evaluate(args, True)
'''
return pd.DataFrame(records).sort_values('subgroup_auc', ascending=True)
# Run validation
# The following 2 lines are not needed but show how to download the model for prediction
for epoch in tq:
model = BertForSequenceClassification(bert_config,num_labels=len(y_columns))
#paralleism
model = nn.DataParallel(model)
model.load_state_dict(torch.load(output_model_file + '_' + str(epoch) ))
model.to(device)
for param in model.parameters():
param.requires_grad=False
model.eval()
valid_preds = np.zeros((len(X_val)))
valid = torch.utils.data.TensorDataset(torch.tensor(X_val,dtype=torch.long))
valid_loader = torch.utils.data.DataLoader(valid, batch_size=32, shuffle=False)
tk0 = tqdm(valid_loader)
for i,(x_batch,) in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch>0).to(device), labels=None)
valid_preds[i*32:(i+1)*32]=pred[:,0].detach().cpu().squeeze().numpy()
MODEL_NAME = 'model1'
test_df[MODEL_NAME] = torch.sigmoid(torch.tensor(valid_preds)).numpy()
M = torch.tensor(seq_padding(M), dtype=torch.long)
Sg = torch.zeros(*T.size(), dtype=torch.long)
logger.info(f'T:{T.size()}, M:{M.size()}, Sg:{Sg.size()}')
yield S, U, O, M, Sg, T
S, U, O, M, T = [], [], [], [], []
pre_obj_t = ''
eval_data = data_generator(log_data_dic)
kg_model_path = Path(data_dir) / 'kg_intent_model.pt'
config_path = Path(data_dir) / 'kg_intent_config.json'
config = BertConfig(str(config_path))
model = BertForSequenceClassification(config, num_labels=num_class)
model.load_state_dict(
torch.load(kg_model_path,
map_location='cpu' if not torch.cuda.is_available() else None))
# device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
n_gpu = torch.cuda.device_count()
if n_gpu > 1:
logger.info(f"let's use {n_gpu} gpu")
model.to(device)
if n_gpu > 1:
model = nn.DataParallel(model)
maps = json.load((Path(data_dir) / 'maps_for_log.json').open())
y_columns = ['target']
train_df = train_df.drop(['comment_text'], axis=1)
train_df['target'] = (train_df['target'] >= 0.5).astype(float)
valid_df = valid_df.fillna(0)
valid_df = valid_df.drop(['comment_text'], axis=1)
valid_df['target'] = (valid_df['toxic'] == 1) | (valid_df['severe_toxic'] == 1)
valid_df['target'] = valid_df['target'] | (valid_df['obscene'] == 1)
valid_df['target'] = valid_df['target'] | (valid_df['threat'] == 1)
valid_df['target'] = valid_df['target'] | (valid_df['insult'] == 1)
valid_df['target'] = valid_df['target'] | (valid_df['identity_hate'] == 1)
valid_df['target'] = valid_df['target'].astype(float)
model = BertForSequenceClassification(bert_config, num_labels=1)
model.load_state_dict(torch.load("./datas/bert_pytorch.bin"))
model.to(device)
for param in model.parameters():
param.requires_grad = False
X = train_seqs[:]
y = train_df['target'].values[:]
valid_X = valid_seqs[:]
valid_y = valid_df['target'].values[:]
X = np.concatenate((X, valid_X), axis=1)
y = np.concatenate((y, valid_y), axis=0)
train_dataset = torch.utils.data.TensorDataset(
torch.tensor(X, dtype=torch.long), torch.tensor(y, dtype=torch.float))
output_model_file = "./datas/mybert.bin"
class ClassificationModel:
def __init__(self,
task,
val=0.1,
bert_model=BERT_MODEL,
gpu=False,
seed=0):
self.gpu = gpu
self.task = task
self.bert_model = bert_model
self.x_train, self.y_train = load_train_dataset(self.task)
self.x_val = np.random.choice(self.x_train,
size=(int(val * len(self.x_train)), ),
replace=False)
self.y_val = np.random.choice(self.y_train,
size=(int(val * len(self.x_train)), ),
replace=False)
self.x_test_ids, self.x_test = load_test_dataset(self.task)
self.num_classes = len(TASK_LABELS[task])
self.model = None
self.optimizer = None
self.tokenizer = BertTokenizer.from_pretrained(self.bert_model)
self.plt_x = []
self.plt_y = []
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if self.gpu:
torch.cuda.manual_seed_all(seed)
def __init_model(self):
if self.gpu:
self.device = torch.device("cuda")
print("Start learning with GPU")
else:
self.device = torch.device("cpu")
print("Start learning with CPU")
self.model.to(self.device)
print(torch.cuda.memory_allocated(self.device))
def new_model(self):
self.model = BertForSequenceClassification.from_pretrained(
self.bert_model, num_labels=self.num_classes)
self.__init_model()
def load_model(self, path_model, path_config):
self.model = BertForSequenceClassification(BertConfig(path_config),
num_labels=self.num_classes)
self.model.load_state_dict(torch.load(path_model))
self.__init_model()
def save_model(self, path_model, path_config):
torch.save(self.model.state_dict(), path_model)
with open(path_config, 'w') as f:
f.write(self.model.config.to_json_string())
# noinspection PyArgumentList
def train(self,
epochs,
plot_path,
batch_size=32,
lr=5e-5,
model_path=None,
config_path=None):
model_params = list(self.model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in model_params
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params':
[p for n, p in model_params if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
self.optimizer = BertAdam(optimizer_grouped_parameters,
lr=lr,
warmup=0.1,
t_total=int(len(self.x_train) / batch_size) *
epochs)
train_features = convert_examples_to_features(self.x_train,
self.y_train,
MAX_SEQ_LENGTH,
self.tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
_, counts = np.unique(self.y_train, return_counts=True)
class_weights = [sum(counts) / c for c in counts]
example_weights = [class_weights[e] for e in self.y_train]
sampler = WeightedRandomSampler(example_weights, len(self.y_train))
train_dataloader = DataLoader(train_data,
sampler=sampler,
batch_size=batch_size)
self.model.train()
temp_loss = 0
nb_tr_steps = 0
for e in range(epochs):
print("Epoch {e}".format(e=e))
f1, acc = self.val()
print("\nF1 score: {f1}, Accuracy: {acc}".format(f1=f1, acc=acc))
if model_path is not None and config_path is not None:
self.save_model(model_path, config_path)
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(self.device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = self.model(input_ids, segment_ids, input_mask,
label_ids)
loss.backward()
self.plt_y.append(loss.item())
self.plt_x.append(nb_tr_steps)
self.save_plot(plot_path)
nb_tr_steps += 1
self.optimizer.step()
self.optimizer.zero_grad()
if self.gpu:
torch.cuda.empty_cache()
def val(self, batch_size=32, test=False):
eval_features = convert_examples_to_features(self.x_val, self.y_val,
MAX_SEQ_LENGTH,
self.tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=batch_size)
f1, acc = 0, 0
nb_eval_examples = 0
for input_ids, input_mask, segment_ids, gnd_labels in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(self.device)
input_mask = input_mask.to(self.device)
segment_ids = segment_ids.to(self.device)
with torch.no_grad():
logits = self.model(input_ids, segment_ids, input_mask)
predicted_labels = np.argmax(logits.detach().cpu().numpy(), axis=1)
acc += np.sum(predicted_labels == gnd_labels.numpy())
tmp_eval_f1 = f1_score(predicted_labels,
gnd_labels,
average='macro')
f1 += tmp_eval_f1 * input_ids.size(0)
nb_eval_examples += input_ids.size(0)
return f1 / nb_eval_examples, acc / nb_eval_examples
def save_plot(self, path):
import matplotlib.pyplot as plt
fig, ax = plt.subplots()
ax.plot(self.plt_x, self.plt_y)
ax.set(xlabel='Training steps', ylabel='Loss')
fig.savefig(path)
plt.close()
def create_test_predictions(self, path):
eval_features = convert_examples_to_features(self.x_test,
[-1] * len(self.x_test),
MAX_SEQ_LENGTH,
self.tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=16)
predictions = []
inverse_labels = {v: k for k, v in TASK_LABELS[self.task].items()}
for input_ids, input_mask, segment_ids, gnd_labels in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(self.device)
input_mask = input_mask.to(self.device)
segment_ids = segment_ids.to(self.device)
with torch.no_grad():
logits = self.model(input_ids, segment_ids, input_mask)
predictions += [
inverse_labels[p]
for p in list(np.argmax(logits.detach().cpu().numpy(), axis=1))
]
with open(path, "w") as csv_file:
writer = csv.writer(csv_file, delimiter=',')
for i, prediction in enumerate(predictions):
writer.writerow([int(self.x_test_ids[i]), prediction])
return predictions
WORK_DIR = os.path.join(args.pytorch_dump_path,args.model_name)
TOXICITY_COLUMN = 'target'
bert_config = BertConfig(os.path.join(args.tf_checkpoint_path,args.model_name,'bert_config.json'))
### tokenizer
BERT_MODEL_PATH=os.path.join(args.tf_checkpoint_path,args.model_name)
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)
test_df = pd.read_csv(os.path.join(args.data_dir,"test.csv"))
test_df['comment_text'] = test_df['comment_text'].astype(str)
X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), args.max_sequence_length,tokenizer)
model = BertForSequenceClassification(bert_config, num_labels=1)
model.load_state_dict(torch.load(os.path.join(args.weight_path_pytorch,args.model_name+"bert_pytorch.bin")))
model.to(device)
for param in model.parameters():
param.requires_grad = False
model.eval()
test_preds = np.zeros((len(X_test)))
test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
tk0 = tqdm(test_loader)
for i, (x_batch,) in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)
test_preds[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
MAX_SEQUENCE_LENGTH = 220
BATCH_SIZE = 32
BERT_MODEL_PATH = 'E:/bertmodel/Bert Pretrained Models/uncased_l-12_h-768_a-12/'
bert_config = BertConfig('E:/bertmodel/bert_inference/bert_config.json')
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None, do_lower_case=True)
df = pd.read_csv('test.csv')
df['comment_text'] = df['comment_text'].astype(str)
x_test = convert_lines(df['comment_text'].fillna('DUMMY_VALUE'), MAX_SEQUENCE_LENGTH, )
# create the model
model = BertForSequenceClassification(bert_config, num_labels=1)
model.load_state_dict(torch.load("E:/bertmodel/bert_inference/bert_pytorch.bin"))
for param in model.parameters():
param.require_grad = False
model.eval()
test_pred = np.zeros(len(x_test))
test = torch.util.data.TensorDataset(torch.tensor(x_test, dtype=torch.long))
test_loader = torch.util.Data.DataLoader(test, batch_size=32, shuffle=False)
tk = tqdm(test_loader)
for i, (x_batch) in enumerate(tk):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)
test_preds[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
submission_bert = pd.DataFrame.from_dict({
'id': test_input_df['id'],
'prediction': test_pred
})
return float(submission_bert['prediction'].values)
seed_everything()
model = BertForSequenceClassification(bert_config, num_labels=1)
#torch.hub.load_state_dict_from_url("https://www.dropbox.com/s/4320po4qph1lrx6/bert_pytorch.bin?dl=1", model_dir="./",map_location=torch.device('cpu'))
model.load_state_dict(
torch.hub.load_state_dict_from_url(
"https://www.googleapis.com/drive/v3/files/1RYFMsASHW7a92qa7zW296zgnToRQFeb5?alt=media&key=AIzaSyA0OHTKp3e0TvdIyua79c8jH_v6WBmGEKI",
model_dir="input/arti-bert-inference/bert",
map_location=torch.device('cpu')))
#model.load_state_dict()
for param in model.parameters():
param.requires_grad = False
model.eval()
#Initialize the flask App
app = Flask(__name__)
#model = pickle.load(open('model.pkl', 'rb')) #changer par le bin
handler = logging.FileHandler("test.log") # Create the file logger
app.logger.addHandler(handler) # Add it to the built-in logger
app.logger.setLevel(logging.DEBUG)
num_labels=7,
feature_num=50)
validate = True
for fold in [
1,
]:
print('Fold{}:'.format(fold))
validate_idx = kfold[fold][1]
train_idx = kfold[fold][0]
# train_idx = list(range(nrows))[:int(nrows*0.8)]
# validate_idx = list(range(nrows))[int(nrows*0.8):]
model.load_state_dict(
torch.load(os.path.join(models_path, 'bert_fold{}.bin'.format(fold))))
model.cuda()
model.eval()
for param in model.parameters():
param.requires_grad = False
train_pred_fold = []
test_pred_fold = []
train_feature_fold = []
test_feature_fold = []
# on train_set
train_dataset = TensorDataset(torch.tensor(x_train, dtype=torch.long), )
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=False)
avg_accuracy += torch.mean(
((torch.sigmoid(y_pred[:, 0]) > 0.5)
== (y_batch[:, 0] > 0.5).to(device)).to(
torch.float)).item() / len(train_loader)
tq.set_postfix(avg_loss=avg_loss, avg_accuracy=avg_accuracy)
torch.save(model.state_dict(),
output_model_file + '_epoch_' + str(epoch) + '.bin')
#validate
test_model = BertForSequenceClassification(bert_config,
num_labels=len(y_columns))
#paralleism
test_model = nn.DataParallel(test_model)
test_model.load_state_dict(
torch.load(output_model_file + '_epoch_' + str(epoch) + '.bin'))
test_model.to(device)
for param in test_model.parameters():
param.requires_grad = False
test_model.eval()
valid_preds = np.zeros((len(X_val)))
print(valid_preds.size)
valid = torch.utils.data.TensorDataset(
torch.tensor(X_val, dtype=torch.long))
valid_loader = torch.utils.data.DataLoader(valid,
batch_size=256,
shuffle=False)
tk0 = tqdm(valid_loader)
for i, (x_batch, ) in enumerate(tk0):
pred = test_model(x_batch.to(device),
class ClassificationModel:
def __init__(self, bert_model=config.bert_model, gpu=False, seed=0):
self.gpu = gpu
self.bert_model = bert_model
self.train_df = data_reader.load_train_dataset(config.data_path)
self.val_df = data_reader.load_dev_dataset(config.data_path)
self.test_df = data_reader.load_test_dataset(config.data_path)
self.num_classes = len(LABELS)
self.model = None
self.optimizer = None
self.tokenizer = BertTokenizer.from_pretrained(self.bert_model)
# to plot loss during training process
self.plt_x = []
self.plt_y = []
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if self.gpu:
torch.cuda.manual_seed_all(seed)
def __init_model(self):
if self.gpu:
self.device = torch.device("cuda")
else:
self.device = torch.device("cpu")
self.model.to(self.device)
print(torch.cuda.memory_allocated(self.device))
# log available cuda
if self.device.type == 'cuda':
print(torch.cuda.get_device_name(0))
print('Memory Usage:')
print('Allocated:',
round(torch.cuda.memory_allocated(0) / 1024**3, 1), 'GB')
print('Cached: ',
round(torch.cuda.memory_cached(0) / 1024**3, 1), 'GB')
def new_model(self):
self.model = BertForSequenceClassification.from_pretrained(
self.bert_model, num_labels=self.num_classes)
self.__init_model()
def load_model(self, path_model, path_config):
self.model = BertForSequenceClassification(BertConfig(path_config),
num_labels=self.num_classes)
self.model.load_state_dict(torch.load(path_model))
self.__init_model()
def save_model(self, path_model, path_config, epoch_n, acc, f1):
if not os.path.exists(path_model):
os.makedirs(path_model)
model_save_path = os.path.join(
path_model, 'model_{:.4f}_{:.4f}_{:.4f}'.format(epoch_n, acc, f1))
torch.save(self.model.state_dict(), model_save_path)
if not os.path.exists(path_config):
os.makedirs(path_config)
model_config_path = os.path.join(path_config, 'config.cf')
with open(model_config_path, 'w') as f:
f.write(self.model.config.to_json_string())
def train(self,
epochs,
batch_size=config.batch_size,
lr=config.lr,
plot_path=None,
model_path=None,
config_path=None):
model_params = list(self.model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in model_params
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params':
[p for n, p in model_params if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
self.optimizer = BertAdam(
optimizer_grouped_parameters,
lr=lr,
warmup=0.1,
t_total=int(len(self.train_df) / batch_size) * epochs)
nb_tr_steps = 0
train_features = data_reader.convert_examples_to_features(
self.train_df, config.MAX_SEQ_LENGTH, self.tokenizer)
# create tensor of all features
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# eval dataloader
eval_features = data_reader.convert_examples_to_features(
self.val_df, config.MAX_SEQ_LENGTH, self.tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=batch_size)
# class weighting
_, counts = np.unique(self.train_df['label'], return_counts=True)
class_weights = [sum(counts) / c for c in counts]
# assign wight to each input sample
example_weights = [class_weights[e] for e in self.train_df['label']]
sampler = WeightedRandomSampler(example_weights,
len(self.train_df['label']))
train_dataloader = DataLoader(train_data,
sampler=sampler,
batch_size=batch_size)
self.model.train()
for e in range(epochs):
print("Epoch {}".format(e))
if e is not 0:
f1, acc = self.val(eval_dataloader)
print("\nF1 score: {}, Accuracy: {}".format(f1, acc))
if model_path is not None and config_path is not None:
if e is not 0:
self.save_model(model_path, config_path, e, acc, f1)
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(self.device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = self.model(input_ids, segment_ids, input_mask,
label_ids)
loss.backward()
#if plot_path is not None:
# self.plt_y.append(loss.item())
# self.plt_x.append(nb_tr_steps)
# self.save_plot(plot_path)
nb_tr_steps += 1
self.optimizer.step()
self.optimizer.zero_grad()
if self.gpu:
torch.cuda.empty_cache()
def val(self, eval_dataloader, batch_size=config.batch_size):
f1, acc = 0, 0
nb_eval_examples = 0
for input_ids, input_mask, segment_ids, gnd_labels in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(self.device)
input_mask = input_mask.to(self.device)
segment_ids = segment_ids.to(self.device)
with torch.no_grad():
logits = self.model(input_ids, segment_ids, input_mask)
predicted_labels = np.argmax(logits.detach().cpu().numpy(), axis=1)
acc += np.sum(predicted_labels == gnd_labels.numpy())
tmp_eval_f1 = f1_score(predicted_labels,
gnd_labels,
average='macro')
f1 += tmp_eval_f1 * input_ids.size(0)
nb_eval_examples += input_ids.size(0)
return f1 / nb_eval_examples, acc / nb_eval_examples
def save_plot(self, path):
fig, ax = plt.subplots()
ax.plot(self.plt_x, self.plt_y)
ax.set(xlabel='Training steps', ylabel='Loss')
fig.savefig(path)
plt.close()
def create_test_predictions(self, path):
tests_features = data_reader.convert_examples_to_features(
self.test_df, config.MAX_SEQ_LENGTH, self.tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in tests_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in tests_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in tests_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in tests_features],
dtype=torch.long)
all_sample_ids = [f.sample_id for f in tests_features]
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
test_sampler = SequentialSampler(test_data)
test_dataloader = DataLoader(test_data,
sampler=test_sampler,
batch_size=16)
predictions = []
inverse_labels = {v: k for k, v in LABELS}
for input_ids, input_mask, segment_ids, gnd_labels in tqdm(
test_dataloader, desc="Evaluating"):
input_ids = input_ids.to(self.device)
input_mask = input_mask.to(self.device)
segment_ids = segment_ids.to(self.device)
with torch.no_grad():
encoded_layers, logits = self.model(input_ids, segment_ids,
input_mask)
predictions += [
inverse_labels[p]
for p in list(np.argmax(logits.detach().cpu().numpy(), axis=1))
]
with open(path, "w") as csv_file:
writer = csv.writer(csv_file, delimiter=',')
for i, prediction in enumerate(predictions):
writer.writerow([all_sample_ids[i], prediction])
return predictions
def main():
train_df = pd.read_csv(TRAIN_PATH)
train_df['male'] = np.load(
"../input/identity-column-data/male_labeled.npy")
train_df['female'] = np.load(
"../input/identity-column-data/female_labeled.npy")
train_df['homosexual_gay_or_lesbian'] = np.load(
"../input/identity-column-data/homosexual_gay_or_lesbian_labeled.npy")
train_df['christian'] = np.load(
"../input/identity-column-data/christian_labeled.npy")
train_df['jewish'] = np.load(
"../input/identity-column-data/jewish_labeled.npy")
train_df['muslim'] = np.load(
"../input/identity-column-data/muslim_labeled.npy")
train_df['black'] = np.load(
"../input/identity-column-data/black_labeled.npy")
train_df['white'] = np.load(
"../input/identity-column-data/white_labeled.npy")
train_df['psychiatric_or_mental_illness'] = np.load(
"../input/identity-column-data/psychiatric_or_mental_illness_labeled.npy"
)
fold_df = pd.read_csv(FOLD_PATH)
# y = np.where(train_df['target'] >= 0.5, 1, 0)
y = train_df['target'].values
y_aux = train_df[AUX_COLUMNS].values
identity_columns_new = []
for column in identity_columns + ['target']:
train_df[column + "_bin"] = np.where(train_df[column] >= 0.5, True,
False)
if column != "target":
identity_columns_new.append(column + "_bin")
# Overall
weights = np.ones((len(train_df), )) / 4
# Subgroup
weights += (train_df[identity_columns].fillna(0).values >= 0.5).sum(
axis=1).astype(bool).astype(np.int) / 4
# Background Positive, Subgroup Negative
weights += (
((train_df["target"].values >= 0.5).astype(bool).astype(np.int) +
(1 - (train_df[identity_columns].fillna(0).values >= 0.5).sum(
axis=1).astype(bool).astype(np.int))) > 1).astype(bool).astype(
np.int) / 4
# Background Negative, Subgroup Positive
weights += (
((train_df["target"].values < 0.5).astype(bool).astype(np.int) +
(train_df[identity_columns].fillna(0).values >= 0.5).sum(
axis=1).astype(bool).astype(np.int)) > 1).astype(bool).astype(
np.int) / 4
loss_weight = 0.5
with timer('preprocessing text'):
# df["comment_text"] = [analyzer_embed(text) for text in df["comment_text"]]
train_df['comment_text'] = train_df['comment_text'].astype(str)
train_df = train_df.fillna(0)
with timer('load embedding'):
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH,
cache_dir=None,
do_lower_case=True)
X_text = convert_lines_head_tail(
train_df["comment_text"].fillna("DUMMY_VALUE"), max_len, head_len,
tokenizer)
del tokenizer
gc.collect()
LOGGER.info(f"X_text {X_text.shape}")
with timer('train'):
train_index = fold_df.fold_id != fold_id
valid_index = fold_df.fold_id == fold_id
X_train, y_train, y_aux_train, w_train = X_text[train_index].astype(
"int32"), y[train_index], y_aux[train_index], weights[train_index]
X_val, y_val, y_aux_val, w_val = X_text[valid_index].astype("int32"), y[valid_index], y_aux[valid_index], \
weights[
valid_index]
test_df = train_df[valid_index]
del X_text, y, y_aux, weights, train_index, valid_index, train_df
gc.collect()
model = BertForSequenceClassification(bert_config, num_labels=n_labels)
model.load_state_dict(torch.load(model_path))
model.zero_grad()
model = model.to(device)
y_train = np.concatenate(
(y_train.reshape(-1, 1), w_train.reshape(-1, 1), y_aux_train),
axis=1).astype("float32")
y_val = np.concatenate(
(y_val.reshape(-1, 1), w_val.reshape(-1, 1), y_aux_val),
axis=1).astype("float32")
train_dataset = torch.utils.data.TensorDataset(
torch.tensor(X_train, dtype=torch.long),
torch.tensor(y_train, dtype=torch.float32))
valid = torch.utils.data.TensorDataset(
torch.tensor(X_val, dtype=torch.long),
torch.tensor(y_val, dtype=torch.float32))
ran_sampler = torch.utils.data.RandomSampler(train_dataset)
len_sampler = LenMatchBatchSampler(ran_sampler,
batch_size=batch_size,
drop_last=False)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_sampler=len_sampler)
valid_loader = torch.utils.data.DataLoader(valid,
batch_size=batch_size * 2,
shuffle=False)
LOGGER.info(f"done data loader setup")
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
num_train_optimization_steps = int(epochs * len(X_train) / batch_size /
accumulation_steps)
total_step = int(epochs * len(X_train) / batch_size)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=base_lr,
warmup=0.005,
t_total=num_train_optimization_steps)
LOGGER.info(f"done optimizer loader setup")
model, optimizer = amp.initialize(model,
optimizer,
opt_level="O1",
verbosity=0)
# criterion = torch.nn.BCEWithLogitsLoss().to(device)
criterion = CustomLoss(loss_weight).to(device)
LOGGER.info(f"done amp setup")
for epoch in range(1, epochs + 1):
LOGGER.info(f"Starting {epoch} epoch...")
LOGGER.info(f"length {len(X_train)} train {len(X_val)} train...")
if epoch == 1:
for param_group in optimizer.param_groups:
param_group['lr'] = base_lr * gammas[1]
tr_loss, train_losses = train_one_epoch(model,
train_loader,
criterion,
optimizer,
device,
accumulation_steps,
total_step,
n_labels,
base_lr,
gamma=gammas[2 * epoch])
LOGGER.info(f'Mean train loss: {round(tr_loss,5)}')
torch.save(model.state_dict(),
'{}_epoch{}_fold{}.pth'.format(exp, epoch, fold_id))
valid_loss, oof_pred = validate(model, valid_loader, criterion,
device, n_labels)
LOGGER.info(f'Mean valid loss: {round(valid_loss,5)}')
if epochs > 1:
test_df_cp = test_df.copy()
test_df_cp["pred"] = oof_pred[:, 0]
test_df_cp = convert_dataframe_to_bool(test_df_cp)
bias_metrics_df = compute_bias_metrics_for_model(
test_df_cp, identity_columns)
LOGGER.info(bias_metrics_df)
score = get_final_metric(bias_metrics_df,
calculate_overall_auc(test_df_cp))
LOGGER.info(f'score is {score}')
del model
gc.collect()
torch.cuda.empty_cache()
test_df["pred"] = oof_pred[:, 0]
test_df = convert_dataframe_to_bool(test_df)
bias_metrics_df = compute_bias_metrics_for_model(test_df, identity_columns)
LOGGER.info(bias_metrics_df)
score = get_final_metric(bias_metrics_df, calculate_overall_auc(test_df))
LOGGER.info(f'final score is {score}')
test_df.to_csv("oof.csv", index=False)
xs = list(range(1, len(train_losses) + 1))
plt.plot(xs, train_losses, label='Train loss')
plt.legend()
plt.xticks(xs)
plt.xlabel('Iter')
plt.savefig("loss.png")
def train_unfixed():
# 配置文件
cf = Config('./config.yaml')
# 有GPU用GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# 训练数据
train_data = NewsDataset("./data/cnews_final_train.txt", cf.max_seq_len)
train_dataloader = DataLoader(train_data,
batch_size=cf.batch_size,
shuffle=True)
# 测试数据
test_data = NewsDataset("./data/cnews_final_test.txt", cf.max_seq_len)
test_dataloader = DataLoader(test_data,
batch_size=cf.batch_size,
shuffle=True)
# 模型
config = BertConfig("./output/pytorch_bert_config.json")
model = BertForSequenceClassification(config, num_labels=cf.num_labels)
model.load_state_dict(torch.load("./output/pytorch_model.bin"))
# 优化器用adam
for param in model.parameters():
param.requires_grad = True
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
num_train_optimization_steps = int(
len(train_data) / cf.batch_size) * cf.epoch
optimizer = BertAdam(optimizer_grouped_parameters,
lr=cf.lr,
t_total=num_train_optimization_steps)
# 把模型放到指定设备
model.to(device)
# 让模型并行化运算
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
# 训练
start_time = time.time()
total_batch = 0 # 总批次
best_acc_val = 0.0 # 最佳验证集准确率
last_improved = 0 # 记录上一次提升批次
require_improvement = 1500 # 如果超过1500轮未提升，提前结束训练
# 获取当前验证集acc
model.eval()
_, best_acc_val = evaluate(model, test_dataloader, device)
flag = False
model.train()
for epoch_id in range(cf.epoch):
print("Epoch %d" % epoch_id)
for step, batch in enumerate(
tqdm(train_dataloader,
desc="batch",
total=len(train_dataloader))):
# for step,batch in enumerate(train_dataloader):
label_id = batch['label_id'].squeeze(1).to(device)
word_ids = batch['word_ids'].to(device)
segment_ids = batch['segment_ids'].to(device)
word_mask = batch['word_mask'].to(device)
loss = model(word_ids, segment_ids, word_mask, label_id)
loss.backward()
optimizer.step()
optimizer.zero_grad()
total_batch += 1
if total_batch % cf.print_per_batch == 0:
model.eval()
with torch.no_grad():
loss_train, acc_train = get_model_loss_acc(
model, word_ids, segment_ids, word_mask, label_id)
loss_val, acc_val = evaluate(model, test_dataloader, device)
if acc_val > best_acc_val:
# 保存最好结果
best_acc_val = acc_val
last_improved = total_batch
torch.save(model.state_dict(),
"./output/pytorch_model.bin")
with open("./output/pytorch_bert_config.json", 'w') as f:
f.write(model.config.to_json_string())
improved_str = "*"
else:
improved_str = ""
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>6.2}, Train Acc: {2:>7.2%},' \
+ ' Val Loss: {3:>6.2}, Val Acc: {4:>7.2%}, Time: {5} {6}'
print(
msg.format(total_batch, loss_train, acc_train, loss_val,
acc_val, time_dif, improved_str))
model.train()
if total_batch - last_improved > require_improvement:
print("长时间未优化")
flag = True
break
if flag:
break
