no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
logplot_file = os.path.join(log_folder,
"summary_loop_%s.log" % (args.experiment))
logplot = LogPlot(logplot_file)
optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
time_save = time.time()
time_ckpt = time.time()
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
summarizer.model, optimizer = amp.initialize(
summarizer.model, optimizer, opt_level="O1"
) # For now O1. See details at https://nvidia.github.io/apex/amp.html
dataloader = DataLoader(dataset=dataset, batch_size=args.train_batch_size, sampler=RandomSampler(dataset), drop_last=True, collate_fn=collate_func)
kw_cov = KeywordCoverage(args.device, keyword_model_file=os.path.join(models_folder, "keyword_extractor.joblib"), n_kws=args.n_kws) # , model_file=os.path.join(models_folder, "news_bert_bs64.bin")
kw_cov.model.train()
print("Loaded model")
param_optimizer = list(kw_cov.model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=2e-5)
scheduler = WarmupLinearSchedule(optimizer, warmup_steps=0, t_total=len(dataloader))
logplot = LogPlot("/home/phillab/logs/coverage/bert_coverage_"+args.experiment+".log")
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
kw_cov.model, optimizer = amp.initialize(kw_cov.model, optimizer, opt_level="O1") # For now O1. See details at https://nvidia.github.io/apex/amp.html
time_save = time.time()
optim_every = 4
for ib, batch in enumerate(dataloader):
contents, summaries = batch
loss, acc = kw_cov.train_batch(contents, summaries)
if args.fp16:
# kw_cov = KeywordCoverage(args.device, keyword_model_file=os.path.join(models_folder, "keyword_extractor.joblib"), n_kws=args.n_kws) # , model_file=os.path.join(models_folder, "news_bert_bs64.bin")
#kw_cov = KeywordCoverage(args.device, n_kws=args.n_kws) # , model_file=os.path.join(models_folder, "news_bert_bs64.bin")
kw_cov = KeywordCoverage(args.device, n_kws=args.n_kws , model_file=os.path.join(models_folder, "news_bert_train.bin") )
kw_cov.model.train()
print("Loaded model")
param_optimizer = list(kw_cov.model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=2e-5)
logplot = LogPlot(os.path.join(logs_folder, "coverage/bert_coverage_%s.log" % (args.experiment)))
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
kw_cov.model, optimizer = amp.initialize(kw_cov.model, optimizer, opt_level="O1") # For now O1. See details at https://nvidia.github.io/apex/amp.html
time_save = time.time()
optim_every = 4
for ib, batch in enumerate(dataloader):
contents, summaries = batch
loss, acc = kw_cov.train_batch(contents, summaries)
if args.fp16:
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = "" + str(args.gpu_nb)
learning_rate = 2e-5
n_epochs = 3
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
tokenizer.max_len = 10000
model = BertForPreTraining.from_pretrained("bert-base-uncased")
model.to(args.device)
print("Model loaded")
vocab_size = tokenizer.vocab_size
summ = LogPlot("/home/phillab/logs/bert-base-uncased/bert_news.log")
def random_word(tokens, tokenizer):
output_label = []
for i, token in enumerate(tokens):
prob = random.random()
# mask token with 15% probability
if prob < 0.15:
prob /= 0.15
# 80% randomly change token to mask token
if prob < 0.8:
tokens[i] = "[MASK]"
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = "" + str(args.gpu_nb)
learning_rate = 2e-5
n_epochs = 3
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
tokenizer.max_len = 10000
model = BertForPreTraining.from_pretrained("bert-base-uncased")
model.to(args.device)
print("Model loaded")
vocab_size = tokenizer.vocab_size
summ = LogPlot(
"/home/robin/TrySomethingNew/summary_loop_by_me/logs/bert_news.log")
def random_word(tokens, tokenizer):
output_label = []
for i, token in enumerate(tokens):
prob = random.random()
# mask token with 15% probability
if prob < 0.15:
prob /= 0.15
# 80% randomly change token to mask token
if prob < 0.8:
tokens[i] = "[MASK]"
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
logplot_file = os.path.join(logs_folder,
"generator_" + args.experiment + ".log")
summ = LogPlot(logplot_file)
optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
scheduler = WarmupLinearSchedule(optimizer,
warmup_steps=0,
t_total=n_epochs * len(dl_train))
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model.model, optimizer = amp.initialize(
model.model, optimizer, opt_level="O1"