def parse(parser):
parser.add_argument('--conf', '-c', help='path to config file')
parser.add_argument('--path', '-p', help='path to model file')
parser.add_argument('--device', '-d', default='-1', help='ID of GPU to use')
parser.add_argument('--seed', '-s', default=1, type=int, help='seed for generating random numbers')
parser.add_argument('--threads', '-t', default=16, type=int, help='max num of threads')
parser.add_argument('--batch-size', default=5000, type=int, help='batch size')
parser.add_argument("--local_rank", type=int, default=-1, help='node rank for distributed training')
args, unknown = parser.parse_known_args()
args, _ = parser.parse_known_args(unknown, args)
args = Config(**vars(args))
Parser = args.pop('Parser')
torch.set_num_threads(args.threads)
torch.manual_seed(args.seed)
init_device(args.device, args.local_rank)
init_logger(logger, f"{args.path}.{args.mode}.log")
logger.info('\n' + str(args))
if args.mode == 'train':
parser = Parser.build(**args)
parser.train(**args)
elif args.mode == 'evaluate':
parser = Parser.load(args.path)
print(args)
parser.evaluate(**args)
elif args.mode == 'predict':
parser = Parser.load(args.path)
parser.predict(**args)
def predict(self, data, pred=None, buckets=8, batch_size=5000, prob=False, **kwargs):
args = self.args.update(locals())
init_logger(logger, verbose=args.verbose)
self.transform.eval()
if args.prob:
self.transform.append(Field('probs'))
logger.info("Loading the data")
dataset = Dataset(self.transform, data)
dataset.build(args.batch_size, args.buckets)
logger.info(f"\n{dataset}")
logger.info("Making predictions on the dataset")
start = datetime.now()
preds = self._predict(dataset.loader)
elapsed = datetime.now() - start
for name, value in preds.items():
setattr(dataset, name, value)
if pred is not None and is_master():
logger.info(f"Saving predicted results to {pred}")
self.transform.save(pred, dataset.sentences)
logger.info(f"{elapsed}s elapsed, {len(dataset) / elapsed.total_seconds():.2f} Sents/s")
return dataset
def parse(parser):
parser.add_argument('--conf', '-c', help='path to config file')
parser.add_argument('--path', '-p', help='path to model file')
parser.add_argument('--device',
'-d',
default='-1',
help='ID of GPU to use')
parser.add_argument('--seed',
'-s',
default=1,
type=int,
help='seed for generating random numbers')
parser.add_argument('--threads',
'-t',
default=16,
type=int,
help='max num of threads')
parser.add_argument('--batch-size',
default=5000,
type=int,
help='batch size')
parser.add_argument("--local_rank",
type=int,
default=-1,
help='node rank for distributed training')
parser.add_argument(
'--min_freq',
default=2,
type=int,
help='minimum frequency needed to include a token in the vocabulary')
args, unknown = parser.parse_known_args()
args, _ = parser.parse_known_args(unknown, args)
args = Config(**vars(args))
Parser = args.pop('Parser')
torch.set_num_threads(args.threads)
torch.manual_seed(args.seed)
init_device(args.device, args.local_rank)
init_logger(logger, f"{args.path}.{args.mode}.log")
logger.info('\n' + str(args))
if args.mode == 'train':
# min_freq看看是多少
parser = Parser.build(**args)
args.update({'mu': .0, 'nu': 0.95, 'lr': 1e-3, 'weight_decay': 3e-9})
parser.train(**args)
elif args.mode == 'evaluate':
parser = Parser.load(args.path)
parser.evaluate(**args)
elif args.mode == 'predict':
parser = Parser.load(args.path)
parser.predict(**args)
def evaluate(self, data, buckets=8, batch_size=5000, **kwargs):
args = self.args.update(locals())
init_logger(logger, verbose=args.verbose)
self.transform.train()
logger.info("Load the data")
dataset = Dataset(self.transform, data)
dataset.build(args.batch_size, args.buckets)
logger.info(f"\n{dataset}")
logger.info("Evaluate the dataset")
start = datetime.now()
loss, metric = self._evaluate(dataset.loader)
elapsed = datetime.now() - start
logger.info(f"loss: {loss:.4f} - {metric}")
tag_map = {k: self.CPOS.vocab[v] for k, v in metric.tag_map.items()}
pprint(tag_map)
recalled_tags = Counter(tag_map.values())
unrecalled_tags = set(self.CPOS.vocab.stoi) - set(recalled_tags.keys())
pprint(recalled_tags)
pprint(unrecalled_tags)
gold_tag_map = {
self.CPOS.vocab[k]: v
for k, v in metric.gold_tag_map.items()
}
pprint(gold_tag_map)
unrecalled_tag_map = {
g: tag_map[gold_tag_map[g]]
for g in self.CPOS.vocab.stoi
}
unrecalled_tag_map = {
k: v
for k, v in unrecalled_tag_map.items() if k != v
}
pprint(unrecalled_tag_map)
# heatmap(metric.clusters.cpu(), list(self.CPOS.vocab.stoi.keys()), f"{args.path}.evaluate.clusters")
heatmap(
self.model.T.softmax(-1).detach().cpu(),
[f"#C{n}#" for n in range(len(self.CPOS.vocab))],
f"{args.path}.T.clusters")
logger.info(
f"{elapsed}s elapsed, {len(dataset)/elapsed.total_seconds():.2f} Sents/s"
)
return loss, metric
def evaluate(self, data, buckets=8, batch_size=5000, **kwargs):
args = self.args.update(locals())
init_logger(logger, verbose=args.verbose)
self.transform.train()
logger.info("Loading the data")
dataset = Dataset(self.transform, data)
dataset.build(args.batch_size, args.buckets)
logger.info(f"\n{dataset}")
logger.info("Evaluating the dataset")
start = datetime.now()
loss, metric = self._evaluate(dataset.loader)
elapsed = datetime.now() - start
logger.info(f"loss: {loss:.4f} - {metric}")
logger.info(f"{elapsed}s elapsed, {len(dataset)/elapsed.total_seconds():.2f} Sents/s")
return loss, metric
def train(self, train, dev, test,
buckets=32,
batch_size=5000,
lr=2e-3,
mu=.9,
nu=.9,
epsilon=1e-12,
clip=5.0,
decay=.75,
decay_steps=5000,
epochs=5000,
patience=100,
weight_decay=0,
verbose=True,
**kwargs):
args = self.args.update(locals())
init_logger(logger, verbose=args.verbose)
self.transform.train()
if dist.is_initialized():
args.batch_size = args.batch_size // dist.get_world_size()
logger.info("Loading the data")
train = Dataset(self.transform, args.train, **args)
dev = Dataset(self.transform, args.dev)
test = Dataset(self.transform, args.test)
train.build(args.batch_size, args.buckets, True, dist.is_initialized())
dev.build(args.batch_size, args.buckets)
test.build(args.batch_size, args.buckets)
logger.info(f"\n{'train:':6} {train}\n{'dev:':6} {dev}\n{'test:':6} {test}\n")
logger.info(f"{self.model}\n")
if dist.is_initialized():
self.model = DDP(self.model, device_ids=[args.local_rank], find_unused_parameters=True)
self.optimizer = Adam(self.model.parameters(), args.lr, (args.mu, args.nu), args.epsilon, weight_decay=args.weight_decay)
self.scheduler = ExponentialLR(self.optimizer, args.decay**(1/args.decay_steps))
elapsed = timedelta()
best_e, best_metric = 1, Metric()
for epoch in range(1, args.epochs + 1):
start = datetime.now()
logger.info(f"Epoch {epoch} / {args.epochs}:")
self._train(train.loader)
loss, dev_metric = self._evaluate(dev.loader)
logger.info(f"{'dev:':6} - loss: {loss:.4f} - {dev_metric}")
loss, test_metric = self._evaluate(test.loader)
logger.info(f"{'test:':6} - loss: {loss:.4f} - {test_metric}")
t = datetime.now() - start
# save the model if it is the best so far
if dev_metric > best_metric:
best_e, best_metric = epoch, dev_metric
if is_master():
self.save(args.path)
logger.info(f"{t}s elapsed (saved)\n")
else:
logger.info(f"{t}s elapsed\n")
elapsed += t
if epoch - best_e >= args.patience:
break
loss, metric = self.load(**args)._evaluate(test.loader)
logger.info(f"Epoch {best_e} saved")
logger.info(f"{'dev:':6} - {best_metric}")
logger.info(f"{'test:':6} - {metric}")
logger.info(f"{elapsed}s elapsed, {elapsed / epoch}s/epoch")
def train(self,
train,
dev,
test,
buckets=32,
batch_size=5000,
update_steps=1,
clip=5.0,
epochs=5000,
patience=100,
**kwargs):
args = self.args.update(locals())
init_logger(logger, verbose=args.verbose)
self.transform.train()
if dist.is_initialized():
args.batch_size = args.batch_size // dist.get_world_size()
logger.info("Loading the data")
train = Dataset(self.transform, args.train, **args)
dev = Dataset(self.transform, args.dev)
test = Dataset(self.transform, args.test)
train.build(args.batch_size // args.update_steps, args.buckets, True,
dist.is_initialized())
dev.build(args.batch_size, args.buckets)
test.build(args.batch_size, args.buckets)
logger.info(
f"\n{'train:':6} {train}\n{'dev:':6} {dev}\n{'test:':6} {test}\n")
if args.encoder == 'lstm':
self.optimizer = Adam(self.model.parameters(), args.lr,
(args.mu, args.nu), args.eps,
args.weight_decay)
self.scheduler = ExponentialLR(self.optimizer,
args.decay**(1 / args.decay_steps))
else:
from transformers import AdamW, get_linear_schedule_with_warmup
steps = len(train.loader) * epochs // args.update_steps
self.optimizer = AdamW(
[{
'params': c.parameters(),
'lr': args.lr * (1 if n == 'encoder' else args.lr_rate)
} for n, c in self.model.named_children()], args.lr)
self.scheduler = get_linear_schedule_with_warmup(
self.optimizer, int(steps * args.warmup), steps)
if dist.is_initialized():
self.model = DDP(self.model,
device_ids=[args.local_rank],
find_unused_parameters=True)
elapsed = timedelta()
best_e, best_metric = 1, Metric()
for epoch in range(1, args.epochs + 1):
start = datetime.now()
logger.info(f"Epoch {epoch} / {args.epochs}:")
self._train(train.loader)
loss, dev_metric = self._evaluate(dev.loader)
logger.info(f"{'dev:':5} loss: {loss:.4f} - {dev_metric}")
loss, test_metric = self._evaluate(test.loader)
logger.info(f"{'test:':5} loss: {loss:.4f} - {test_metric}")
t = datetime.now() - start
if dev_metric > best_metric:
best_e, best_metric = epoch, dev_metric
if is_master():
self.save(args.path)
logger.info(f"{t}s elapsed (saved)\n")
else:
logger.info(f"{t}s elapsed\n")
elapsed += t
if epoch - best_e >= args.patience:
break
loss, metric = self.load(**args)._evaluate(test.loader)
logger.info(f"Epoch {best_e} saved")
logger.info(f"{'dev:':5} {best_metric}")
logger.info(f"{'test:':5} {metric}")
logger.info(f"{elapsed}s elapsed, {elapsed / epoch}s/epoch")
def train(self,
train,
dev,
test,
buckets=32,
batch_size=5000,
clip=5.0,
epochs=5000,
patience=100,
**kwargs):
args = self.args.update(locals())
init_logger(logger, verbose=args.verbose)
self.transform.train()
if dist.is_initialized():
args.batch_size = args.batch_size // dist.get_world_size()
logger.info("Loading the data")
train = Dataset(self.transform, args.train, **args)
dev = Dataset(self.transform, args.dev)
test = Dataset(self.transform, args.test)
train.build(args.batch_size, args.buckets, True, dist.is_initialized())
dev.build(args.batch_size, args.buckets)
test.build(args.batch_size, args.buckets)
logger.info(
f"\n{'train:':6} {train}\n{'dev:':6} {dev}\n{'test:':6} {test}\n")
if dist.is_initialized():
self.model = DDP(self.model,
device_ids=[args.local_rank],
find_unused_parameters=True)
elapsed = timedelta()
best_e, best_metric = 1, Metric()
for epoch in range(1, args.epochs + 1):
start = datetime.now()
logger.info(f"Epoch {epoch} / {args.epochs}:")
#if epoch < 2:
# self._train(train.loader)
#else:
#print('Using margin loss')
self._train(train.loader, loss_type='margin')
loss, dev_metric = self._evaluate(dev.loader)
logger.info(f"{'dev:':5} loss: {loss:.4f} - {dev_metric}")
loss, test_metric = self._evaluate(test.loader)
logger.info(f"{'test:':5} loss: {loss:.4f} - {test_metric}")
t = datetime.now() - start
# save the model if it is the best so far
if dev_metric > best_metric:
best_e, best_metric = epoch, dev_metric
if is_master():
self.save(args.path)
logger.info(f"{t}s elapsed (saved)\n")
else:
logger.info(f"{t}s elapsed\n")
elapsed += t
if epoch - best_e >= args.patience:
break
loss, metric = self.load(**args)._evaluate(test.loader)
logger.info(f"Epoch {best_e} saved")
logger.info(f"{'dev:':5} {best_metric}")
logger.info(f"{'test:':5} {metric}")
logger.info(f"{elapsed}s elapsed, {elapsed / epoch}s/epoch")
def train(self,
train,
dev,
test,
buckets=32,
batch_size=5000,
update_steps=1,
clip=5.0,
epochs=5000,
patience=100,
**kwargs):
args = self.args.update(locals())
init_logger(logger, verbose=args.verbose)
self.transform.train()
batch_size = batch_size // update_steps
if dist.is_initialized():
batch_size = batch_size // dist.get_world_size()
logger.info("Loading the data")
train = Dataset(self.transform, args.train,
**args).build(batch_size, buckets, True,
dist.is_initialized())
dev = Dataset(self.transform, args.dev).build(batch_size, buckets)
test = Dataset(self.transform, args.test).build(batch_size, buckets)
logger.info(
f"\n{'train:':6} {train}\n{'dev:':6} {dev}\n{'test:':6} {test}\n")
if args.encoder == 'lstm':
self.optimizer = Adam(self.model.parameters(), args.lr,
(args.mu, args.nu), args.eps,
args.weight_decay)
self.scheduler = ExponentialLR(self.optimizer,
args.decay**(1 / args.decay_steps))
else:
from transformers import AdamW, get_linear_schedule_with_warmup
steps = len(train.loader) * epochs // args.update_steps
self.optimizer = AdamW([{
'params':
p,
'lr':
args.lr * (1 if n.startswith('encoder') else args.lr_rate)
} for n, p in self.model.named_parameters()], args.lr)
self.scheduler = get_linear_schedule_with_warmup(
self.optimizer, int(steps * args.warmup), steps)
if dist.is_initialized():
self.model = DDP(self.model,
device_ids=[args.local_rank],
find_unused_parameters=True)
self.epoch, self.best_e, self.patience, self.best_metric, self.elapsed = 1, 1, patience, Metric(
), timedelta()
if self.args.checkpoint:
self.optimizer.load_state_dict(
self.checkpoint_state_dict.pop('optimizer_state_dict'))
self.scheduler.load_state_dict(
self.checkpoint_state_dict.pop('scheduler_state_dict'))
set_rng_state(self.checkpoint_state_dict.pop('rng_state'))
for k, v in self.checkpoint_state_dict.items():
setattr(self, k, v)
train.loader.batch_sampler.epoch = self.epoch
for epoch in range(self.epoch, args.epochs + 1):
start = datetime.now()
logger.info(f"Epoch {epoch} / {args.epochs}:")
self._train(train.loader)
loss, dev_metric = self._evaluate(dev.loader)
logger.info(f"{'dev:':5} loss: {loss:.4f} - {dev_metric}")
loss, test_metric = self._evaluate(test.loader)
logger.info(f"{'test:':5} loss: {loss:.4f} - {test_metric}")
t = datetime.now() - start
self.epoch += 1
self.patience -= 1
self.elapsed += t
if dev_metric > self.best_metric:
self.best_e, self.patience, self.best_metric = epoch, patience, dev_metric
if is_master():
self.save_checkpoint(args.path)
logger.info(f"{t}s elapsed (saved)\n")
else:
logger.info(f"{t}s elapsed\n")
if self.patience < 1:
break
parser = self.load(**args)
loss, metric = parser._evaluate(test.loader)
parser.save(args.path)
logger.info(f"Epoch {self.best_e} saved")
logger.info(f"{'dev:':5} {self.best_metric}")
logger.info(f"{'test:':5} {metric}")
logger.info(f"{self.elapsed}s elapsed, {self.elapsed / epoch}s/epoch")
