def train_and_evaluate(config):
token_makers = create_by_factory(TokenMakersFactory, config.token)
tokenizers = token_makers["tokenizers"]
del token_makers["tokenizers"]
config.data_reader.tokenizers = tokenizers
if nsml.IS_ON_NSML:
config.data_reader.train_file_path = os.path.join(
DATASET_PATH, "train", "train_data",
config.data_reader.train_file_path)
config.data_reader.valid_file_path = os.path.join(
DATASET_PATH, "train", "train_data",
config.data_reader.valid_file_path)
data_reader = create_by_factory(DataReaderFactory, config.data_reader)
datas, helpers = data_reader.read()
# Vocab & Indexing
text_handler = TextHandler(token_makers, lazy_indexing=True)
texts = data_reader.filter_texts(datas)
token_counters = text_handler.make_token_counters(texts)
text_handler.build_vocabs(token_counters)
text_handler.index(datas, data_reader.text_columns)
# Iterator
datasets = data_reader.convert_to_dataset(datas, helpers=helpers)
train_loader = create_data_loader(datasets["train"],
batch_size=config.iterator.batch_size,
shuffle=True,
cuda_device_id=device)
valid_loader = create_data_loader(datasets["valid"],
batch_size=config.iterator.batch_size,
shuffle=False,
cuda_device_id=device)
# Model & Optimizer
model = create_model(token_makers,
ModelFactory,
config.model,
device,
helpers=helpers)
model_parameters = [
param for param in model.parameters() if param.requires_grad
]
optimizer = get_optimizer_by_name("adam")(model_parameters)
if IS_ON_NSML:
bind_nsml(model, optimizer=optimizer)
# Trainer
trainer_config = vars(config.trainer)
trainer_config["model"] = model
trainer = Trainer(**trainer_config)
trainer.train_and_evaluate(train_loader, valid_loader, optimizer)
def set_train_mode(self):
"""
Training Mode
- Pipeline
1. read raw_data (DataReader)
2. build vocabs (DataReader, Token)
3. indexing tokens (DataReader, Token)
4. convert to DataSet (DataReader)
5. create DataLoader (DataLoader)
6. define model and optimizer
7. run!
"""
logger.info("Config. \n" + pretty_json_dumps(self.config_dict) + "\n")
data_reader, token_makers = self._create_data_and_token_makers()
datas, helpers = data_reader.read()
# Token & Vocab
text_handler = TextHandler(token_makers, lazy_indexing=True)
texts = data_reader.filter_texts(datas)
token_counters = text_handler.make_token_counters(texts, config=self.config)
text_handler.build_vocabs(token_counters)
text_handler.index(datas, data_reader.text_columns)
# iterator
datasets = data_reader.convert_to_dataset(datas, helpers=helpers) # with name
self.config.iterator.cuda_devices = self.config.cuda_devices
train_loader, valid_loader, test_loader = self._create_by_factory(
DataLoaderFactory, self.config.iterator, param={"datasets": datasets}
)
checkpoint_dir = Path(self.config.trainer.log_dir) / "checkpoint"
checkpoints = None
if checkpoint_dir.exists():
checkpoints = self._load_exist_checkpoints(checkpoint_dir) # contain model and optimizer
if checkpoints is None:
model = self._create_model(token_makers, helpers=helpers)
op_dict = self._create_by_factory(
OptimizerFactory, self.config.optimizer, param={"model": model}
)
else:
model = self._create_model(token_makers, checkpoint=checkpoints)
op_dict = self._create_by_factory(
OptimizerFactory, self.config.optimizer, param={"model": model}
)
utils.load_optimizer_checkpoint(op_dict["optimizer"], checkpoints)
self.set_trainer(model, op_dict=op_dict)
return train_loader, valid_loader, op_dict["optimizer"]
def re_train_and_evaluate(config):
NSML_SESSEION = 'team_6/19_tcls_qa/258' # NOTE: need to hard code
NSML_CHECKPOINT = '1' # NOTE: nghhhhed to hard code
assert NSML_CHECKPOINT is not None, "You must insert NSML Session's checkpoint for submit"
assert NSML_SESSEION is not None, "You must insert NSML Session's name for submit"
token_makers = create_by_factory(TokenMakersFactory, config.token)
tokenizers = token_makers["tokenizers"]
del token_makers["tokenizers"]
config.data_reader.tokenizers = tokenizers
if nsml.IS_ON_NSML:
config.data_reader.train_file_path = os.path.join(
DATASET_PATH, "train", "train_data",
config.data_reader.train_file_path)
config.data_reader.valid_file_path = os.path.join(
DATASET_PATH, "train", "train_data",
config.data_reader.valid_file_path)
data_reader = create_by_factory(DataReaderFactory, config.data_reader)
datas, helpers = data_reader.read()
# Vocab & Indexing
text_handler = TextHandler(token_makers, lazy_indexing=True)
texts = data_reader.filter_texts(datas)
token_counters = text_handler.make_token_counters(texts)
text_handler.build_vocabs(token_counters)
text_handler.index(datas, data_reader.text_columns)
def bind_load_vocabs(config, token_makers):
CHECKPOINT_FNAME = "checkpoint.bin"
def load(dir_path):
checkpoint_path = os.path.join(dir_path, CHECKPOINT_FNAME)
checkpoint = torch.load(checkpoint_path)
vocabs = {}
token_config = config.token
for token_name in token_config.names:
token = getattr(token_config, token_name, {})
vocab_config = getattr(token, "vocab", {})
texts = checkpoint["vocab_texts"][token_name]
if type(vocab_config) != dict:
vocab_config = vars(vocab_config)
vocabs[token_name] = Vocab(token_name,
**vocab_config).from_texts(texts)
for token_name, token_maker in token_makers.items():
token_maker.set_vocab(vocabs[token_name])
return token_makers
nsml.bind(load=load)
bind_load_vocabs(config, token_makers)
nsml.load(checkpoint=NSML_CHECKPOINT, session=NSML_SESSEION)
# Raw to Tensor Function
text_handler = TextHandler(token_makers, lazy_indexing=False)
raw_to_tensor_fn = text_handler.raw_to_tensor_fn(
data_reader,
cuda_device=device,
)
# Iterator
datasets = data_reader.convert_to_dataset(datas, helpers=helpers)
train_loader = create_data_loader(datasets["train"],
batch_size=config.iterator.batch_size,
shuffle=True,
cuda_device_id=device)
valid_loader = create_data_loader(datasets["valid"],
batch_size=config.iterator.batch_size,
shuffle=False,
cuda_device_id=device)
# Model & Optimizer
model = create_model(token_makers,
ModelFactory,
config.model,
device,
helpers=helpers)
model_parameters = [
param for param in model.parameters() if param.requires_grad
]
optimizer = get_optimizer_by_name("adam")(model_parameters)
def bind_load_model(config, model, **kwargs):
CHECKPOINT_FNAME = "checkpoint.bin"
def load(dir_path):
checkpoint_path = os.path.join(dir_path, CHECKPOINT_FNAME)
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint["weights"])
model.config = checkpoint["config"]
model.metrics = checkpoint["metrics"]
model.init_params = checkpoint["init_params"],
model.predict_helper = checkpoint["predict_helper"],
model.train_counter = TrainCounter(display_unit=1000)
# model.vocabs = load_vocabs(checkpoint)
if "optimizer" in kwargs:
kwargs["optimizer"].load_state_dict(checkpoint["optimizer"][0])
print(f"Model reload checkpoints...! {checkpoint_path}")
nsml.bind(load=load)
bind_load_model(config, model, optimizer=optimizer)
nsml.load(checkpoint=NSML_CHECKPOINT, session=NSML_SESSEION)
if IS_ON_NSML:
bind_nsml(model, optimizer=optimizer)
# Trainer
trainer_config = vars(config.trainer)
trainer_config["model"] = model
trainer = Trainer(**trainer_config)
trainer.train_and_evaluate(train_loader, valid_loader, optimizer)