def fit(self, tasks):
examples = self._prepare_examples(tasks)
config = TrainConfig()
self._model = RubertMulticlassClassifier(
load_bert(self.bert_path),
class_count=len(_ERRORS),
output_name="error_type",
).to(self._device)
batch_collector = _get_batch_collector(self._device, is_train=True)
train_loader = DataLoader(
examples,
batch_size=config.train_batch_size,
shuffle=True,
collate_fn=batch_collector,
pin_memory=False,
)
print(next(iter(train_loader)))
optimizer, scheduler = _get_optimizer(self._model, len(examples), config)
trainer = ClassifierTrainer(self._model, optimizer, scheduler, use_tqdm=True)
trainer.fit(
train_iter=train_loader,
train_batches_per_epoch=int(len(examples) / config.train_batch_size),
epochs_count=config.epoch_count,
)
def fit(self, tasks):
examples = []
for task in tasks:
for example in self._convert_task(task):
examples.append(example)
print('Examples count:', len(examples))
config = TrainConfig()
self._model = BertClassifier(load_bert('data/'),
output_name='label').to(self._device)
batch_collector = _get_batch_collector(self._device, is_train=True)
train_loader = DataLoader(examples,
batch_size=config.train_batch_size,
shuffle=True,
collate_fn=batch_collector,
pin_memory=False)
print(next(iter(train_loader)))
optimizer, scheduler = _get_optimizer(self._model, len(examples),
config)
trainer = ClassifierTrainer(self._model,
optimizer,
scheduler,
use_tqdm=True)
trainer.fit(
train_iter=train_loader,
train_batches_per_epoch=int(
len(examples) / config.train_batch_size),
epochs_count=config.epoch_count,
)
def fit(self, tasks, test_tasks=None):
print("START FIT")
train_examples = self._prepare_examples(tasks)
test_examples = None
if test_tasks is not None:
test_examples = self._prepare_examples(test_tasks)
config = TrainConfig()
self._model = RubertClassifier(load_bert(self.bert_path),
output_name="label").to(self._device)
batch_collector = _get_batch_collector(self._device, is_train=True)
train_loader = DataLoader(
train_examples,
batch_size=config.train_batch_size,
shuffle=True,
collate_fn=batch_collector,
pin_memory=False,
)
test_loader, test_batches_per_epoch = None, 0
if test_examples is not None:
test_loader = DataLoader(
test_examples,
batch_size=config.test_batch_size,
shuffle=False,
collate_fn=batch_collector,
pin_memory=False,
)
test_batches_per_epoch = int(
len(test_examples) / config.test_batch_size)
optimizer, scheduler = _get_optimizer(self._model, len(train_examples),
config)
trainer = ClassifierTrainer(self._model,
optimizer,
scheduler,
use_tqdm=True)
trainer.fit(
train_iter=train_loader,
train_batches_per_epoch=int(
len(train_examples) / config.train_batch_size),
val_iter=test_loader,
val_batches_per_epoch=test_batches_per_epoch,
epochs_count=config.epoch_count,
)
def fit(self, tasks, test_tasks=None):
train_examples = self._prepare_examples(tasks)
test_examples = None
if test_tasks is not None:
test_examples = self._prepare_examples(test_tasks)
print('Examples count:', len(train_examples))
for example in train_examples[0:100:10]:
print(example)
config = TrainConfig()
self._model = BertClassifier(load_bert('data/'),
output_name='label').to(self._device)
batch_collector = _get_batch_collector(self._device, is_train=True)
train_loader = DataLoader(train_examples,
batch_size=config.train_batch_size,
shuffle=True,
collate_fn=batch_collector,
pin_memory=False)
test_loader, test_batches_per_epoch = None, 0
if test_examples is not None:
test_loader = DataLoader(test_examples,
batch_size=config.test_batch_size,
shuffle=False,
collate_fn=batch_collector,
pin_memory=False)
test_batches_per_epoch = int(
len(test_examples) / config.test_batch_size)
print(next(iter(train_loader)))
optimizer, scheduler = _get_optimizer(self._model, len(train_examples),
config)
trainer = ClassifierTrainer(self._model,
optimizer,
scheduler,
use_tqdm=True)
trainer.fit(train_iter=train_loader,
train_batches_per_epoch=int(
len(train_examples) / config.train_batch_size),
val_iter=test_loader,
val_batches_per_epoch=test_batches_per_epoch,
epochs_count=config.epoch_count)