def convert_to_transformers():
farm_input_dir = Path(
"../saved_models/farm-bert-base-german-cased-hatespeech-GermEval18Coarse"
)
transformers_output_dir = "../saved_models/bert-base-german-cased-hatespeech-GermEval18Coarse"
#
# # # load from FARM format
model = AdaptiveModel.load(farm_input_dir, device="cpu")
processor = Processor.load_from_dir(farm_input_dir)
model.connect_heads_with_processor(processor.tasks)
# convert to transformers
transformer_model = Converter.convert_to_transformers(model)[0]
# # Alternative way to convert to transformers:
#transformer_model = model.convert_to_transformers()[0]
# save it (note: transformers use str instead of Path objects)
Path(transformers_output_dir).mkdir(parents=True, exist_ok=True)
transformer_model.save_pretrained(transformers_output_dir)
processor.tokenizer.save_pretrained(transformers_output_dir)
# run predictions (using transformers)
nlp = pipeline('sentiment-analysis',
model=str(transformers_output_dir),
tokenizer=str(transformers_output_dir))
res = nlp("Was ein scheiß Nazi!")
pprint.pprint(res)
def load(cls, load_dir, batch_size=4, gpu=False, embedder_only=False):
"""
Initializes inferencer from directory with saved model.
:param load_dir: Directory where the saved model is located.
:type load_dir: str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
:param embedder_only: If true, a faster processor (InferenceProcessor) is loaded. This should only be used
for extracting embeddings (no downstream predictions).
:type embedder_only: bool
:return: An instance of the Inferencer.
"""
device, n_gpu = initialize_device_settings(
use_cuda=gpu, local_rank=-1, fp16=False
)
model = AdaptiveModel.load(load_dir, device)
if embedder_only:
# model.prediction_heads = []
processor = InferenceProcessor.load_from_dir(load_dir)
else:
processor = Processor.load_from_dir(load_dir)
name = os.path.basename(load_dir)
return cls(model, processor, batch_size=batch_size, gpu=gpu, name=name)
def __init__(self, load_dir, batch_size=4, gpu=False):
"""
Initializes inferencer from directory with saved model.
:param load_dir: Directory containing a saved AdaptiveModel
:type load_dir str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
"""
# Init device and distributed settings
device, n_gpu = initialize_device_settings(
use_cuda=gpu, local_rank=-1, fp16=False
)
self.processor = Processor.load_from_dir(load_dir)
self.model = AdaptiveModel.load(load_dir, device)
self.model.eval()
self.batch_size = batch_size
self.device = device
self.language = self.model.language_model.language
# TODO adjust for multiple prediction heads
if len(self.model.prediction_heads) == 1:
self.prediction_type = self.model.prediction_heads[0].model_type
self.label_map = self.processor.label_maps[0]
elif len(self.model.prediction_heads) == 0:
self.prediction_type = "embedder"
self.name = os.path.basename(load_dir)
set_all_seeds(42, n_gpu)
def convert_to_transformers():
farm_model_dir = Path("../saved_models/bert-english-qa-large")
# load from FARM format
model = AdaptiveModel.load(farm_model_dir, device="cpu")
tokenizer = Tokenizer.load(farm_model_dir)
# convert to transformers
transformer_model = model.convert_to_transformers()
# save it (Note: transformers uses strings rather than Path objects)
model_dir = "../saved_models/bert-large-uncased-whole-word-masking-squad2"
os.makedirs(model_dir, exist_ok=True)
transformer_model.save_pretrained(model_dir)
tokenizer.save_pretrained(model_dir)
# run predictions (using transformers)
nlp = pipeline('question-answering', model=model_dir, tokenizer=model_dir)
res = nlp({
'question':
'Why is model conversion important?',
'context':
'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.'
})
pprint.pprint(res)
def test_conversion_inferencer_qa():
# input
question = "Why is model conversion important?"
text = "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."
# Load from model hub
model = "deepset/bert-base-cased-squad2"
nlp = Inferencer.load(model, task_type="question_answering", num_processes=0)
assert nlp.processor.tokenizer.do_lower_case == False
assert nlp.processor.tokenizer.is_fast == True
QA_input = [{"questions": [question], "text": text}]
result_farm = nlp.inference_from_dicts(dicts=QA_input)
answer_farm = result_farm[0]["predictions"][0]["answers"][0]["answer"]
assert answer_farm == 'gives freedom to the user'
# save it
farm_model_dir = Path("testsave/bert-conversion-test")
nlp.save(farm_model_dir)
# free RAM
del nlp
# load from disk in FARM format
model = AdaptiveModel.load(farm_model_dir, device="cpu")
tokenizer = Tokenizer.load(farm_model_dir)
# convert to transformers
transformer_model = Converter.convert_to_transformers(model)[0]
# free RAM
del model
# save it (Note: transformers uses strings rather than Path objects)
model_dir = "testsave/bert-conversion-test-hf"
os.makedirs(model_dir, exist_ok=True)
transformer_model.save_pretrained(model_dir)
tokenizer.save_pretrained(model_dir)
del transformer_model
del tokenizer
# run predictions (using transformers)
nlp = pipeline('question-answering', model=model_dir, tokenizer=model_dir)
result_transformers = nlp({
'question': question,
'context': text
})
answer_transformers = result_transformers["answer"]
assert answer_farm == answer_transformers
del nlp
def test_prediction_head_load_save_class_weights(tmp_path, caplog=None):
"""This is a regression test for #428 and #422."""
if caplog:
caplog.set_level(logging.CRITICAL)
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=False)
batch_size = 1
lang_model = "bert-base-german-cased"
data_dir_path = "samples/doc_class"
tokenizer = Tokenizer.load(
pretrained_model_name_or_path=lang_model,
do_lower_case=False)
tcp_params = dict(tokenizer=tokenizer,
max_seq_len=8,
data_dir=Path(data_dir_path),
train_filename="train-sample.tsv",
label_list=["OTHER", "OFFENSE"],
metric="f1_macro",
dev_filename="test-sample.tsv",
test_filename=None,
dev_split=0.0,
label_column_name="coarse_label")
processor = TextClassificationProcessor(**tcp_params)
data_silo = DataSilo(
processor=processor,
batch_size=batch_size)
language_model = LanguageModel.load(lang_model)
prediction_head = TextClassificationHead(
num_labels=2,
class_weights=data_silo.calculate_class_weights(task_name="text_classification"))
model = AdaptiveModel(
language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
lm_output_types=["per_sequence"],
device=device)
model.save(tmp_path)
model_loaded = AdaptiveModel.load(tmp_path, device='cpu')
assert model_loaded is not None
def load(
cls,
load_dir,
batch_size=4,
gpu=False,
embedder_only=False,
return_class_probs=False,
strict=True
):
"""
Initializes Inferencer from directory with saved model.
:param load_dir: Directory where the saved model is located.
:type load_dir: str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
:param embedder_only: If true, a faster processor (InferenceProcessor) is loaded. This should only be used for extracting embeddings (no downstream predictions).
:type embedder_only: bool
:param strict: whether to strictly enforce that the keys loaded from saved model match the ones in
the PredictionHead (see torch.nn.module.load_state_dict()).
Set to `False` for backwards compatibility with PHs saved with older version of FARM.
:type strict: bool
:return: An instance of the Inferencer.
"""
device, n_gpu = initialize_device_settings(use_cuda=gpu, local_rank=-1, fp16=False)
model = AdaptiveModel.load(load_dir, device, strict=strict)
if embedder_only:
# model.prediction_heads = []
processor = InferenceProcessor.load_from_dir(load_dir)
else:
processor = Processor.load_from_dir(load_dir)
name = os.path.basename(load_dir)
return cls(
model,
processor,
batch_size=batch_size,
gpu=gpu,
name=name,
return_class_probs=return_class_probs,
)
def test_conversion_inferencer_ner():
# input
text = "Paris is a town in France."
# Load from model hub
model = "dslim/bert-base-NER"
nlp = Inferencer.load(model, task_type="ner", num_processes=0)
assert nlp.processor.tokenizer.do_lower_case == False
assert nlp.processor.tokenizer.is_fast == True
input = [{"text": text}]
result_farm = nlp.inference_from_dicts(dicts=input)
pred_farm = result_farm[0]["predictions"]
assert pred_farm[0][0]["label"] == 'LOC'
assert pred_farm[0][1]["label"] == 'LOC'
assert len(pred_farm[0]) == 2
# save it
farm_model_dir = Path("testsave/bert-conversion-test-hf")
nlp.save(farm_model_dir)
del nlp
# load from disk in FARM format
model = AdaptiveModel.load(farm_model_dir, device="cpu")
tokenizer = Tokenizer.load(farm_model_dir)
# convert to transformers
transformer_model = Converter.convert_to_transformers(model)[0]
del model
# save it (Note: transformers uses strings rather than Path objects)
model_dir = "testsave/bert-conversion-test-hf"
os.makedirs(model_dir, exist_ok=True)
transformer_model.save_pretrained(model_dir)
tokenizer.save_pretrained(model_dir)
del transformer_model
del tokenizer
# run predictions (using transformers)
nlp = pipeline('ner', model=model_dir, tokenizer=model_dir)
result_transformers = nlp(text)
assert result_transformers[0]["entity"] == 'B-LOC'
assert result_transformers[1]["entity"] == 'B-LOC'
assert len(result_transformers) == 2
del nlp
def __init__(self, load_dir, batch_size=4, gpu=False):
# Init device and distributed settings
device, n_gpu = initialize_device_settings(use_cuda=gpu,
local_rank=-1,
fp16=False)
self.processor = Processor.load_from_dir(load_dir)
self.model = AdaptiveModel.load(load_dir, device)
self.model.eval()
self.batch_size = batch_size
self.device = device
self.language = self.model.language_model.language
# TODO adjust for multiple prediction heads
self.prediction_type = self.model.prediction_heads[0].model_type
self.name = os.path.basename(load_dir)
self.label_map = self.processor.label_maps[0]
set_all_seeds(42, 1)
def test_conversion_inferencer_classification():
# input
text = "Das ist blöd."
# Load from model hub
model = "deepset/bert-base-german-cased-hatespeech-GermEval18Coarse"
nlp = Inferencer.load(model,
task_type="text_classification",
num_processes=0)
assert nlp.processor.tokenizer.do_lower_case == False
assert nlp.processor.tokenizer.is_fast == True
input = [{"text": text}]
result_farm = nlp.inference_from_dicts(dicts=input)
pred_farm = result_farm[0]["predictions"][0]["label"]
assert pred_farm == 'OFFENSE'
# save it
farm_model_dir = Path("testsave/bert-conversion-test-hf")
nlp.save(farm_model_dir)
del nlp
# load from disk in FARM format
model = AdaptiveModel.load(farm_model_dir, device="cpu")
tokenizer = Tokenizer.load(farm_model_dir)
# convert to transformers
transformer_model = Converter.convert_to_transformers(model)[0]
del model
# save it (Note: transformers uses strings rather than Path objects)
model_dir = "testsave/bert-conversion-test-hf"
os.makedirs(model_dir, exist_ok=True)
transformer_model.save_pretrained(model_dir)
tokenizer.save_pretrained(model_dir)
del transformer_model
del tokenizer
# run predictions (using transformers)
nlp = pipeline('sentiment-analysis', model=model_dir, tokenizer=model_dir)
result_transformers = nlp(text)
pred_transformers = result_transformers[0]["label"]
assert pred_farm == pred_transformers
del nlp
def load(cls, load_dir, batch_size=4, gpu=False):
"""
Initializes inferencer from directory with saved model.
:param load_dir: Directory where the saved model is located.
:type load_dir: str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
:return: An instance of the Inferencer.
"""
device, n_gpu = initialize_device_settings(use_cuda=gpu,
local_rank=-1,
fp16=False)
model = AdaptiveModel.load(load_dir, device)
processor = Processor.load_from_dir(load_dir)
name = os.path.basename(load_dir)
return cls(model, processor, batch_size=batch_size, gpu=gpu, name=name)
def train(self):
""" Perform the training procedure. """
# connect the prediction heads with the right output from processor
self.model.connect_heads_with_processor(self.data_silo.processor.tasks,
require_labels=True)
# Check that the tokenizer fits the language model
self.model.verify_vocab_size(
vocab_size=len(self.data_silo.processor.tokenizer))
logger.info(f"\n {GROWING_TREE}")
self.model.train()
do_stopping = False
evalnr = 0
loss = 0
resume_from_step = self.from_step
for epoch in range(self.from_epoch + 1, self.epochs + 1):
train_data_loader = self.data_silo.get_data_loader("train")
progress_bar = tqdm(train_data_loader)
for step, batch in enumerate(progress_bar, start=1):
# when resuming training from a checkpoint, we want to fast forward to the step of the checkpoint
if resume_from_step and step <= resume_from_step:
if resume_from_step == step:
resume_from_step = None
continue
if self.sigterm_handler and self.sigterm_handler.kill_now: # save the current state as a checkpoint
logger.info(
"Received a SIGTERM signal. Saving the current train state as a checkpoint ..."
)
self._save()
sys.exit(0)
# save a checkpoint and continue train (do not create a new checkpoint if just resumed from a checkpoint)
if self.checkpoint_every and step % self.checkpoint_every == 0 and resume_from_step + 1 != step:
self._save()
progress_bar.set_description(
f"Train epoch {epoch}/{self.epochs} (Cur. train loss: {loss:.4f})"
)
# Move batch of samples to device
batch = {key: batch[key].to(self.device) for key in batch}
# Forward pass through model
logits = self.model.forward(**batch)
per_sample_loss = self.model.logits_to_loss(
logits=logits, global_step=self.global_step, **batch)
loss = self.backward_propagate(per_sample_loss, step)
# Perform evaluation
if self.global_step % self.evaluate_every == 0 and self.global_step != 0:
# When using StreamingDataSilo, each evaluation creates a new instance of
# dev_data_loader. In cases like training from scratch, this could cause
# some variance across evaluators due to the randomness in word masking.
dev_data_loader = self.data_silo.get_data_loader("dev")
if dev_data_loader is not None:
evaluator_dev = Evaluator(
data_loader=dev_data_loader,
tasks=self.data_silo.processor.tasks,
device=self.device)
evalnr += 1
result = evaluator_dev.eval(self.model)
evaluator_dev.log_results(result, "Dev",
self.global_step)
if self.early_stopping:
do_stopping, save_model, eval_value = self.early_stopping.check_stopping(
result)
if save_model:
logger.info(
"Saving current best model to {}, eval={}".
format(self.early_stopping.save_dir,
eval_value))
self.model.save(self.early_stopping.save_dir)
self.data_silo.processor.save(
self.early_stopping.save_dir)
if do_stopping:
# log the stopping
logger.info(
"STOPPING EARLY AT EPOCH {}, STEP {}, EVALUATION {}"
.format(epoch, step, evalnr))
if do_stopping:
break
self.global_step += 1
self.from_step = step
self.from_epoch = epoch
if do_stopping:
break
# With early stopping we want to restore the best model
if self.early_stopping and self.early_stopping.save_dir:
logger.info("Restoring best model so far from {}".format(
self.early_stopping.save_dir))
lm_name = self.model.language_model.name
model = AdaptiveModel.load(self.early_stopping.save_dir,
self.device,
lm_name=lm_name)
model.connect_heads_with_processor(self.data_silo.processor.tasks,
require_labels=True)
# Eval on test set
test_data_loader = self.data_silo.get_data_loader("test")
if test_data_loader is not None:
evaluator_test = Evaluator(data_loader=test_data_loader,
tasks=self.data_silo.processor.tasks,
device=self.device)
result = evaluator_test.eval(self.model)
evaluator_test.log_results(result, "Test", self.global_step)
return self.model
def train(self):
"""
Perform the training procedure.
The training is visualized by a progress bar. It counts the epochs in a zero based manner.
For example, when you specify ``epochs=20`` it starts to count from 0 to 19.
If trainer evaluates the model with a test set the result of the
evaluation is stored in ``test_result``.
:return: Returns the model after training. When you do ``early_stopping``
with a ``save_dir`` the best model is loaded and returned.
"""
# connect the prediction heads with the right output from processor
self.model.connect_heads_with_processor(self.data_silo.processor.tasks,
require_labels=True)
# Check that the tokenizer(s) fits the language model(s)
if hasattr(self.model, "language_model2"):
self.model.verify_vocab_size(
vocab_size1=len(self.data_silo.processor.query_tokenizer),
vocab_size2=len(self.data_silo.processor.passage_tokenizer))
else:
self.model.verify_vocab_size(
vocab_size=len(self.data_silo.processor.tokenizer))
self.model.train()
do_stopping = False
evalnr = 0
loss = 0
resume_from_step = self.from_step
if self.local_rank in [0, -1]:
logger.info(f"\n {GROWING_TREE}")
for epoch in range(self.from_epoch, self.epochs):
early_break = False
self.from_epoch = epoch
train_data_loader = self.data_silo.get_data_loader("train")
progress_bar = tqdm(train_data_loader,
disable=self.local_rank not in [0, -1]
or self.disable_tqdm)
for step, batch in enumerate(progress_bar):
# when resuming training from a checkpoint, we want to fast forward to the step of the checkpoint
if resume_from_step and step <= resume_from_step:
# TODO: Improve skipping for StreamingDataSilo
# The seeds before and within the loop are currently needed, if you need full reproducibility
# of runs with vs. without checkpointing using StreamingDataSilo. Reason: While skipping steps in StreamingDataSilo,
# we update the state of the random number generator (e.g. due to masking words), which can impact the model behaviour (e.g. dropout)
if step % 10000 == 0:
logger.info(
f"Skipping {step} out of {resume_from_step} steps ..."
)
if resume_from_step == step:
logger.info(
f"Finished skipping {resume_from_step} steps ...")
resume_from_step = None
else:
continue
progress_bar.set_description(
f"Train epoch {epoch}/{self.epochs-1} (Cur. train loss: {loss:.4f})"
)
# Only for distributed training: we need to ensure that all ranks still have a batch left for training
if self.local_rank != -1:
if not self._all_ranks_have_data(has_data=1, step=step):
early_break = True
break
# Move batch of samples to device
batch = {key: batch[key].to(self.device) for key in batch}
# Forward & backward pass through model
logits = self.model.forward(**batch)
per_sample_loss = self.model.logits_to_loss(
logits=logits, global_step=self.global_step, **batch)
loss = self.backward_propagate(per_sample_loss, step)
# Perform evaluation
if self.evaluate_every != 0 \
and self.global_step % self.evaluate_every == 0 \
and self.global_step != 0\
and self.local_rank in [0,-1]:
# When using StreamingDataSilo, each evaluation creates a new instance of
# dev_data_loader. In cases like training from scratch, this could cause
# some variance across evaluators due to the randomness in word masking.
dev_data_loader = self.data_silo.get_data_loader("dev")
if dev_data_loader is not None:
evaluator_dev = Evaluator(
data_loader=dev_data_loader,
tasks=self.data_silo.processor.tasks,
device=self.device,
report=self.eval_report)
evalnr += 1
result = evaluator_dev.eval(self.model)
evaluator_dev.log_results(result, "Dev",
self.global_step)
if self.early_stopping:
do_stopping, save_model, eval_value = self.early_stopping.check_stopping(
result)
if save_model:
logger.info(
"Saving current best model to {}, eval={}".
format(self.early_stopping.save_dir,
eval_value))
self.model.save(self.early_stopping.save_dir)
self.data_silo.processor.save(
self.early_stopping.save_dir)
if do_stopping:
# log the stopping
logger.info(
"STOPPING EARLY AT EPOCH {}, STEP {}, EVALUATION {}"
.format(epoch, step, evalnr))
if do_stopping:
break
self.global_step += 1
self.from_step = step + 1
# save the current state as a checkpoint before exiting if a SIGTERM signal is received
if self.sigterm_handler and self.sigterm_handler.kill_now:
logger.info(
"Received a SIGTERM signal. Saving the current train state as a checkpoint ..."
)
if self.local_rank in [0, -1]:
self._save()
torch.distributed.destroy_process_group()
sys.exit(0)
# save a checkpoint and continue train
if self.checkpoint_every and step % self.checkpoint_every == 0:
if self.local_rank in [0, -1]:
self._save()
# Let other ranks wait until rank 0 has finished saving
if self.local_rank != -1:
torch.distributed.barrier()
if do_stopping:
break
# Only for distributed training: we need to ensure that all ranks still have a batch left for training
if self.local_rank != -1 and not early_break:
self._all_ranks_have_data(has_data=False)
# With early stopping we want to restore the best model
if self.early_stopping and self.early_stopping.save_dir:
logger.info("Restoring best model so far from {}".format(
self.early_stopping.save_dir))
lm_name = self.model.language_model.name
self.model = AdaptiveModel.load(self.early_stopping.save_dir,
self.device,
lm_name=lm_name)
self.model.connect_heads_with_processor(
self.data_silo.processor.tasks, require_labels=True)
# Eval on test set
if self.evaluator_test and self.local_rank in [0, -1]:
test_data_loader = self.data_silo.get_data_loader("test")
if test_data_loader is not None:
evaluator_test = Evaluator(
data_loader=test_data_loader,
tasks=self.data_silo.processor.tasks,
device=self.device)
self.test_result = evaluator_test.eval(self.model)
evaluator_test.log_results(self.test_result, "Test",
self.global_step)
return self.model
def doc_classification_crossvalidation():
##########################
########## Logging
##########################
logger = logging.getLogger(__name__)
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO)
# reduce verbosity from transformers library
logging.getLogger('transformers').setLevel(logging.WARNING)
# ml_logger = MLFlowLogger(tracking_uri="https://public-mlflow.deepset.ai/")
# for local logging instead:
ml_logger = MLFlowLogger(tracking_uri="logs")
# ml_logger.init_experiment(experiment_name="Public_FARM", run_name="DocClassification_ES_f1_1")
##########################
########## Settings
##########################
xval_folds = 5
xval_stratified = True
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
n_epochs = 20
batch_size = 32
evaluate_every = 100
lang_model = "bert-base-german-cased"
use_amp = None
# 1.Create a tokenizer
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=False)
# The evaluation on the dev-set can be done with one of the predefined metrics or with a
# metric defined as a function from (preds, labels) to a dict that contains all the actual
# metrics values. The function must get registered under a string name and the string name must
# be used.
# For xval, we also store the actual predictions and labels in each result so we can
# calculate overall metrics over all folds later
def mymetrics(preds, labels):
acc = simple_accuracy(preds, labels).get("acc")
f1other = f1_score(y_true=labels, y_pred=preds, pos_label="OTHER")
f1offense = f1_score(y_true=labels, y_pred=preds, pos_label="OFFENSE")
f1macro = f1_score(y_true=labels, y_pred=preds, average="macro")
f1micro = f1_score(y_true=labels, y_pred=preds, average="macro")
mcc = matthews_corrcoef(labels, preds)
return {
"acc": acc,
"f1_other": f1other,
"f1_offense": f1offense,
"f1_macro": f1macro,
"f1_micro": f1micro,
"mcc": mcc
}
register_metrics('mymetrics', mymetrics)
metric = 'mymetrics'
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# Here we load GermEval 2018 Data.
# The processor wants to know the possible labels ...
label_list = ["OTHER", "OFFENSE"]
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=64,
data_dir=Path("../data/germeval18"),
label_list=label_list,
metric=metric,
label_column_name="coarse_label")
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(processor=processor, batch_size=batch_size)
# Load one silo for each fold in our cross-validation
silos = DataSiloForCrossVal.make(data_silo, n_splits=xval_folds)
# the following steps should be run for each of the folds of the cross validation, so we put them
# into a function
def train_on_split(silo_to_use, n_fold, save_dir):
logger.info(
f"############ Crossvalidation: Fold {n_fold} ############")
# Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(lang_model)
# b) and a prediction head on top that is suited for our task => Text classification
prediction_head = TextClassificationHead(
class_weights=data_silo.calculate_class_weights(
task_name="text_classification"),
num_labels=len(label_list))
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.2,
lm_output_types=["per_sequence"],
device=device)
# Create an optimizer
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=0.5e-5,
device=device,
n_batches=len(silo_to_use.loaders["train"]),
n_epochs=n_epochs,
use_amp=use_amp)
# Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it from time to time
# Also create an EarlyStopping instance and pass it on to the trainer
# An early stopping instance can be used to save the model that performs best on the dev set
# according to some metric and stop training when no improvement is happening for some iterations.
# NOTE: Using a different save directory for each fold, allows us afterwards to use the
# nfolds best models in an ensemble!
save_dir = Path(str(save_dir) + f"-{n_fold}")
earlystopping = EarlyStopping(
metric="f1_offense",
mode=
"max", # use the metric from our own metrics function instead of loss
save_dir=save_dir, # where to save the best model
patience=
5 # number of evaluations to wait for improvement before terminating the training
)
trainer = Trainer(model=model,
optimizer=optimizer,
data_silo=silo_to_use,
epochs=n_epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=evaluate_every,
device=device,
early_stopping=earlystopping,
evaluator_test=False)
# train it
trainer.train()
return trainer.model
# for each fold, run the whole training, earlystopping to get a model, then evaluate the model
# on the test set of each fold
# Remember all the results for overall metrics over all predictions of all folds and for averaging
allresults = []
all_preds = []
all_labels = []
bestfold = None
bestf1_offense = -1
save_dir = Path("saved_models/bert-german-doc-tutorial-es")
for num_fold, silo in enumerate(silos):
model = train_on_split(silo, num_fold, save_dir)
# do eval on test set here (and not in Trainer),
# so that we can easily store the actual preds and labels for a "global" eval across all folds.
evaluator_test = Evaluator(data_loader=silo.get_data_loader("test"),
tasks=silo.processor.tasks,
device=device)
result = evaluator_test.eval(model, return_preds_and_labels=True)
evaluator_test.log_results(result,
"Test",
steps=len(silo.get_data_loader("test")),
num_fold=num_fold)
allresults.append(result)
all_preds.extend(result[0].get("preds"))
all_labels.extend(result[0].get("labels"))
# keep track of best fold
f1_offense = result[0]["f1_offense"]
if f1_offense > bestf1_offense:
bestf1_offense = f1_offense
bestfold = num_fold
# Save the per-fold results to json for a separate, more detailed analysis
with open("doc_classification_xval.results.json", "wt") as fp:
json.dump(allresults, fp)
# calculate overall metrics across all folds
xval_f1_micro = f1_score(all_labels,
all_preds,
labels=label_list,
average="micro")
xval_f1_macro = f1_score(all_labels,
all_preds,
labels=label_list,
average="macro")
xval_f1_offense = f1_score(all_labels,
all_preds,
labels=label_list,
pos_label="OFFENSE")
xval_f1_other = f1_score(all_labels,
all_preds,
labels=label_list,
pos_label="OTHER")
xval_mcc = matthews_corrcoef(all_labels, all_preds)
logger.info("XVAL F1 MICRO: ", xval_f1_micro)
logger.info("XVAL F1 MACRO: ", xval_f1_macro)
logger.info("XVAL F1 OFFENSE: ", xval_f1_offense)
logger.info("XVAL F1 OTHER: ", xval_f1_other)
logger.info("XVAL MCC: ", xval_mcc)
# -----------------------------------------------------
# Just for illustration, use the best model from the best xval val for evaluation on
# the original (still unseen) test set.
logger.info(
"###### Final Eval on hold out test set using best model #####")
evaluator_origtest = Evaluator(
data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# restore model from the best fold
lm_name = model.language_model.name
save_dir = Path(f"saved_models/bert-german-doc-tutorial-es-{bestfold}")
model = AdaptiveModel.load(save_dir, device, lm_name=lm_name)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
result = evaluator_origtest.eval(model)
logger.info("TEST F1 MICRO: ", result[0]["f1_micro"])
logger.info("TEST F1 MACRO: ", result[0]["f1_macro"])
logger.info("TEST F1 OFFENSE: ", result[0]["f1_offense"])
logger.info("TEST F1 OTHER: ", result[0]["f1_other"])
logger.info("TEST MCC: ", result[0]["mcc"])
xval_mcc = matthews_corrcoef(all_labels, all_preds)
logger.info("XVAL F1 MICRO: ", xval_f1_micro)
logger.info("XVAL F1 MACRO: ", xval_f1_macro)
logger.info("XVAL F1 OFFENSE: ", xval_f1_offense)
logger.info("XVAL F1 OTHER: ", xval_f1_other)
logger.info("XVAL MCC: ", xval_mcc)
# -----------------------------------------------------
# Just for illustration, use the best model from the best xval val for evaluation on
# the original (still unseen) test set.
logger.info("###### Final Eval on hold out test set using best model #####")
evaluator_origtest = Evaluator(data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# restore model from the best fold
lm_name = model.language_model.name
save_dir = "saved_models/bert-german-doc-tutorial-es-{}".format(bestfold)
model = AdaptiveModel.load(save_dir, device, lm_name=lm_name)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
result = evaluator_origtest.eval(model)
logger.info("TEST F1 MICRO: ", result[0]["f1_micro"])
logger.info("TEST F1 MACRO: ", result[0]["f1_macro"])
logger.info("TEST F1 OFFENSE: ", result[0]["f1_offense"])
logger.info("TEST F1 OTHER: ", result[0]["f1_other"])
logger.info("TEST MCC: ", result[0]["mcc"])
# fmt: on
def train(self, model):
""" Perform the training procedure. """
# connect the prediction heads with the right output from processor
model.connect_heads_with_processor(self.data_silo.processor.tasks,
require_labels=True)
# Check that the tokenizer fits the language model
model.verify_vocab_size(
vocab_size=len(self.data_silo.processor.tokenizer))
logger.info(f"\n {GROWING_TREE}")
model.train()
do_stopping = False
evalnr = 0
loss = 0
for epoch in range(1, self.epochs + 1):
progress_bar = tqdm(self.data_loader_train)
for step, batch in enumerate(progress_bar):
progress_bar.set_description(
f"Train epoch {epoch}/{self.epochs} (Cur. train loss: {loss:.4f})"
)
# Move batch of samples to device
batch = {key: batch[key].to(self.device) for key in batch}
# Forward pass through model
logits = model.forward(**batch)
per_sample_loss = model.logits_to_loss(logits=logits, **batch)
loss = self.backward_propagate(per_sample_loss, step)
# Perform evaluation
if self.evaluator_dev:
if self.global_step != 0 and (self.global_step %
self.evaluate_every == 0):
evalnr += 1
result = self.evaluator_dev.eval(model)
self.evaluator_dev.log_results(result, "Dev",
self.global_step)
if self.early_stopping:
do_stopping, save_model, eval_value = self.early_stopping.check_stopping(
result)
if save_model:
logger.info(
"Saving current best model to {}, eval={}".
format(self.early_stopping.save_dir,
eval_value))
model.save(self.early_stopping.save_dir)
self.data_silo.processor.save(
self.early_stopping.save_dir)
if do_stopping:
# log the stopping
logger.info(
"STOPPING EARLY AT EPOCH {}, STEP {}, EVALUATION {}"
.format(epoch, step, evalnr))
if do_stopping:
break
self.global_step += 1
if do_stopping:
break
# With early stopping we want to restore the best model
if self.early_stopping and self.early_stopping.save_dir:
logger.info("Restoring best model so far from {}".format(
self.early_stopping.save_dir))
lm_name = model.language_model.name
model = AdaptiveModel.load(self.early_stopping.save_dir,
self.device,
lm_name=lm_name)
model.connect_heads_with_processor(self.data_silo.processor.tasks,
require_labels=True)
# Eval on test set
if self.evaluator_test:
result = self.evaluator_test.eval(model)
self.evaluator_test.log_results(result, "Test", self.global_step)
return model
def load(cls,
model_name_or_path,
batch_size=4,
gpu=False,
task_type=None,
return_class_probs=False,
strict=True,
max_seq_len=256):
"""
Load an Inferencer incl. all relevant components (model, tokenizer, processor ...) either by
1. specifying a public name from transformers' model hub (https://huggingface.co/models)
2. or pointing to a local directory it is saved in.
:param model_name_or_path: Local directory or public name of the model to load.
:type model_name_or_path: str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
:param task_type: Type of task the model should be used for. Currently supporting:
"embeddings", "question_answering", "text_classification". More coming soon...
:param task_type: str
:param strict: whether to strictly enforce that the keys loaded from saved model match the ones in
the PredictionHead (see torch.nn.module.load_state_dict()).
Set to `False` for backwards compatibility with PHs saved with older version of FARM.
:type strict: bool
:return: An instance of the Inferencer.
"""
device, n_gpu = initialize_device_settings(use_cuda=gpu,
local_rank=-1,
use_amp=None)
name = os.path.basename(model_name_or_path)
# a) either from local dir
if os.path.exists(model_name_or_path):
model = AdaptiveModel.load(model_name_or_path,
device,
strict=strict)
if task_type == "embeddings":
processor = InferenceProcessor.load_from_dir(
model_name_or_path)
else:
processor = Processor.load_from_dir(model_name_or_path)
# b) or from remote transformers model hub
else:
logger.info(
f"Could not find `{model_name_or_path}` locally. Try to download from model hub ..."
)
if not task_type:
raise ValueError(
"Please specify the 'task_type' of the model you want to load from transformers. "
"Valid options for arg `task_type`:"
"'question_answering', 'embeddings', 'text_classification'"
)
model = AdaptiveModel.convert_from_transformers(
model_name_or_path, device, task_type)
config = AutoConfig.from_pretrained(model_name_or_path)
tokenizer = Tokenizer.load(model_name_or_path)
# TODO infer task_type automatically from config (if possible)
if task_type == "question_answering":
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
label_list=["start_token", "end_token"],
metric="squad",
data_dir=None,
)
elif task_type == "embeddings":
processor = InferenceProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len)
elif task_type == "text_classification":
label_list = list(config.id2label[id]
for id in range(len(config.id2label)))
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir=None,
label_list=label_list,
label_column_name="label",
metric="acc",
quote_char='"',
)
# elif task_type == "multilabel-classification":
# # label_list = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
# label_list = list(config.label2id.keys())
#
# processor = TextClassificationProcessor(tokenizer=tokenizer,
# max_seq_len=max_seq_len,
# data_dir=None,
# label_list=label_list,
# label_column_name="label",
# metric="acc",
# quote_char='"',
# multilabel=True,
# )
elif task_type == "ner":
label_list = list(config.label2id.keys())
processor = NERProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir=None,
metric="seq_f1",
label_list=label_list)
else:
raise ValueError(
f"`task_type` {task_type} is not supported yet. "
f"Valid options for arg `task_type`: 'question_answering', 'embeddings', 'text_classification'"
)
return cls(
model,
processor,
batch_size=batch_size,
gpu=gpu,
name=name,
return_class_probs=return_class_probs,
)
def perform_fine_tuning(current_info_need,
bert_model,
label_list,
num_epochs,
condition,
folds=10,
stratified=True,
learning_rate=2e-5,
batch_size=32,
embeds_dropout_prob=.1):
## Define evaluation metrics ##
def evaluation_metrics(preds, labels):
acc = simple_accuracy(preds, labels).get("acc")
f1other = f1_score(y_true=labels, y_pred=preds, pos_label="Other")
f1infoneed = f1_score(y_true=labels,
y_pred=preds,
pos_label=current_info_need)
recall_infoneed = recall_score(y_true=labels,
y_pred=preds,
pos_label=current_info_need)
precision_infoneed = precision_score(y_true=labels,
y_pred=preds,
pos_label=current_info_need)
recall_other = recall_score(y_true=labels,
y_pred=preds,
pos_label="Other")
precision_other = precision_score(y_true=labels,
y_pred=preds,
pos_label="Other")
recall_macro = recall_score(y_true=labels,
y_pred=preds,
average="macro")
precision_macro = precision_score(y_true=labels,
y_pred=preds,
average="macro")
recall_micro = recall_score(y_true=labels,
y_pred=preds,
average="micro")
precision_micro = precision_score(y_true=labels,
y_pred=preds,
average="micro")
recall_weighted = recall_score(y_true=labels,
y_pred=preds,
average="weighted")
precision_weighted = precision_score(y_true=labels,
y_pred=preds,
average="weighted")
f1macro = f1_score(y_true=labels, y_pred=preds, average="macro")
f1micro = f1_score(y_true=labels, y_pred=preds, average="micro")
mcc = matthews_corrcoef(labels, preds)
f1weighted = f1_score(y_true=labels,
y_pred=preds,
average="weighted")
return {
"info_need": current_info_need,
"model": bert_model,
"num_epochs": num_epochs,
"condition": condition,
"acc": acc,
"f1_other": f1other,
"f1_infoneed": f1infoneed,
"precision_infoneed": precision_infoneed,
"recall_infoneed": recall_infoneed,
"recall_other": recall_other,
"precision_other": precision_other,
"recall_macro": recall_macro,
"precision_macro": precision_macro,
"recall_micro": recall_micro,
"precision_micro": precision_micro,
"recall_weighted": recall_weighted,
"precision_weighted": precision_weighted,
"f1_weighted": f1weighted,
"f1_macro": f1macro,
"f1_micro": f1micro,
"f1_weighted": f1weighted,
"mcc": mcc
}
register_metrics(
f'eval_metrics_{current_info_need}_{bert_model}_{condition}__{num_epochs}_epochs',
evaluation_metrics)
metric = f'eval_metrics_{current_info_need}_{bert_model}_{condition}__{num_epochs}_epochs'
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
logger, ml_logger = init_logging()
tokenizer = Tokenizer.load(pretrained_model_name_or_path=bert_model,
do_lower_case=False)
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=256,
train_filename=
f"{current_info_need}_{condition}_{num_epochs}_epochs_train.csv",
test_filename=
f"{current_info_need}_{condition}_{num_epochs}_epochs_test.csv",
data_dir="data/",
label_list=label_list,
metric=metric,
text_column_name="utterance",
label_column_name=level,
delimiter=";")
data_silo = DataSilo(processor=processor, batch_size=batch_size)
silos = DataSiloForCrossVal.make(data_silo,
n_splits=folds,
sets=['train', 'test'])
# the following steps should be run for each of the folds of the cross validation, so we put them
# into a function
def train_on_split(silo_to_use, n_fold, save_dir):
logger.info(
f"############ Crossvalidation: Fold {n_fold} ############")
# Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(bert_model)
# b) and a prediction head on top that is suited for our task => Text classification
prediction_head = TextClassificationHead(
class_weights=data_silo.calculate_class_weights(
task_name="text_classification"),
num_labels=len(label_list))
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=embeds_dropout_prob,
lm_output_types=["per_sequence"],
device=device)
# Create an optimizer
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=learning_rate,
device=device,
n_batches=len(silo_to_use.loaders["train"]),
n_epochs=num_epochs,
use_amp=None)
# Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it from time to time
# Also create an EarlyStopping instance and pass it on to the trainer
# An early stopping instance can be used to save the model that performs best on the dev set
# according to some metric and stop training when no improvement is happening for some iterations.
# NOTE: Using a different save directory for each fold, allows us afterwards to use the
# nfolds best models in an ensemble!
save_dir = Path(str(save_dir) + f"-{n_fold}")
earlystopping = EarlyStopping(
metric="f1_infoneed",
mode=
"max", # use the metric from our own metrics function instead of loss
save_dir=save_dir, # where to save the best model
patience=
5 # number of evaluations to wait for improvement before terminating the training
)
trainer = Trainer(model=model,
optimizer=optimizer,
data_silo=silo_to_use,
epochs=num_epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=100,
device=device,
early_stopping=earlystopping,
evaluator_test=False)
# train it
trainer.train()
return trainer.model
# for each fold, run the whole training, earlystopping to get a model, then evaluate the model
# on the test set of each fold
# Remember all the results for overall metrics over all predictions of all folds and for averaging
allresults = []
all_preds = []
all_labels = []
bestfold = None
bestf1_info_need = -1
language_model_name = bert_model
if language_model_name.find("/") != -1:
language_model_name = language_model_name.replace("/", "_")
save_dir = Path(
f"saved_models/{current_info_need}-{condition}-{num_epochs}_epochs-cook-{language_model_name}"
)
for num_fold, silo in enumerate(silos):
model = train_on_split(silo, num_fold, save_dir)
# do eval on test set here (and not in Trainer),
# so that we can easily store the actual preds and labels for a "global" eval across all folds.
evaluator_test = Evaluator(
data_loader=silo.get_data_loader("test"),
tasks=silo.processor.tasks,
device=device)
result = evaluator_test.eval(model, return_preds_and_labels=True)
evaluator_test.log_results(result,
"Test",
steps=len(silo.get_data_loader("test")),
num_fold=num_fold)
allresults.append(result)
all_preds.extend(result[0].get("preds"))
all_labels.extend(result[0].get("labels"))
# keep track of best fold
f1_info_need = result[0]["f1_infoneed"]
if f1_info_need > bestf1_info_need:
bestf1_info_need = f1_info_need
bestfold = num_fold
# emtpy cache to avoid memory leak and cuda OOM across multiple folds
model.cpu()
torch.cuda.empty_cache()
# Save the per-fold results to json for a separate, more detailed analysis
with open(
f"classification_results/test/{current_info_need}-{language_model_name}-{condition}-{num_epochs}_epochs-{folds}-fold-cv.results.json",
"wt") as fp:
json.dump(allresults, fp)
# calculate overall metrics across all folds
xval_f1_other = f1_score(all_labels,
all_preds,
labels=label_list,
pos_label="Other")
xval_f1_info_need = f1_score(all_labels,
all_preds,
labels=label_list,
pos_label=current_info_need)
xval_f1_micro = f1_score(all_labels,
all_preds,
labels=label_list,
average="micro")
xval_f1_macro = f1_score(all_labels,
all_preds,
labels=label_list,
average="macro")
xval_mcc = matthews_corrcoef(all_labels, all_preds)
xval_overall_results = {
"xval_f1_other": xval_f1_other,
f"xval_f1_infoneed": xval_f1_info_need,
"xval_f1_micro": xval_f1_micro,
"xval_f1_macro": xval_f1_macro,
"xval_f1_mcc": xval_mcc
}
logger.info(f"XVAL F1 MICRO: {xval_f1_micro}")
logger.info(f"XVAL F1 MACRO: {xval_f1_macro}")
logger.info(f"XVAL F1 OTHER: {xval_f1_other}")
logger.info(
f"XVAL F1 {current_info_need} {condition} {num_epochs} epochs: {xval_f1_info_need}"
)
logger.info(f"XVAL MCC: {xval_mcc}")
# -----------------------------------------------------
# Just for illustration, use the best model from the best xval val for evaluation on
# the original (still unseen) test set.
logger.info(
"###### Final Eval on hold out test set using best model #####")
evaluator_origtest = Evaluator(
data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# restore model from the best fold
lm_name = model.language_model.name
save_dir = Path(
f"saved_models/{current_info_need}-{condition}-{num_epochs}_epochs-cook-{language_model_name}-{bestfold}"
)
model = AdaptiveModel.load(save_dir, device, lm_name=lm_name)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
result = evaluator_origtest.eval(model)
logger.info("TEST F1 MICRO: {}".format(result[0]["f1_micro"]))
logger.info("TEST F1 MACRO: {}".format(result[0]["f1_macro"]))
logger.info("TEST F1 OTHER: {}".format(result[0]["f1_other"]))
logger.info("TEST F1 {0}: {1}".format(current_info_need,
result[0]["f1_infoneed"]))
logger.info("TEST MCC: {}".format(result[0]["mcc"]))
test_set_results = {
"test_f1_other": result[0]["f1_other"],
"test_f1_infoneed": result[0][f"f1_infoneed"],
"test_f1_micro": result[0]["f1_micro"],
"test_f1_macro": result[0]["f1_macro"],
"test_f1_mcc": result[0]["mcc"]
}
def outcome_pretraining(task_config,
model_name,
cache_dir,
run_name="0",
lr=1e-05,
warmup_steps=5000,
embeds_dropout=0.1,
epochs=200, # large because we use early stopping by default
batch_size=20,
grad_acc_steps=1,
early_stopping_metric="loss",
early_stopping_mode="min",
early_stopping_patience=10,
model_class="Bert",
tokenizer_class="BertTokenizer",
do_lower_case=True,
do_train=True,
do_eval=True,
do_hpo=False,
max_seq_len=512,
seed=11,
eval_every=500,
use_amp=False,
use_cuda=True,
):
# Load task config
task_config = yaml.safe_load(open(task_config))
data_dir = Path(task_config["data"]["data_dir"])
# General Settings
set_all_seeds(seed=seed)
device, n_gpu = initialize_device_settings(use_cuda=use_cuda, use_amp=use_amp)
# 1.Create a tokenizer
tokenizer = Tokenizer.load(pretrained_model_name_or_path=model_name, tokenizer_class=tokenizer_class,
do_lower_case=do_lower_case)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
processor = OutcomePretrainingProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir=data_dir,
train_filename=task_config["data"]["train_filename"],
dev_filename=task_config["data"]["dev_filename"],
seed=seed,
max_size_admission=50,
max_size_discharge=50,
cache_dir=cache_dir)
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a
# few descriptive statistics of our datasets
data_silo = OutcomePretrainingDataSilo(
processor=processor,
caching=True,
cache_dir=cache_dir,
batch_size=batch_size,
max_multiprocessing_chunksize=200)
if do_train:
# Set save dir for experiment output
save_dir = Path(task_config["output_dir"]) / f'{task_config["experiment_name"]}_{run_name}'
# Use HPO config args if config is passed
if do_hpo:
save_dir = save_dir / tune.session.get_trial_name()
else:
exp_name = f"exp_{random.randint(100000, 999999)}"
save_dir = save_dir / exp_name
# Create save dir
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# Setup MLFlow logger
ml_logger = MLFlowLogger(tracking_uri=task_config["log_dir"])
ml_logger.init_experiment(experiment_name=task_config["experiment_name"],
run_name=f'{task_config["experiment_name"]}_{run_name}')
# 4. Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(model_name, language_model_class=model_class)
# b) and NextSentenceHead prediction head or TextClassificationHead if it's not a Bert Model
if model_class == "Bert":
next_sentence_head = NextSentenceHead.load(model_class)
else:
next_sentence_head = TextClassificationHead(num_labels=2)
model = AdaptiveModel(
language_model=language_model,
prediction_heads=[next_sentence_head],
embeds_dropout_prob=embeds_dropout,
lm_output_types=["per_sequence"],
device=device,
)
# 5. Create an optimizer
schedule_opts = {"name": "LinearWarmup",
"num_warmup_steps": warmup_steps}
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=lr,
device=device,
n_batches=len(data_silo.loaders["train"]),
n_epochs=epochs,
use_amp=use_amp,
grad_acc_steps=grad_acc_steps,
schedule_opts=schedule_opts)
# 6. Create an early stopping instance
early_stopping = None
if early_stopping_mode != "none":
early_stopping = EarlyStopping(
mode=early_stopping_mode,
min_delta=0.0001,
save_dir=save_dir,
metric=early_stopping_metric,
patience=early_stopping_patience
)
# 7. Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it
# from time to time
trainer = ExtendedTrainer(
model=model,
optimizer=optimizer,
data_silo=data_silo,
epochs=epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=eval_every,
early_stopping=early_stopping,
device=device,
grad_acc_steps=grad_acc_steps,
evaluator_test=do_eval
)
def score_callback(eval_score, train_loss):
tune.report(roc_auc_dev=eval_score, train_loss=train_loss)
# 8. Train the model
trainer.train(score_callback=score_callback if do_hpo else None)
# 9. Save model if not saved in early stopping
model.save(save_dir / "final_model")
processor.save(save_dir / "final_model")
if do_eval:
# Load newly trained model or existing model
if do_train:
model_dir = save_dir
else:
model_dir = Path(model_name)
logger.info("###### Eval on TEST SET #####")
evaluator_test = Evaluator(
data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device
)
# Load trained model for evaluation
model = AdaptiveModel.load(model_dir, device)
model.connect_heads_with_processor(data_silo.processor.tasks, require_labels=True)
# Evaluate
results = evaluator_test.eval(model, return_preds_and_labels=True)
# Log results
utils.log_results(results, dataset_name="test", steps=len(evaluator_test.data_loader),
save_path=model_dir / "eval_results.txt")
def doc_classification_crossvalidation():
# the code for this function is partially taken from:
# https://github.com/deepset-ai/FARM/blob/master/examples/doc_classification_multilabel.py and
# https://github.com/deepset-ai/FARM/blob/master/examples/doc_classification_crossvalidation.py
# for local logging:
ml_logger = MLFlowLogger(tracking_uri="")
ml_logger.init_experiment(experiment_name="covid-document-classification",
run_name=RUNNAME)
# model settings
xval_folds = FOLDS
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
if RUNLOCAL:
device = "cpu"
n_epochs = NEPOCHS
batch_size = BATCHSIZE
evaluate_every = EVALEVERY
lang_model = MODELTYPE
do_lower_case = False
# 1.Create a tokenizer
tokenizer = Tokenizer.load(
pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
metric = "f1_macro"
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# The processor wants to know the possible labels ...
label_list = LABELS
processor = TextClassificationProcessor(tokenizer=tokenizer,
max_seq_len=MAXLEN,
data_dir=DATADIR,
train_filename=TRAIN,
test_filename=TEST,
dev_split=0.1,
label_list=label_list,
metric=metric,
label_column_name="Categories",
# confusing parameter name: it should be called multiCLASS
# not multiLABEL
multilabel=True
)
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(
processor=processor,
batch_size=batch_size)
# Load one silo for each fold in our cross-validation
silos = DataSiloForCrossVal.make(data_silo, n_splits=xval_folds)
# the following steps should be run for each of the folds of the cross validation, so we put them
# into a function
def train_on_split(silo_to_use, n_fold, save_dir, dev):
# Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(lang_model)
# b) and a prediction head on top that is suited for our task => Text classification
prediction_head = MultiLabelTextClassificationHead(
# there is still an error with class weights ...
# class_weights=data_silo.calculate_class_weights(task_name="text_classification"),
num_labels=len(label_list))
model = AdaptiveModel(
language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.2,
lm_output_types=["per_sequence"],
device=dev)
# Create an optimizer
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=0.5e-5,
device=dev,
n_batches=len(silo_to_use.loaders["train"]),
n_epochs=n_epochs)
# Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it from time to time
# Also create an EarlyStopping instance and pass it on to the trainer
save_dir = Path(str(save_dir) + f"-{n_fold}")
# unfortunately, early stopping is still not working
earlystopping = EarlyStopping(
metric="f1_macro", mode="max",
save_dir=save_dir, # where to save the best model
patience=5 # number of evaluations to wait for improvement before terminating the training
)
trainer = Trainer(model=model, optimizer=optimizer,
data_silo=silo_to_use, epochs=n_epochs,
n_gpu=n_gpu, lr_schedule=lr_schedule,
evaluate_every=evaluate_every,
device=dev, evaluator_test=False,
#early_stopping=earlystopping)
)
# train it
trainer.train()
trainer.model.save(save_dir)
return trainer.model
# for each fold, run the whole training, earlystopping to get a model, then evaluate the model
# on the test set of each fold
# Remember all the results for overall metrics over all predictions of all folds and for averaging
allresults = []
all_preds = []
all_labels = []
bestfold = None
bestf1_macro = -1
save_dir = Path("saved_models/covid-classification-v1")
for num_fold, silo in enumerate(silos):
model = train_on_split(silo, num_fold, save_dir, device)
# do eval on test set here (and not in Trainer),
# so that we can easily store the actual preds and labels for a "global" eval across all folds.
evaluator_test = Evaluator(
data_loader=silo.get_data_loader("test"),
tasks=silo.processor.tasks,
device=device,
)
result = evaluator_test.eval(model, return_preds_and_labels=True)
os.makedirs(os.path.dirname(BESTMODEL + "/classification_report.txt"), exist_ok=True)
with open(BESTMODEL + "/classification_report.txt", "a+") as file:
file.write("Evaluation on withheld split for numfold no. {} \n".format(num_fold))
file.write(result[0]["report"])
file.write("\n\n")
file.close()
evaluator_test.log_results(result, "Test", steps=len(silo.get_data_loader("test")), num_fold=num_fold)
allresults.append(result)
all_preds.extend(result[0].get("preds"))
all_labels.extend(result[0].get("labels"))
# keep track of best fold
f1_macro = result[0]["f1_macro"]
if f1_macro > bestf1_macro:
bestf1_macro = f1_macro
bestfold = num_fold
# Save the per-fold results to json for a separate, more detailed analysis
with open("../data/predictions/covid-classification-xval.results.json", "wt") as fp:
json.dump(allresults, fp, cls=NumpyArrayEncoder)
# calculate overall f1 score across all folds
xval_f1_macro = f1_score(all_labels, all_preds, average="macro")
ml_logger.log_metrics({"f1 macro across all folds": xval_f1_macro}, step=None)
# test performance
evaluator_origtest = Evaluator(
data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device
)
# restore model from the best fold
lm_name = model.language_model.name
save_dir = Path(f"saved_models/covid-classification-v1-{bestfold}")
model = AdaptiveModel.load(save_dir, device, lm_name=lm_name)
model.connect_heads_with_processor(data_silo.processor.tasks, require_labels=True)
result = evaluator_origtest.eval(model)
ml_logger.log_metrics({"f1 macro on final test set": result[0]["f1_macro"]}, step=None)
with open(BESTMODEL + "/classification_report.txt", "a+") as file:
file.write("Final result of the best model \n")
file.write(result[0]["report"])
file.write("\n\n")
file.close()
ml_logger.log_artifacts(BESTMODEL + "/")
# save model for later use
processor.save(BESTMODEL)
model.save(BESTMODEL)
return model
def doc_classification_crossvalidation():
##########################
########## Logging
##########################
logger = logging.getLogger(__name__)
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO)
# reduce verbosity from transformers library
logging.getLogger('transformers').setLevel(logging.WARNING)
# ml_logger = MLFlowLogger(tracking_uri="https://public-mlflow.deepset.ai/")
# for local logging instead:
ml_logger = MLFlowLogger(tracking_uri="logs")
# ml_logger.init_experiment(experiment_name="Public_FARM", run_name="DocClassification_ES_f1_1")
##########################
########## Settings
##########################
xval_folds = 5
xval_stratification = True
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
n_epochs = 20
batch_size = 32
evaluate_every = 100
dev_split = 0.1
# For xval the dev_stratification parameter must not be None: with None, the devset cannot be created
# using the default method of only splitting by the available chunks as initial train set for each fold
# is just a single chunk!
dev_stratification = True
lang_model = "bert-base-german-cased"
do_lower_case = False
use_amp = None
# 1.Create a tokenizer
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
# The evaluation on the dev-set can be done with one of the predefined metrics or with a
# metric defined as a function from (preds, labels) to a dict that contains all the actual
# metrics values. The function must get registered under a string name and the string name must
# be used.
# For xval, we also store the actual predictions and labels in each result so we can
# calculate overall metrics over all folds later
def mymetrics(preds, labels):
acc = simple_accuracy(preds, labels).get("acc")
f1other = f1_score(y_true=labels, y_pred=preds, pos_label="OTHER")
f1offense = f1_score(y_true=labels, y_pred=preds, pos_label="OFFENSE")
f1macro = f1_score(y_true=labels, y_pred=preds, average="macro")
f1micro = f1_score(y_true=labels, y_pred=preds, average="micro")
mcc = matthews_corrcoef(labels, preds)
return {
"acc": acc,
"f1_other": f1other,
"f1_offense": f1offense,
"f1_macro": f1macro,
"f1_micro": f1micro,
"mcc": mcc
}
register_metrics('mymetrics', mymetrics)
metric = 'mymetrics'
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# Here we load GermEval 2018 Data automaticaly if it is not available.
# GermEval 2018 only has train.tsv and test.tsv dataset - no dev.tsv
# The processor wants to know the possible labels ...
label_list = ["OTHER", "OFFENSE"]
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=64,
data_dir=Path("../data/germeval18"),
label_list=label_list,
metric=metric,
dev_split=dev_split,
dev_stratification=dev_stratification,
label_column_name="coarse_label")
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(processor=processor, batch_size=batch_size)
# Load one silo for each fold in our cross-validation
silos = DataSiloForCrossVal.make(data_silo,
sets=["train", "dev"],
n_splits=xval_folds,
stratification=xval_stratification)
# the following steps should be run for each of the folds of the cross validation, so we put them
# into a function
def train_on_split(silo_to_use, n_fold, save_dir):
logger.info(
f"############ Crossvalidation: Fold {n_fold} of {xval_folds} ############"
)
logger.info(
f"Fold training samples: {len(silo_to_use.data['train'])}")
logger.info(f"Fold dev samples: {len(silo_to_use.data['dev'])}")
logger.info(
f"Fold testing samples: {len(silo_to_use.data['test'])}")
logger.info(
"Total number of samples: "
f"{len(silo_to_use.data['train'])+len(silo_to_use.data['dev'])+len(silo_to_use.data['test'])}"
)
# Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(lang_model)
# b) and a prediction head on top that is suited for our task => Text classification
prediction_head = TextClassificationHead(
class_weights=data_silo.calculate_class_weights(
task_name="text_classification"),
num_labels=len(label_list))
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.2,
lm_output_types=["per_sequence"],
device=device)
# Create an optimizer
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=0.5e-5,
device=device,
n_batches=len(silo_to_use.loaders["train"]),
n_epochs=n_epochs,
use_amp=use_amp)
# Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it from time to time
# Also create an EarlyStopping instance and pass it on to the trainer
# An early stopping instance can be used to save the model that performs best on the dev set
# according to some metric and stop training when no improvement is happening for some iterations.
# NOTE: Using a different save directory for each fold, allows us afterwards to use the
# nfolds best models in an ensemble!
save_dir = Path(str(save_dir) + f"-{n_fold}")
earlystopping = EarlyStopping(
metric="f1_offense",
mode=
"max", # use the metric from our own metrics function instead of loss
save_dir=save_dir, # where to save the best model
patience=
5 # number of evaluations to wait for improvement before terminating the training
)
trainer = Trainer(model=model,
optimizer=optimizer,
data_silo=silo_to_use,
epochs=n_epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=evaluate_every,
device=device,
early_stopping=earlystopping,
evaluator_test=False)
# train it
trainer.train()
return trainer.model
# for each fold, run the whole training, earlystopping to get a model, then evaluate the model
# on the test set of each fold
# remember all individual evaluation results
allresults = []
bestfold = None
bestf1_offense = -1
save_dir = Path("saved_models/bert-german-doc-tutorial-es")
for num_fold, silo in enumerate(silos):
mlflow.start_run(run_name=f"fold-{num_fold + 1}-of-{len(silos)}",
nested=True)
model = train_on_split(silo, num_fold, save_dir)
# do eval on test set here (and not in Trainer),
# so that we can easily store the actual preds and labels for a "global" eval across all folds.
evaluator_test = Evaluator(data_loader=silo.get_data_loader("test"),
tasks=silo.processor.tasks,
device=device)
result = evaluator_test.eval(model, return_preds_and_labels=True)
evaluator_test.log_results(result,
"Test",
steps=len(silo.get_data_loader("test")),
num_fold=num_fold)
allresults.append(result)
# keep track of best fold
f1_offense = result[0]["f1_offense"]
if f1_offense > bestf1_offense:
bestf1_offense = f1_offense
bestfold = num_fold
mlflow.end_run()
# emtpy cache to avoid memory leak and cuda OOM across multiple folds
model.cpu()
torch.cuda.empty_cache()
# Save the per-fold results to json for a separate, more detailed analysis
with open("doc_classification_xval.results.json", "wt") as fp:
json.dump(allresults, fp)
# log the best fold metric and fold
logger.info(f"Best fold f1_offense: {bestf1_offense} in fold {bestfold}")
# calculate overall metrics across all folds: we only have one head so we do this only for the first head
# information in each of the per-fold results
# First create a dict where for each metric, we have a list of values from each fold
xval_metric_lists_head0 = defaultdict(list)
for results in allresults:
head0results = results[0]
for name in head0results.keys():
if name not in ["preds", "labels"] and not name.startswith("_") and \
isinstance(head0results[name], numbers.Number):
xval_metric_lists_head0[name].append(head0results[name])
# Now calculate the mean and stdev for each metric, also copy over the task name
xval_metric = {}
xval_metric["task_name"] = allresults[0][0].get("task_name",
"UNKNOWN TASKNAME")
for name in xval_metric_lists_head0.keys():
values = xval_metric_lists_head0[name]
vmean = statistics.mean(values)
vstdev = statistics.stdev(values)
xval_metric[name + "_mean"] = vmean
xval_metric[name + "_stdev"] = vstdev
logger.info(
f"XVAL Accuracy: mean {xval_metric['acc_mean']} stdev {xval_metric['acc_stdev']}"
)
logger.info(
f"XVAL F1 MICRO: mean {xval_metric['f1_micro_mean']} stdev {xval_metric['f1_micro_stdev']}"
)
logger.info(
f"XVAL F1 MACRO: mean {xval_metric['f1_macro_mean']} stdev {xval_metric['f1_macro_stdev']}"
)
logger.info(
f"XVAL F1 OFFENSE: mean {xval_metric['f1_offense_mean']} stdev {xval_metric['f1_offense_stdev']}"
)
logger.info(
f"XVAL F1 OTHER: mean {xval_metric['f1_other_mean']} stdev {xval_metric['f1_other_stdev']}"
)
logger.info(
f"XVAL MCC: mean {xval_metric['mcc_mean']} stdev {xval_metric['mcc_stdev']}"
)
# -----------------------------------------------------
# Just for illustration, use the best model from the best xval val for evaluation on
# the original (still unseen) test set.
logger.info(
"###### Final Eval on hold out test set using best model #####")
evaluator_origtest = Evaluator(
data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# restore model from the best fold
lm_name = model.language_model.name
save_dir = Path(f"saved_models/bert-german-doc-tutorial-es-{bestfold}")
model = AdaptiveModel.load(save_dir, device, lm_name=lm_name)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
result = evaluator_origtest.eval(model)
logger.info(f"TEST Accuracy: {result[0]['acc']}")
logger.info(f"TEST F1 MICRO: {result[0]['f1_micro']}")
logger.info(f"TEST F1 MACRO: {result[0]['f1_macro']}")
logger.info(f"TEST F1 OFFENSE: {result[0]['f1_offense']}")
logger.info(f"TEST F1 OTHER: {result[0]['f1_other']}")
logger.info(f"TEST MCC: {result[0]['mcc']}")
def text_pair_classification():
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO)
ml_logger = MLFlowLogger(tracking_uri="https://public-mlflow.deepset.ai/")
ml_logger.init_experiment(experiment_name="Public_FARM",
run_name="Run_text_pair_classification")
##########################
########## Settings ######
##########################
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
n_epochs = 2
batch_size = 32
evaluate_every = 500
lang_model = "bert-base-cased"
label_list = ["0", "1"]
# 1.Create a tokenizer
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=False)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# We do not have a sample dataset for regression yet, add your own dataset to run the example
processor = TextPairClassificationProcessor(
tokenizer=tokenizer,
label_list=label_list,
metric="acc",
label_column_name="label",
max_seq_len=64,
train_filename="train.tsv",
test_filename="test.tsv",
dev_filename="dev.tsv",
#dev_split = 0.5,
data_dir=Path("/mnt/data/datasets/patents/patent_matching"),
tasks={"text_classification"},
delimiter="\t")
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = StreamingDataSilo(processor=processor, batch_size=batch_size)
#Alte Version vor StreamingDataSilo
#data_silo = DataSilo(
# processor=processor,
# batch_size=batch_size, max_processes=4)
# 4. Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(lang_model)
# b) and a prediction head on top that is suited for our task
prediction_head = TextClassificationHead(
num_labels=len(label_list),
class_weights=[0.56, 0.44] # Todo: Reihenfolge checken
)
model = AdaptiveModel.load("saved_models/text_pair_classification_model",
device=device)
# 5. Create an optimizer
model, optimizer, lr_schedule = initialize_optimizer(model=model,
learning_rate=5e-6,
device=device,
n_batches=1466,
n_epochs=n_epochs)
# An early stopping instance can be used to save the model that performs best on the dev set
# according to some metric and stop training when no improvement is happening for some iterations.
earlystopping = EarlyStopping(
#metric="f1_weighted", mode="max", # use f1_macro from the dev evaluator of the trainer
metric="loss",
mode="min", # use loss from the dev evaluator of the trainer
save_dir=Path("saved_models/text_pair_classification_model_Tuned"
), # where to save the best model
patience=
2 # number of evaluations to wait for improvement before terminating the training
)
# 6. Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it from time to time
trainer = Trainer(model=model,
optimizer=optimizer,
data_silo=data_silo,
epochs=n_epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=evaluate_every,
device=device,
early_stopping=earlystopping)
# 7. Let it grow
trainer.train()
# 8. Hooray! You have a model. Store it:
save_dir = Path("saved_models/text_pair_classification_model")
model.save(save_dir)
processor.save(save_dir)
# 9. Load it & harvest your fruits (Inference)
# Add your own text adapted to the dataset you provide
basic_texts = [
{
"text":
"<claim-text>The method of claim 10, wherein the indium metal layer is 10 nm to 100 µm thick.</claim-text>",
"text_b":
"<p id="
"p0001"
" num="
"0001"
">The present invention is directed to metal plating compositions and methods. More specifically, the present invention is directed to metal plating compositions and methods which provide improved leveling and throwing power.</p <p id="
"p0039"
" num="
"0039"
">One or more conventional surfactants may be used. Typically, surfactants include, but are not limited to, nonionic surfactants such as alkyl phenoxy polyethoxyethanols. Other suitable surfactants containing multiple oxyethylene groups also may be used. Such surfactants include compounds of polyoxyethylene polymers having from as many as 20 to 150 repeating units. Such compounds also may perform as suppressors. Also included in the class of polymers are both block and random copolymers of polyoxyethylene (EO) and polyoxypropylene (PO). Surfactants may be added in conventional amounts, such as from 0.05 g/L to 20 g/L or such as from 0.5 g/L to 5 g/L.</p <p id="
"p0040"
" num="
"0040"
">Conventional levelers include, but are not limited to, one or more of alkylated polyalkyleneimines and organic sulfo sulfonates. Examples of such compounds include, 4-mercaptopyridine, 2-mercaptothiazoline, ethylene thiourea, thiourea, 1-(2-hydroxyethyl)-2-imidazolidinethion (HIT) and alkylated polyalkyleneimines. Such levelers are included in conventional amounts. Typically, such levelers are included in amounts of 1ppb to 1 g/L, or such as from 10ppb to 500ppm.</p <p id="
"p0042"
" num="
"0042"
">Alkali metal salts which may be included in the plating compositions include, but are not limited to, sodium and potassium salts of halogens, such as chloride, fluoride and bromide. Typically chloride is used. Such alkali metal salts are used in conventional amounts.</p <p id="
"p0053"
" num="
"0053"
">The metal plating compositions may be used to plate a metal or metal alloy on a substrate by any method known in the art and literature. Typically, the metal or metal alloy is electroplated using conventional electroplating processes with conventional apparatus. A soluble or insoluble anode may be used with the electroplating compositions.</p <p id="
"p0022"
" num="
"0022"
">One or more sources of metal ions are included in metal plating compositions to plate metals. The one or more sources of metal ions provide metal ions which include, but are not limited to, copper, tin, nickel, gold, silver, palladium, platinum and indium. Alloys include, but are not limited to, binary and ternary alloys of the foregoing metals. Typically, metals chosen from copper, tin, nickel, gold, silver or indium are plated with the metal plating compositions. More typically, metals chosen from copper, tin, silver or indium are plated. Most typically, copper is plated.</p <p id="
"p0030"
" num="
"0030"
">Indium salts which may be used include, but are not limited to, one or more of indium salts of alkane sulfonic acids and aromatic sulfonic acids, such as methanesulfonic acid, ethanesulfonic acid, butane sulfonic acid, benzenesulfonic acid and toluenesulfonic acid, salts of sulfamic acid, sulfate salts, chloride and bromide salts of indium, nitrate salts, hydroxide salts, indium oxides, fluoroborate salts, indium salts of carboxylic acids, such as citric acid, acetoacetic acid, glyoxylic acid, pyruvic acid, glycolic acid, malonic acid, hydroxamic acid, iminodiacetic acid, salicylic acid, glyceric acid, succinic acid, malic acid, tartaric acid, hydroxybutyric acid, indium salts of amino acids, such as arginine, aspartic acid, asparagine, glutamic acid, glycine, glutamine, leucine, lysine, threonine, isoleucine, and valine.</p"
},
{
"text":
"<claim-text>A toner comprising: <claim-text>toner base particles; and</claim-text> <claim-text>an external additive,</claim-text> <claim-text>the toner base particles each comprising a binder resin and a colorant,</claim-text> <claim-text>wherein the external additive comprises coalesced particles,</claim-text> <claim-text>wherein the coalesced particles are each a non-spherical secondary particle in which primary particles are coalesced together, and</claim-text> <claim-text>wherein an index of a particle size distribution of the coalesced particles is expressed by the following Formula (1): <maths id="
"math0004"
" num="
"(formula (1)"
"><math display="
"block"
"><mfrac><msub><mi>Db</mi><mn>50</mn></msub><msub><mi>Db</mi><mn>10</mn></msub></mfrac><mo>≦</mo><mn>1.20</mn></math><img id="
"ib0008"
" file="
"imgb0008.tif"
" wi="
"93"
" he="
"21"
" img-content="
"math"
" img-format="
"tif"
"/></maths><br/> where, in a distribution diagram in which particle diameters in nm of the coalesced particles are on a horizontal axis and cumulative percentages in % by number of the coalesced particles are on a vertical axis and in which the coalesced particles are accumulated from the coalesced particles having smaller particle diameters to the coalesced particles having larger particle diameters, Db<sub>50</sub> denotes a particle diameter of the coalesced particle at which the cumulative percentage is 50% by number, and Db<sub>10</sub> denotes a particle diameter of the coalesced particle at which the cumulative percentage is 10% by number.</claim-text></claim-text>",
"text_b":
"<p id="
"p0177"
" num="
"0177"
">For a similar reason, it is preferred that the electroconductive fine powder has a volume-average particle size of 0.5 - 5 µm, more preferably 0.8 - 5 µm, further preferably 1.1 - 5 µm and has a particle size distribution such that particles of 0.5 µm or smaller occupy at most 70 % by volume and particles of 5.0 µm or larger occupy at most 5 % by number.</p <p id="
"p0189"
" num="
"0189"
">The volume-average particle size and particle size distribution of the electroconductive fine powder described herein are based on values measured in the following manner. A laser diffraction-type particle size distribution measurement apparatus ("
"Model LS-230"
", available from Coulter Electronics Inc.) is equipped with a liquid module, and the measurement is performed in a particle size range of 0.04 - 2000 µm to obtain a volume-basis particle size distribution. For the measurement, a minor amount of surfactant is added to 10 cc of pure water and 10 mg of a sample electroconductive fine powder is added thereto, followed by 10 min. of dispersion by means of an ultrasonic disperser (ultrasonic homogenizer) to obtain a sample dispersion liquid, which is subjected to a single time of measurement for 90 sec.</p <p id="
"p0191"
" num="
"0191"
">In the case where the electroconductive fine powder is composed of agglomerate particles, the particle size of the electroconductive fine powder is determined as the particle size of the agglomerate. The electroconductive fine powder in the form of agglomerated secondary particles can be used as well as that in the form of primary particles. Regardless of its agglomerated form, the electroconductive fine powder can exhibit its desired function of charging promotion by presence in the form of the agglomerate in the charging section at the contact position<!-- EPO <DP n="
"85"
"> --> between the charging member and the image-bearing member or in a region in proximity thereto.</p"
},
]
model = Inferencer.load(save_dir)
result = model.inference_from_dicts(dicts=basic_texts)
print(result)