##########################
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
n_epochs = 5
batch_size = 32
evaluate_every = 30
lang_model = "bert-base-cased"
# 1.Create a tokenizer
tokenizer = Tokenizer.from_pretrained(pretrained_model_name_or_path=lang_model,
do_lower_case=False)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# We do not have a sample dataset for regression yet, add your own dataset to run the example
processor = RegressionProcessor(tokenizer=tokenizer,
max_seq_len=128,
data_dir="../data/<YOUR-DATASET>",
label_column_name="label")
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(processor=processor, batch_size=batch_size)
# 4. Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(lang_model)
# b) and a prediction head on top that is suited for our task => Text regression
prediction_head = RegressionHead(layer_dims=[768, 1])
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
lm_output_types=["per_sequence_continuous"],
def test_doc_regression(caplog):
caplog.set_level(logging.CRITICAL)
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=False)
n_epochs = 1
batch_size = 1
evaluate_every = 2
lang_model = "bert-base-cased"
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=False)
processor = RegressionProcessor(tokenizer=tokenizer,
max_seq_len=8,
data_dir=Path("samples/doc_regr"),
train_filename="train-sample.tsv",
dev_filename="test-sample.tsv",
test_filename=None,
label_column_name="label")
data_silo = DataSilo(processor=processor, batch_size=batch_size)
language_model = LanguageModel.load(lang_model)
prediction_head = RegressionHead()
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
lm_output_types=["per_sequence_continuous"],
device=device)
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=2e-5,
#optimizer_opts={'name': 'AdamW', 'lr': 2E-05},
n_batches=len(data_silo.loaders["train"]),
n_epochs=1,
device=device,
schedule_opts={
'name': 'CosineWarmup',
'warmup_proportion': 0.1
})
trainer = Trainer(model=model,
optimizer=optimizer,
data_silo=data_silo,
epochs=n_epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=evaluate_every,
device=device)
trainer.train()
save_dir = Path("testsave/doc_regr")
model.save(save_dir)
processor.save(save_dir)
basic_texts = [
{
"text":
"The dress is just fabulous and it totally fits my size. The fabric is of great quality and the seams are really well hidden. I am super happy with this purchase and I am looking forward to trying some more from the same brand."
},
{
"text":
"it just did not fit right. The top is very thin showing everything."
},
]
model = Inferencer.load(save_dir)
result = model.inference_from_dicts(dicts=basic_texts)
assert isinstance(result[0]["predictions"][0]["pred"], np.float32)
def test_doc_regression(caplog):
caplog.set_level(logging.CRITICAL)
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=False)
n_epochs = 1
batch_size = 8
evaluate_every = 30
lang_model = "bert-base-cased"
tokenizer = BertTokenizer.from_pretrained(
pretrained_model_name_or_path=lang_model, do_lower_case=False)
processor = RegressionProcessor(tokenizer=tokenizer,
max_seq_len=128,
data_dir="samples/doc_regr",
columns=["text", "label"],
label_list=[],
metrics=["mse"],
train_filename="train-sample.tsv",
test_filename=None)
data_silo = DataSilo(processor=processor, batch_size=batch_size)
language_model = Bert.load(lang_model)
prediction_head = RegressionHead(layer_dims=[768, 1])
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
lm_output_types=["per_sequence_continuous"],
device=device)
optimizer, warmup_linear = initialize_optimizer(
model=model,
learning_rate=2e-5,
warmup_proportion=0.1,
n_batches=len(data_silo.loaders["train"]),
n_epochs=1)
trainer = Trainer(optimizer=optimizer,
data_silo=data_silo,
epochs=n_epochs,
n_gpu=n_gpu,
warmup_linear=warmup_linear,
evaluate_every=evaluate_every,
device=device)
model = trainer.train(model)
save_dir = "testsave/doc_regr"
model.save(save_dir)
processor.save(save_dir)
basic_texts = [
{
"text":
"The dress is just fabulous and it totally fits my size. The fabric is of great quality and the seams are really well hidden. I am super happy with this purchase and I am looking forward to trying some more from the same brand."
},
{
"text":
"it just did not fit right. The top is very thin showing everything."
},
]
model = Inferencer.load(save_dir)
result = model.run_inference(dicts=basic_texts)
print(result)
assert abs(float(result[0]["predictions"][0]["pred"]) -
4.2121115) <= 0.0001
assert abs(float(result[0]["predictions"][1]["pred"]) -
4.1987348) <= 0.0001
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
n_epochs = 5
batch_size = 32
evaluate_every = 30
lang_model = "bert-base-cased"
# 1.Create a tokenizer
tokenizer = BertTokenizer.from_pretrained(
pretrained_model_name_or_path=lang_model, do_lower_case=False)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# We do not have a sample dataset for regression yet, add your own dataset to run the example
processor = RegressionProcessor(
tokenizer=tokenizer,
max_seq_len=128,
data_dir="../data/<YOUR-DATASET>",
)
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(processor=processor, batch_size=batch_size)
# 4. Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = Bert.load(lang_model)
# b) and a prediction head on top that is suited for our task => Text regression
prediction_head = RegressionHead(layer_dims=[768, 1])
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
n_epochs = 5
batch_size = 32
evaluate_every = 30
lang_model = "bert-base-cased"
# 1.Create a tokenizer
tokenizer = BertTokenizer.from_pretrained(
pretrained_model_name_or_path=lang_model, do_lower_case=False)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# We do not have a sample dataset for regression yet, add your own dataset to run the example
processor = RegressionProcessor(tokenizer=tokenizer,
max_seq_len=128,
data_dir="",
columns=["text", "label"],
label_list=[],
metrics=["mse"])
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(processor=processor, batch_size=batch_size)
# 4. Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = Bert.load(lang_model)
# b) and a prediction head on top that is suited for our task => Text regression
prediction_head = RegressionHead(layer_dims=[768, 1])
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
def doc_regression():
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO)
ml_logger = MLFlowLogger(tracking_uri="https://public-mlflow.deepset.ai/")
ml_logger.init_experiment(experiment_name="Public_FARM", run_name="Run_doc_regression")
##########################
########## Settings
##########################
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
n_epochs = 5
batch_size = 32
evaluate_every = 30
lang_model = "bert-base-cased"
do_lower_case = False
# 1.Create a tokenizer
tokenizer = Tokenizer.load(
pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# We do not have a sample dataset for regression yet, add your own dataset to run the example
processor = RegressionProcessor(tokenizer=tokenizer,
max_seq_len=128,
data_dir=Path("../data/<YOUR-DATASET>"),
label_column_name="label"
)
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(
processor=processor,
batch_size=batch_size)
# 4. Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(lang_model)
# b) and a prediction head on top that is suited for our task => Text regression
prediction_head = RegressionHead()
model = AdaptiveModel(
language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
lm_output_types=["per_sequence_continuous"],
device=device)
# 5. Create an optimizer
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=2e-5,
device=device,
n_batches=len(data_silo.loaders["train"]),
n_epochs=n_epochs)
# 6. Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it from time to time
trainer = Trainer(
model=model,
optimizer=optimizer,
data_silo=data_silo,
epochs=n_epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=evaluate_every,
device=device)
# 7. Let it grow
trainer.train()
# 8. Hooray! You have a model. Store it:
save_dir = Path("saved_models/bert-doc-regression-tutorial")
model.save(save_dir)
processor.save(save_dir)
# 9. Load it & harvest your fruits (Inference)
# Add your own text adapted to the dataset you provide
basic_texts = [
{"text": ""},
{"text": ""},
]
model = Inferencer.load(save_dir)
result = model.inference_from_dicts(dicts=basic_texts)
print(result)
