def save(self, save_dir):
"""
Saves
:param save_dir: path to save to
:type save_dir: str
:return: into the void
"""
create_folder(save_dir)
self.language_model.save(save_dir)
for i, ph in enumerate(self.prediction_heads):
ph.save(save_dir, i)
def save(self, save_dir):
create_folder(save_dir)
config = {}
config["tokenizer"] = self.tokenizer.__class__.__name__
self.tokenizer.save_vocabulary(save_dir)
# TODO make this generic to other tokenizers. We will probably want an own abstract Tokenizer
config["lower_case"] = self.tokenizer.basic_tokenizer.do_lower_case
config["max_seq_len"] = self.max_seq_len
config["processor"] = self.__class__.__name__
output_config_file = os.path.join(save_dir, "processor_config.json")
with open(output_config_file, "w") as file:
json.dump(config, file)
def save(self, save_dir):
"""
Saves the language model and prediction heads. This will generate a config file
and model weights for each.
:param save_dir: path to save to
:type save_dir: str
"""
create_folder(save_dir)
self.language_model.save(save_dir)
for i, ph in enumerate(self.prediction_heads):
ph.save(save_dir, i)
def save(self, save_dir):
"""
Saves the vocabulary to file and also creates a json file containing all the
information needed to load the same processor.
:param save_dir: Directory where the files are to be saved
:type save_dir: str
"""
create_folder(save_dir)
config = {}
config["tokenizer"] = self.tokenizer.__class__.__name__
self.tokenizer.save_vocabulary(save_dir)
# TODO make this generic to other tokenizers. We will probably want an own abstract Tokenizer
config["lower_case"] = self.tokenizer.basic_tokenizer.do_lower_case
config["max_seq_len"] = self.max_seq_len
config["processor"] = self.__class__.__name__
output_config_file = os.path.join(save_dir, "processor_config.json")
with open(output_config_file, "w") as file:
json.dump(config, file)