def from_text_vision_configs(cls, text_config: PretrainedConfig,
vision_config: PretrainedConfig, **kwargs):
r"""
Instantiate a :class:`HybridCLIPConfig` (or a derived class) from text model configuration and
vision model configuration.
Returns:
:class:`HybridCLIPConfig`: An instance of a configuration object
"""
return cls(text_config=text_config.to_dict(),
vision_config=vision_config.to_dict(),
**kwargs)
def from_layout_lm_bert_configs(cls, layout_lm_config: PretrainedConfig,
bert_config: PretrainedConfig,
**kwargs) -> PretrainedConfig:
# logger.info(
# "Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config"
# )
# decoder_config.is_decoder = True
# decoder_config.add_cross_attention = True
return cls(layout_lm=layout_lm_config.to_dict(),
bert=bert_config.to_dict(),
**kwargs)
def from_encoder_decoder_configs(cls, encoder_config: PretrainedConfig,
decoder_config: PretrainedConfig,
**kwargs) -> PretrainedConfig:
r"""
Instantiate a [`EncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model configuration and
decoder model configuration.
Returns:
[`EncoderDecoderConfig`]: An instance of a configuration object
"""
logger.info(
"Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config"
)
decoder_config.is_decoder = True
decoder_config.add_cross_attention = True
return cls(encoder=encoder_config.to_dict(),
decoder=decoder_config.to_dict(),
**kwargs)
def __init__(
self,
embedding,
rnn_hidden_size,
rnn_num_layers,
num_attn_heads,
num_encoder_layers,
num_decoder_layers,
ffn_dim,
dropout,
):
super().__init__(PretrainedConfig())
self.embedding = embedding
self.word_embed_dim = embedding.embedding_dim
# model config
self.model_config = {
"rnn_hidden_size": rnn_hidden_size,
"rnn_num_layers": rnn_num_layers,
"num_attn_heads": num_attn_heads,
"num_encoder_layers": num_encoder_layers,
"num_decoder_layers": num_decoder_layers,
"ffn_dim": ffn_dim,
"dropout": dropout,
}
self.word_encoder = WordEncoder(
hidden_size=rnn_hidden_size,
embedding_size=self.word_embed_dim,
embedding=embedding,
num_layers=rnn_num_layers,
dropout=dropout,
)
self.sentence_embed_dim = rnn_hidden_size
self.config = BartConfig(
d_model=self.sentence_embed_dim,
encoder_attention_heads=num_attn_heads,
decoder_attention_heads=num_attn_heads,
attention_dropout=0.0,
dropout=dropout,
activation_dropout=0.0,
encoder_ffn_dim=ffn_dim,
decoder_ffn_dim=ffn_dim,
)
self.encoder_layers = nn.ModuleList(
[EncoderLayer(self.config) for _ in range(num_encoder_layers)])
self.decoder_layers = nn.ModuleList(
[DecoderLayer(self.config) for _ in range(num_decoder_layers)])
self.pointer_head = PointerHead(self.sentence_embed_dim)
self.apply(self._init_weights)
def test_config_common_kwargs_is_complete(self):
base_config = PretrainedConfig()
missing_keys = [key for key in base_config.__dict__ if key not in config_common_kwargs]
# If this part of the test fails, you have arguments to addin config_common_kwargs above.
self.assertListEqual(missing_keys, ["is_encoder_decoder", "_name_or_path", "transformers_version"])
keys_with_defaults = [key for key, value in config_common_kwargs.items() if value == getattr(base_config, key)]
if len(keys_with_defaults) > 0:
raise ValueError(
"The following keys are set with the default values in `test_configuration_common.config_common_kwargs` "
f"pick another value for them: {', '.join(keys_with_defaults)}."
)
def main():
# processors need to be updated
processors = {
'agnews-processor': AgNewsDataProcessor,
'thcnews-processor': THCNewsDataProcessor
}
config: Config = Config.instance()
if not config.do_train and not config.do_eval and not config.do_predict:
raise ValueError(
"At least one of `do_train`, `do_eval` or `do_predict' must be True.")
bert_config = PretrainedConfig.from_pretrained(config.pretrained_model_name)
# 根据不同的任务，处理不同的数据集
task_name = config.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = AutoTokenizer.from_pretrained(config.pretrained_model_name)
train_examples = None
num_train_steps = None
num_warmup_steps = None
if config.do_train:
train_examples: List[InputExample] = processor.get_train_examples(config.data_dir)
# TODO: complete the code
train_dataset_loader = None
num_train_steps = int(
len(train_examples) / config.train_batch_size * config.epochs
)
num_warmup_steps = int(num_train_steps * config.warmup_proportion)
model = create_model(config=config)
training_arguments = TrainingArguments(
output_dir=config.output_dir,
overwrite_output_dir=True,
)
trainer = SequenceClassificationTrainer(
model=model,
)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPUs
if config.do_train:
train_file = os.path.join(config.output_dir, "train.tf_record")
file_based_convert_examples_to_features(
train_examples, label_list, config.max_seq_length, tokenizer, train_file)
# tf.logging.info("***** Running training *****")
# tf.logging.info(" Num examples = %d", len(train_examples))
# tf.logging.info(" Batch size = %d", config.train_batch_size)
# tf.logging.info(" Num steps = %d", num_train_steps)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=config.max_seq_length,
is_training=True,
drop_remainder=True)
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
def __init__(self, config: DIETClassifierConfig):
"""
Create DIETClassifier model
:param config: config for model
"""
if path.exists(config.model):
try:
json_config = json.load(
open(f"{config.model}/config.json", "r"))
except Exception as ex:
raise RuntimeError(
f"Cannot load configuration fil from {config.model} by error: {ex}"
)
try:
checkpoint = torch.load(f"{config.model}/pytorch_model.bin")
except Exception as ex:
raise RuntimeError(
f"Cannot load model from {config.model} by error: {ex}")
pretrained_model = None
config = PretrainedConfig.from_dict(json_config)
else:
pretrained_model = BertForTokenClassification.from_pretrained(
config.model)
checkpoint = None
if config.intents is None or config.entities is None:
raise ValueError(
f"Using pretrained from transformers should specific entities and intents"
)
pretrained_model.config.update({
"model": config.model,
"entities": config.entities,
"intents": config.intents
})
config = pretrained_model.config
super().__init__(config)
self.entities_list = ["O"] + config.entities
self.num_entities = len(self.entities_list)
self.intents_list = config.intents
self.num_intents = len(self.intents_list)
self.bert = BertModel(
config, add_pooling_layer=False
) if not pretrained_model else pretrained_model.bert
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.entities_classifier = nn.Linear(config.hidden_size,
self.num_entities)
self.intents_classifier = nn.Linear(config.hidden_size,
self.num_intents)
self.init_weights()
if not pretrained_model:
try:
self.load_state_dict(checkpoint)
except Exception as ex:
raise RuntimeError(
f"Cannot load state dict from checkpoint by error: {ex}")
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
print("********")
print(model_args)
print("********")
model2 = Wav2Vec2ForCTC.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
config = PretrainedConfig.from_json_file("wav2vec_config.json")
model = Wav2Vec2ForCTC(config)
processor = Wav2Vec2Processor.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
train_dataset = datasets.load_dataset(
data_args.dataset_name, data_args.dataset_config_name, split=data_args.train_split_name)
val_dataset = datasets.load_dataset(
data_args.dataset_name, data_args.dataset_config_name, split="validation")
wer_metric = datasets.load_metric("wer")
def map_to_array(batch):
speech_array, sampling_rate = sf.read(batch["file"])
batch["speech"] = speech_array
batch["sampling_rate"] = sampling_rate
return batch
train_dataset = train_dataset.map(map_to_array, remove_columns=["file"])
val_dataset = val_dataset.map(map_to_array, remove_columns=["file"])
def prepare_dataset(batch):
# check that all files have the correct sampling rate
assert (
len(set(batch["sampling_rate"])) == 1
), f"Make sure all inputs have the same sampling rate of {processor.feature_extractor.sampling_rate}."
batch["input_values"] = processor(
batch["speech"], sampling_rate=batch["sampling_rate"][0]).input_values
with processor.as_target_processor():
batch["labels"] = processor(batch["text"]).input_ids
return batch
train_dataset = train_dataset.map(
prepare_dataset,
batch_size=training_args.per_device_train_batch_size,
batched=True,
num_proc=data_args.preprocessing_num_workers,
)
val_dataset = val_dataset.map(
prepare_dataset,
batch_size=training_args.per_device_train_batch_size,
batched=True,
num_proc=data_args.preprocessing_num_workers,
)
data_collator = DataCollatorCTCWithPadding(
processor=processor, padding=True)
def compute_metrics(pred):
pred_logits = pred.predictions
pred_ids = np.argmax(pred_logits, axis=-1)
pred.label_ids[pred.label_ids == -100] = 0
pred_str = processor.batch_decode(pred_ids)
# we do not want to group tokens when computing the metrics
label_str = processor.batch_decode(pred.label_ids, group_tokens=False)
wer = wer_metric.compute(predictions=pred_str, references=label_str)
return {"wer": wer}
if model_args.freeze_feature_extractor:
model.freeze_feature_extractor()
trainer = CTCTrainer(
model=model,
data_collator=data_collator,
args=training_args,
compute_metrics=compute_metrics,
train_dataset=train_dataset,
eval_dataset=val_dataset,
tokenizer=processor.feature_extractor,
)
trainer.train()