def test_load_with_mismatched_shapes(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() for model_class in self.all_model_classes: if model_class not in get_values(FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING): continue with self.subTest(msg=f"Testing {model_class}"): with tempfile.TemporaryDirectory() as tmp_dir: model = model_class(config) model.save_pretrained(tmp_dir) # Fails when we don't set ignore_mismatched_sizes=True with self.assertRaises(ValueError): new_model = FlaxAutoModelForSequenceClassification.from_pretrained(tmp_dir, num_labels=42) logger = logging.get_logger("transformers.modeling_flax_utils") with CaptureLogger(logger) as cl: new_model = FlaxAutoModelForSequenceClassification.from_pretrained( tmp_dir, num_labels=42, ignore_mismatched_sizes=True ) self.assertIn("the shapes did not match", cl.out) logits = new_model(**inputs_dict)["logits"] self.assertEqual(logits.shape[1], 42)
def test_warning_logs(self): transformers_logging.set_verbosity_debug() logger_ = transformers_logging.get_logger( "transformers.pipelines.base") alias = "text-classification" with CaptureLogger(logger_) as cm: PIPELINE_REGISTRY.register_pipeline(alias, {}) self.assertIn(f"{alias} is already registered", cm.out)
def __init__(self, args: Namespace): self.logger = logging.get_logger("transformers-cli/training") self.framework = "tf" if is_tf_available() else "torch" os.makedirs(args.output, exist_ok=True) self.output = args.output self.column_label = args.column_label self.column_text = args.column_text self.column_id = args.column_id self.logger.info("Loading {} pipeline for {}".format( args.task, args.model)) if args.task == "text_classification": self.pipeline = TextClassificationPipeline.from_pretrained( args.model) elif args.task == "token_classification": raise NotImplementedError elif args.task == "question_answering": raise NotImplementedError self.logger.info("Loading dataset from {}".format(args.train_data)) self.train_dataset = Processor.create_from_csv( args.train_data, column_label=args.column_label, column_text=args.column_text, column_id=args.column_id, skip_first_row=args.skip_first_row, ) self.valid_dataset = None if args.validation_data: self.logger.info("Loading validation dataset from {}".format( args.validation_data)) self.valid_dataset = Processor.create_from_csv( args.validation_data, column_label=args.column_label, column_text=args.column_text, column_id=args.column_id, skip_first_row=args.skip_first_row, ) self.validation_split = args.validation_split self.train_batch_size = args.train_batch_size self.valid_batch_size = args.valid_batch_size self.learning_rate = args.learning_rate self.adam_epsilon = args.adam_epsilon
def test_warning_logs(self): transformers_logging.set_verbosity_debug() logger_ = transformers_logging.get_logger( "transformers.pipelines.base") alias = "text-classification" # Get the original task, so we can restore it at the end. # (otherwise the subsequential tests in `TextClassificationPipelineTests` will fail) original_task, original_task_options = PIPELINE_REGISTRY.check_task( alias) try: with CaptureLogger(logger_) as cm: PIPELINE_REGISTRY.register_pipeline(alias, {}) self.assertIn(f"{alias} is already registered", cm.out) finally: # restore PIPELINE_REGISTRY.register_pipeline(alias, original_task)
import torch from dataclasses import dataclass from torch import nn from torch.nn import functional as F from transformers.utils import logging from .. import core as qc from ..core import utils as qu from ..core import forward as qf from ..core import output as qo from ..core.embed import pos_enc from ..core.ffnet import Classifier, FFNet, PoolBeg, PoolEnd, PoolProj from ..prep.config.xlnet import PreTrained log = logging.get_logger(__name__) class Model(PreTrained): def __init__(self, **kw): super().__init__(**kw) cfg = self.get_cfg(kw) self.embed = qc.Embed(cfg.s_vocab, cfg.d_model, **kw) self.mask = nn.Parameter(torch.FloatTensor(1, 1, cfg.d_model)) self.lays = qc.Stack([Layer(**kw) for _ in range(cfg.n_lays)]) self.drop = qc.Dropout(cfg.drop, **kw) def create_mask(self, qlen, mlen): mask = torch.ones([qlen, qlen]) up = torch.triu(mask, diagonal=1) pad = torch.zeros([qlen, mlen])
from transformers.utils import logging import torch from torch import nn from transformers.models.t5.modeling_t5 import T5LayerFF from transformer_vae.model_outputs import BaseVAE_Output logger = logging.get_logger() class LatentEncoderNTokens(nn.Module): ''' Converts N hidden tokens into N seperate latent codes. ''' def __init__(self, config): super().__init__() self.token_to_latent = nn.Linear(config.t5.d_model, config.latent_size) self.n_tokens = config.n_latent_tokens self.tanh = nn.Tanh() def forward(self, encoding) -> torch.Tensor: return self.tanh(self.token_to_latent(encoding))[:, :self.n_tokens, :] class LatentDecoderNTokens(nn.Module): ''' Take several latent tokens and convert them each full token hidden states. ''' def __init__(self, config): super().__init__() self.latent_size = config.latent_size
# limitations under the License. from inspect import signature from itertools import chain from pathlib import Path from typing import Iterable, List, Tuple, Union import numpy as np from packaging.version import Version, parse from transformers import PreTrainedModel, PreTrainedTokenizer, TensorType, TFPreTrainedModel, is_torch_available from transformers.file_utils import is_tf_available, is_torch_onnx_dict_inputs_support_available from transformers.onnx.config import OnnxConfig from transformers.utils import logging logger = logging.get_logger(__name__) # pylint: disable=invalid-name # This is the minimal required version to support some ONNX Runtime features ORT_QUANTIZE_MINIMUM_VERSION = parse("1.4.0") def check_onnxruntime_requirements(minimum_version: Version): """ Check onnxruntime is installed and if the installed version match is recent enough Raises: ImportError: If onnxruntime is not installed or too old version is found """ try: import onnxruntime
import os from pathlib import Path from typing import Dict, List import fire import torch from transformers import AutoModelForSeq2SeqLM, AutoTokenizer from transformers.utils.logging import get_logger logger = get_logger(__name__) def remove_prefix(text: str, prefix: str): if text.startswith(prefix): return text[len(prefix):] return text # or whatever def sanitize(sd): return {remove_prefix(k, "model."): v for k, v in sd.items()} def average_state_dicts(state_dicts: List[Dict[str, torch.Tensor]]): new_sd = {} for k in state_dicts[0].keys(): tensors = [sd[k] for sd in state_dicts] new_t = sum(tensors) / len(tensors) assert isinstance(new_t, torch.Tensor) new_sd[k] = new_t return new_sd