def __init__(self, pretrained_tokenizer: str, max_instances: int = None): super().__init__(max_instances=max_instances) self.tokenizer = PretrainedTransformerTokenizer(pretrained_tokenizer, max_length=2000) self.token_indexers = { "tokens": PretrainedTransformerIndexer(pretrained_tokenizer) }
def __init__(self) -> None: self.config: Config = Config().parse_args(known_only=True) bert_token_indexers = PretrainedTransformerIndexer( model_name=self.config.model_name) bert_tokenizer = PretrainedTransformerTokenizer( model_name=self.config.model_name) reader = TextClassificationJsonReader( token_indexers={"tokens": bert_token_indexers}, tokenizer=bert_tokenizer) train_instances = list(reader.read(self.config.train_file)) dev_instances = list(reader.read(self.config.dev_file)) test_instances = list(reader.read(self.config.test_file)) self.vocab: Vocabulary = Vocabulary.from_instances(train_instances) # 2. init the data loader self.train_data_loader = SimpleDataLoader(train_instances, self.config.batch_size, shuffle=True) self.dev_data_loader = SimpleDataLoader(dev_instances, self.config.batch_size, shuffle=False) self.train_data_loader.index_with(self.vocab) self.dev_data_loader.index_with(self.vocab) # 3. init the model self.model = self.init_model() self.trainer = self.init_trainer()
def __init__( self, sent1_col: str, sent2_col: str = None, label_col: str = 'label', bert_model: str = 'bert-base-uncased', max_sequence_length: int = 500, skip_label_indexing: bool = False, lower: bool = True, lazy: bool = False, ) -> None: super().__init__(lazy=lazy) self._sent1_col = sent1_col self._sent2_col = sent2_col self._label_col = label_col self._tokenizer = PretrainedTransformerTokenizer( bert_model, add_special_tokens=False, max_length=max_sequence_length ) # type: PretrainedTransformerTokenizer self._max_sequence_length = max_sequence_length self._skip_label_indexing = skip_label_indexing self._lower = lower self._token_indexers = { "tokens": PretrainedTransformerIndexer(model_name=bert_model) }
def __init__( self, model_name: str, vocab: Vocabulary, indexer: PretrainedTransformerIndexer = None, max_decoding_steps: int = 140, beam_size: int = 4, encoder: Seq2SeqEncoder = None, ): """ # Parameters model_name : `str`, required Name of the pre-trained BART model to use. Available options can be found in `transformers.models.bart.modeling_bart.BART_PRETRAINED_MODEL_ARCHIVE_MAP`. vocab : `Vocabulary`, required Vocabulary containing source and target vocabularies. indexer : `PretrainedTransformerIndexer`, optional (default = `None`) Indexer to be used for converting decoded sequences of ids to to sequences of tokens. max_decoding_steps : `int`, optional (default = `128`) Number of decoding steps during beam search. beam_size : `int`, optional (default = `5`) Number of beams to use in beam search. The default is from the BART paper. encoder : `Seq2SeqEncoder`, optional (default = `None`) Encoder to used in BART. By default, the original BART encoder is used. """ super().__init__(vocab) self.bart = BartForConditionalGeneration.from_pretrained(model_name) self._indexer = indexer or PretrainedTransformerIndexer( model_name, namespace="tokens") self._start_id = self.bart.config.bos_token_id # CLS self._decoder_start_id = self.bart.config.decoder_start_token_id or self._start_id self._end_id = self.bart.config.eos_token_id # SEP self._pad_id = self.bart.config.pad_token_id # PAD self._max_decoding_steps = max_decoding_steps self._beam_search = BeamSearch(self._end_id, max_steps=max_decoding_steps, beam_size=beam_size or 1) self._rouge = ROUGE( exclude_indices={self._start_id, self._pad_id, self._end_id}) self._bleu = BLEU( exclude_indices={self._start_id, self._pad_id, self._end_id}) # Replace bart encoder with given encoder. We need to extract the two embedding layers so that # we can use them in the encoder wrapper if encoder is not None: assert (encoder.get_input_dim() == encoder.get_output_dim() == self.bart.config.hidden_size) self.bart.model.encoder = _BartEncoderWrapper( encoder, self.bart.model.encoder.embed_tokens, self.bart.model.encoder.embed_positions, )
def __init__( self, hf_pretrained_tokenizer: str, cache_directory: str = 'data/cache', clue_prefix="cryptic crossword clue: ", ): super().__init__(cache_directory=cache_directory) self.clue_prefix = clue_prefix self.tokenizer = PretrainedTransformerTokenizer( hf_pretrained_tokenizer) self.token_indexers = { "tokens": PretrainedTransformerIndexer(hf_pretrained_tokenizer) }
def __init__( self, model_name: str, vocab: Vocabulary, beam_search: Lazy[BeamSearch] = Lazy(BeamSearch), indexer: PretrainedTransformerIndexer = None, encoder: Seq2SeqEncoder = None, **kwargs, ): super().__init__(vocab) self.bart = BartForConditionalGeneration.from_pretrained(model_name) self._indexer = indexer or PretrainedTransformerIndexer(model_name, namespace="tokens") self._start_id = self.bart.config.bos_token_id # CLS self._decoder_start_id = self.bart.config.decoder_start_token_id or self._start_id self._end_id = self.bart.config.eos_token_id # SEP self._pad_id = self.bart.config.pad_token_id # PAD # At prediction time, we'll use a beam search to find the best target sequence. # For backwards compatibility, check if beam_size or max_decoding_steps were passed in as # kwargs. If so, update the BeamSearch object before constructing and raise a DeprecationWarning deprecation_warning = ( "The parameter {} has been deprecated." " Provide this parameter as argument to beam_search instead." ) beam_search_extras = {} if "beam_size" in kwargs: beam_search_extras["beam_size"] = kwargs["beam_size"] warnings.warn(deprecation_warning.format("beam_size"), DeprecationWarning) if "max_decoding_steps" in kwargs: beam_search_extras["max_steps"] = kwargs["max_decoding_steps"] warnings.warn(deprecation_warning.format("max_decoding_steps"), DeprecationWarning) self._beam_search = beam_search.construct( end_index=self._end_id, vocab=self.vocab, **beam_search_extras ) self._rouge = ROUGE(exclude_indices={self._start_id, self._pad_id, self._end_id}) self._bleu = BLEU(exclude_indices={self._start_id, self._pad_id, self._end_id}) # Replace bart encoder with given encoder. We need to extract the two embedding layers so that # we can use them in the encoder wrapper if encoder is not None: assert ( encoder.get_input_dim() == encoder.get_output_dim() == self.bart.config.hidden_size ) self.bart.model.encoder = _BartEncoderWrapper( encoder, self.bart.model.encoder.embed_tokens, self.bart.model.encoder.embed_positions, )
def __init__( self, lazy: bool = True, bert_model_name: str = 'bert-base-uncased', max_bpe: int = None, # token_indexers: Dict[str, TokenIndexer] = PretrainedTransformerIndexer(bert_model_name), debug: bool = False, bertsum_oracle: bool = True, semantic_red_map: bool = True, semantic_red_map_key: List[str] = None) -> None: super().__init__(lazy=lazy) self._token_indexers = PretrainedTransformerIndexer( bert_model_name, True) if 'roberta' in bert_model_name: from transformers import RobertaTokenizer self._token_indexers.tokenizer = RobertaTokenizer.from_pretrained( bert_model_name) self.generic_tokenzier = RobertaTokenizer.from_pretrained( bert_model_name) elif 'bert' in bert_model_name: from transformers import BertTokenizer self._token_indexers.tokenizer = BertTokenizer.from_pretrained( bert_model_name) self.generic_tokenzier = BertTokenizer.from_pretrained( bert_model_name) # if max_bpe is not None: # self._token_indexers['bert'].max_pieces = max_bpe self._debug = debug # self.bert_tokenizer = BertTokenizer.from_pretrained(bert_model_name) # self.bert_tokenizer = WordTokenizer(word_splitter=BertBasicWordSplitter()) logger.info("Finish Initializing of Dataset Reader") if 'roberta' in bert_model_name: self.bert_lut = self.generic_tokenzier.decoder else: self.bert_lut = self.generic_tokenzier.ids_to_tokens self.max_bpe = max_bpe self.train_pts = [] self.bertsum_oracle = bertsum_oracle self.semantic_red_map = semantic_red_map if semantic_red_map: self.map_kiosk = MapKiosk(semantic_red_map_key) random.seed(1112)
def test_sequence_tagging_reader(): model_name = 'bert-base-chinese' bert_token_indexers = PretrainedTransformerIndexer(model_name=model_name) reader = SequenceTaggingDatasetReader( token_indexers={"tokens": bert_token_indexers}) train_file = './data/weibo/train.corpus' dev_file = './data/weibo/dev.corpus' test_file = './data/weibo/dev.corpus' train_instances = list(reader.read(train_file)) dev_instances = list(reader.read(dev_file)) test_instances = list(reader.read(test_file)) vocab: Vocabulary = Vocabulary.from_instances(train_instances) assert vocab.get_namespaces() is not None bert_text_field_embedder = PretrainedTransformerEmbedder( model_name=model_name) tagger = SimpleTagger( vocab=vocab, text_field_embedder=BasicTextFieldEmbedder( token_embedders={'tokens': bert_text_field_embedder}), encoder=PassThroughEncoder(bert_text_field_embedder.get_output_dim()), calculate_span_f1=True, label_encoding="BMES", # verbose_metrics=True ) train_data_loader, dev_data_loader = build_data_loaders( train_instances, dev_instances) train_data_loader.index_with(vocab) dev_data_loader.index_with(vocab) trainer = build_trainer(model=tagger, serialization_dir='./output', train_loader=train_data_loader, dev_loader=dev_data_loader) print("Starting training") trainer.train() print("Finished training")
def __init__( self, model_name: str, vocab: Vocabulary, indexer: PretrainedTransformerIndexer = None, max_decoding_steps: int = 140, beam_size: int = 4, encoder: Seq2SeqEncoder = None, ): super().__init__(vocab) self.bart = BartForConditionalGeneration.from_pretrained(model_name) self._indexer = indexer or PretrainedTransformerIndexer( model_name, namespace="tokens") self._start_id = self.bart.config.bos_token_id # CLS self._decoder_start_id = self.bart.config.decoder_start_token_id or self._start_id self._end_id = self.bart.config.eos_token_id # SEP self._pad_id = self.bart.config.pad_token_id # PAD self._max_decoding_steps = max_decoding_steps self._beam_search = BeamSearch(self._end_id, max_steps=max_decoding_steps, beam_size=beam_size or 1) self._rouge = ROUGE( exclude_indices={self._start_id, self._pad_id, self._end_id}) self._bleu = BLEU( exclude_indices={self._start_id, self._pad_id, self._end_id}) # Replace bart encoder with given encoder. We need to extract the two embedding layers so that # we can use them in the encoder wrapper if encoder is not None: assert (encoder.get_input_dim() == encoder.get_output_dim() == self.bart.config.hidden_size) self.bart.model.encoder = _BartEncoderWrapper( encoder, self.bart.model.encoder.embed_tokens, self.bart.model.encoder.embed_positions, )
def test_dataset_reader(): model_name = 'bert-base-uncased' source_tokenizer = PretrainedTransformerTokenizer(model_name=model_name, do_lowercase=True) target_tokenizer = PretrainedTransformerTokenizer(model_name=model_name, do_lowercase=True) source_token_indexers = { "tokens": PretrainedTransformerIndexer(model_name=model_name, do_lowercase=True, namespace='bert') } ds = CopySeq2MultiSeqNetDatasetReader( target_namespace='bert', source_tokenizer=source_tokenizer, target_tokenizer=target_tokenizer, source_token_indexers=source_token_indexers, lazy=True, max_tokens=500, bert=True, max_extractions=10) instances = ds._read( "/Users/mostafa/git/deep/openie/imojie/data/train/4cr_qpbo_extractions.tsv" ) for instance in list(instances)[:10]: print(instance) print('*' * 70) ds._validation = True instances2 = ds._read( "/Users/mostafa/git/deep/openie/imojie/data/dev/carb_sentences.txt") for instance in list(instances2)[:10]: print(instance) print('*' * 70)
if __name__ == '__main__': set_seed(1234) # TODO: make this tokenizer initialization a method? uncased = True tokenizer = PretrainedTransformerTokenizer(model_name="bert-base-uncased", do_lowercase=uncased) start_tokens = tokenizer._start_tokens end_tokens = tokenizer._end_tokens tokenizer._start_tokens = [] tokenizer._end_tokens = [] token_indexer = PretrainedTransformerIndexer(model_name="bert-base-cased", do_lowercase=uncased) reader = SemTagDatasetReader(tokenizer, {"model_tokens": token_indexer}, start_tokens, end_tokens) train_dataset = reader.read('sem-0.1.0/data/gold/train') val_dataset = reader.read('sem-0.1.0/data/gold/val') # NOTE: PretrainedTransformerIndexer does not implement the # count_vocab_items method, so this vocabulary reflects only the new # dataset, not the pretrained model's vocabulary # see: https://github.com/allenai/allennlp/blob/master/allennlp/data/ # token_indexers/pretrained_transformer_indexer.py#L47-L50 data_vocab = Vocabulary.from_instances(train_dataset + val_dataset) bert_token_embedder = PretrainedTransformerEmbedder("bert-base-uncased")
from allennlp.training.trainer import Trainer def set_seed(seed): torch.manual_seed(seed) np.random.seed(seed) if __name__ == '__main__': set_seed(1234) model_string = "bert-base-uncased" tokenizer = PretrainedTransformerTokenizer(model_string, do_lowercase=True) token_indexer = PretrainedTransformerIndexer(model_string, do_lowercase=True) reader = SSTDatasetReader(tokenizer, {"tokens": token_indexer}) train_dataset = reader.read('sst/trees/train.txt') val_dataset = reader.read('sst/trees/dev.txt') print(train_dataset[0]) vocab = Vocabulary.from_instances(train_dataset + val_dataset) bert_token_embedder = PretrainedTransformerEmbedder(model_string) bert_textfield_embedder = BasicTextFieldEmbedder( {"tokens": bert_token_embedder}) model = BertClassifier(vocab,
def build_dataset_reader() -> DatasetReader: tokenizer = PretrainedTransformerTokenizer(model_name=PRETRAIN_MODEL) token_indexers = PretrainedTransformerIndexer(model_name=PRETRAIN_MODEL) return ActionCLSTrainDataReader(tokenizer=tokenizer, token_indexers={'tokens': token_indexers}, max_tokens=150,)