def load_transformer(model_type): if model_type == "distilbert": tokenizer = DistilBertTokenizer.from_pretrained( 'distilbert-base-uncased') model = TFDistilBertForSequenceClassification.from_pretrained( "distilbert-base-uncased", num_labels=1) elif model_type == "bert_x12": tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForSequenceClassification.from_pretrained( "bert-base-uncased", num_labels=1) elif model_type == "bert_x24": tokenizer = BertTokenizer.from_pretrained('bert-large-uncased') model = TFBertForSequenceClassification.from_pretrained( "bert-large-uncased", num_labels=1) elif model_type == "albert_v2_x12": tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') model = TFAlbertForSequenceClassification.from_pretrained( "albert-base-v2", num_labels=1) elif model_type == "longformer_x12": tokenizer = LongformerTokenizer.from_pretrained( 'allenai/longformer-base-4096') model = TFLongformerForSequenceClassification.from_pretrained( "allenai/longformer-base-4096", num_labels=1) elif model_type == "longformer_x24": tokenizer = LongformerTokenizer.from_pretrained( 'allenai/longformer-large-4096') model = TFLongformerForSequenceClassification.from_pretrained( "allenai/longformer-large-4096", num_labels=1) else: raise ValueError(model_type + " was invalid") return model, tokenizer
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.bert_layer = TFBertForSequenceClassification.from_pretrained( 'bert-base-cased') self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased') self.cashed_train_dataset = None self.cashed_val_dataset = None
def load_model_from_zip(self, zip_archive): """a function that loads components of a serialized model from a zip given zip file using the python ZipFile interface and returns an instance of the model Arguments: zip_archive: ZipFile an instance of the python ZipFile interface that has loaded the file path specified by self.resource.disk_target Returns: model: Any any format of of machine learning model that will be stored in the self.model attribute for later use """ # read the h5 bytes from the zip file with zip_archive.open('transformer.zip', "r") as file: model_bytes = file.read() # read model bytes in the h5 format # load the bytes of the hugging face save path from a zip with tempfile.TemporaryDirectory() as directory: with tempfile.NamedTemporaryFile(suffix=".zip") as archive: archive.write(model_bytes) shutil.unpack_archive(archive.name, directory) return TFBert.from_pretrained(directory)
def carga_modelo_BERT(model_path): """Carga el modelo BERT preentrenado y que se encuentra en la ruta `model_path`""" # ParĂ¡metros del script usado por HuggingFace para hacer anĂ¡lisis de sentimientos sobre otro conjunto de datos USE_XLA = False USE_AMP = False #TASK = "sst-2" #TFDS_TASK = "sst2" num_labels = 2 tf.config.optimizer.set_jit(USE_XLA) tf.config.optimizer.set_experimental_options( {"auto_mixed_precision": USE_AMP}) # Load tokenizer and model from pretrained model/vocabulary. Specify the number of labels to classify (2+: classification, 1: regression) config = BertConfig.from_pretrained("bert-base-cased", num_labels=num_labels) tokenizer = BertTokenizer.from_pretrained("bert-base-cased") model = TFBertForSequenceClassification.from_pretrained("bert-base-cased", config=config) opt = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08) loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy") model.compile(optimizer=opt, loss=loss, metrics=[metric]) model.load_weights(model_path) return model, tokenizer, config
def _load_remote_model(self, model_name, tokenizer_kwargs, model_kwargs): if model_name not in ModelsByFamily.Supported: raise ValueError(f'Model {model_name} not supported.') do_lower_case = False if 'uncased' in model_name.lower(): do_lower_case = True tokenizer_kwargs.update({'do_lower_case': do_lower_case}) self._tokenizer = None self._model = None if model_name in ModelsByFamily.Bert: self._tokenizer = BertTokenizer.from_pretrained( model_name, **tokenizer_kwargs) self._model = TFBertForSequenceClassification.from_pretrained( model_name, **model_kwargs) elif model_name in ModelsByFamily.Roberta: self._tokenizer = RobertaTokenizer.from_pretrained( model_name, **tokenizer_kwargs) self._model = TFRobertaForSequenceClassification.from_pretrained( model_name, **model_kwargs) elif model_name in ModelsByFamily.XLNet: self._tokenizer = XLNetTokenizer.from_pretrained( model_name, **tokenizer_kwargs) self._model = TFXLNetForSequenceClassification.from_pretrained( model_name, **model_kwargs) elif model_name in ModelsByFamily.DistilBert: self._tokenizer = DistilBertTokenizer.from_pretrained( model_name, **tokenizer_kwargs) self._model = TFDistilBertForSequenceClassification.from_pretrained( model_name, **model_kwargs) assert self._tokenizer and self._model
def build_model(self, max_length, train_batch_size, learning_rate, epochs, num_labels, tagset=None, gpu_growth=True, eval_batch_size=32): #if gpu_growth: # model_utils.set_tf_memory_growth() if self.task == "pos": self.model = TFBertForTokenClassification.from_pretrained( self.model_name, num_labels=num_labels, from_pt=True) self.tokenizer = MBERT_Tokenizer_pos.from_pretrained( self.model_name, do_lower_case=False) else: self.model = TFBertForSequenceClassification.from_pretrained( self.model_name, num_labels=num_labels, from_pt=True) self.tokenizer = BertTokenizer.from_pretrained(self.model_name, do_lower_case=False) #self.model, self.tokenizer = model_utils.create_model(self.short_model_name, self.task, num_labels) self.model = model_utils.compile_model(self.model, self.task, learning_rate) print("Successfully built", self.model_name) self.max_length = max_length self.train_batch_size = train_batch_size self.learning_rate = learning_rate self.epochs = epochs self.num_labels = num_labels if tagset: self.tagset = tagset self.label_map = {label: i for i, label in enumerate(tagset)} self.eval_batch_size = eval_batch_size
def build(self, **kwargs): optimizer = kwargs.get("optimizer", "adam") metrics = kwargs.get("metrics", ['accuracy']) dropout_rate = kwargs.get('dropout_rate', 0.5) ## BUILDING THE GRAPH input_ids = tf.keras.layers.Input(shape=(1,50), name='input_ids', dtype=tf.int32) input_mask = tf.keras.layers.Input(shape=(1,50), name='input_mask', dtype=tf.int32) #bert_layer = Lambda_Bert_Layer(trainable=False, dynamic=True) bert_layer = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') bert_layer.bert.trainable=False bert_output = bert_layer(input_ids[:,0,:], attention_mask=input_mask[:,0,:]) """ bert_layer.trainable = False bert_output=bert_output[0][:,-1,:] last_state = tf.reshape(bert_output, shape=(-1,768)) dense_out_1 = tf.keras.layers.Dense(units=768, activation="relu")(last_state) # reshape_lambda_layer dense_out_1 = tf.keras.layers.Dropout(dropout_rate)(dense_out_1) dense_out_2 = tf.keras.layers.Dense(units=200, activation="relu")(dense_out_1) dense_out_2 = tf.keras.layers.Dropout(dropout_rate)(dense_out_2) logits = tf.keras.layers.Dense(units=2, activation='softmax')(dense_out_2) """ logits = bert_output[0] self.model = tf.keras.Model(inputs=(input_ids,input_mask), outputs=logits) self.model.compile(optimizer=optimizer, loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), metrics=metrics, run_eagerly=True) self.model.summary()
def train(self, train_data, train_labels, dev_data, dev_labels, save_model_path=f"models/bert.pkl"): ds_train_encoded = self.encode_examples( train_data, train_labels).batch(self.batch_size) ds_dev_encoded = self.encode_examples(dev_data, dev_labels).batch( self.batch_size) self.model = TFBertForSequenceClassification.from_pretrained( 'bert-base-uncased') optimizer = tf.keras.optimizers.Adam(learning_rate=self.learning_rate, epsilon=1e-08) loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') self.model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) self.model.fit(ds_train_encoded, epochs=self.epochs, validation_data=ds_dev_encoded) predictions = self.model.predict(ds_dev_encoded, verbose=1).logits self.model.save_weights(save_model_path) print('Validation Loss:', log_loss(dev_labels, predictions))
def train_model(request): if request.method == "POST": inputed_batch_size = int(request.POST['batch_size']) model = TFBertForSequenceClassification.from_pretrained( "bert-base-uncased") tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") #IMDB movie reviews dataset = get_dataset() #save dataset to file memory clean_dataset(dataset) #training and testing datasets here train = tf.keras.preprocessing.text_dataset_from_directory( 'aclImdb/train', batch_size=inputed_batch_size, validation_split=0.2, subset='training', seed=123) test = tf.keras.preprocessing.text_dataset_from_directory( 'aclImdb/train', batch_size=inputed_batch_size, validation_split=0.2, subset='validation', seed=123) #convert to pandas dataframes train = convert_dataset_to_dataframe(train) test = convert_dataset_to_dataframe(test) #convert data to tf datasets train_InputExamples, validation_InputExamples = convert_data_to_examples( train, test, DATA_COLUMN, LABEL_COLUMN) train_data = convert_examples_to_tf_dataset(list(train_InputExamples), tokenizer) train_data = train_data.shuffle(100).batch(32).repeat(2) validation_data = convert_examples_to_tf_dataset( list(validation_InputExamples), tokenizer) validation_data = validation_data.batch(32) #fine tune it! model.compile( optimizer=tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0), loss=tf.keras.losses.SparseCategoricalCrossentropy( from_logits=True), metrics=[tf.keras.metrics.SparseCategoricalAccuracy('accuracy')]) model.fit(train_data, epochs=2, validation_data=validation_data) #save the model weights to memory to be used later model.save_pretrained(MOOD_MODEL_DIR) context = {} return render(request, 'mood_classifier/train_model_button.html', context)
def __init__(self): model_name = 'bert-base-cased' self.tokenizer = BertTokenizer.from_pretrained(model_name) self.model = TFBertForSequenceClassification.from_pretrained( 'C:/Users/lbbre/Documents/ECAM 5/PRD/content/assets') self.max_seq_len = 32
def load(self, path): """Loads model from path""" self.strategy = self._get_distributed_strategy() with self.strategy.scope(): self.initialise_models() self.model = TFBertForSequenceClassification.from_pretrained(path) self.trained_ = True
def model_build(self): self.bertConfig = BertConfig.from_pretrained( os.path.join(self.pretrain_path, "config.json"), num_labels=self.num_classes) self.model = TFBertForSequenceClassification.from_pretrained( os.path.join(self.pretrain_path, "tf_model.h5"), config=self.bertConfig) self.model.summary()
def build(self): # Handle the Meta Data ids = tf.keras.Input((self.config.max_length, ), dtype=tf.int32, name='input_ids') vn = tf.keras.Input((1, ), dtype=tf.float32, name='version_number') pl = tf.keras.Input((1, ), dtype=tf.float32, name='partisan_lean') cat = tf.keras.Input((self.config.n_sc_id_classes, ), dtype=tf.float32, name='sc_id') meta = tf.concat([vn, pl, cat], axis=-1) # Load the initial weights with the ones trained from the DL model without text if self.config.load_weights_from_no_text: #if 'no_text_dense_layer_initialization_path' in self.config: print( "Usinging pretrained weights from the no_text model! --------------------" ) model_location = self.config.data_vol + "models/no_text/full_model.h5" ntdl = tf.keras.layers.Dense( self.config.n_dense_layers, activation='relu', name="no_text_dense_layer", kernel_initializer=noTextKernelInitializer( model_location=model_location), bias_initializer=noTextBiasInitializer( model_location=model_location)) else: ntdl = tf.keras.layers.Dense(self.config.n_dense_layers, activation='relu', name="no_text_dense_layer") meta = ntdl(meta) # Handle the Transformer self.base_transformer_model = TFBertForSequenceClassification.from_pretrained( "bert-base-uncased") x = self.base_transformer_model.bert(ids) # Get the main Layer x = x['last_hidden_state'][:, 0, :] x = tf.keras.layers.Dropout(0.2)(x) # Combine the two and run through another dense layer. x = tf.concat([x, meta], axis=-1) x = tf.keras.layers.Dense(self.config.n_dense_layers, activation='relu')(x) x = tf.keras.layers.Dropout(0.2)(x) x = tf.keras.layers.Dense(1, activation='sigmoid')(x) dl_model = tf.keras.Model(inputs={ "input_ids": ids, "version_number": vn, "partisan_lean": pl, "sc_id": cat }, outputs=[x]) self.deep_legis_model = dl_model
def create_model_2(self): model = TFBertForSequenceClassification.from_pretrained( 'bert-base-uncased') model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=2e-5, epsilon=1e-08), loss=tf.keras.losses.SparseCategoricalCrossentropy( from_logits=True), metrics=["acc"]) model.summary() return model
def __init__(self, n_intents=None, dropout=0.2, model_name="bert-base-uncased"): super().__init__(name="intent_classifier") self.tokenizer = Tokenizer() self.bert = TFBertForSequenceClassification.from_pretrained(model_name) self.dropout = Dropout(dropout) self.intent_classifier = Dense(n_intents, activation='softmax')
def model_compile(): model = TFBertForSequenceClassification.from_pretrained( 'bert-base-uncased', num_labels=2) loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = keras.metrics.SparseCategoricalAccuracy('accuracy') optimizer = keras.optimizers.Adam(learning_rate=2e-5, epsilon=1e-08) model.compile(loss=loss, optimizer=optimizer, metrics=[metric]) return model
def __init__(self, extractor, config, *args, **kwargs): super(TFVanillaBert_Class, self).__init__(*args, **kwargs) self.extractor = extractor # TFBertForSequenceClassification contains both the BERT and the linear classifier layers self.bert = TFBertForSequenceClassification.from_pretrained( config["pretrained"], hidden_dropout_prob=0.1) assert extractor.config[ "numpassages"] == 1, "numpassages should be 1 for TFVanillaBERT" self.config = config
def build_model(): import tensorflow as tf from transformers import TFBertForSequenceClassification model = TFBertForSequenceClassification.from_pretrained( "bert-base-uncased") loss_fn = tf.keras.losses.SparseCategoricalCrossentropy( reduction=tf.keras.losses.Reduction.NONE, from_logits=True) opt = tf.keras.optimizers.Adam(learning_rate=3e-5) model.compile(optimizer=opt, loss=loss_fn, metrics=['accuracy']) return model
def _init_model(self, num_labels=2): config = {"name": self.pretrained, "from_pt": self.from_pt} if self.pretrained in PRETRAINED_CONFIG: config = PRETRAINED_CONFIG[self.pretrained] pretrained = config["name"] from_pt = config["from_pt"] self.tokenizer = BertTokenizer.from_pretrained(pretrained) self.model = TFBertForSequenceClassification.from_pretrained( pretrained, from_pt=from_pt, num_labels=num_labels)
def fine_tune_model(ds, export_dir): (train_dataset, test_dataset, val_dataset) = get_test_train_val_datasets(ds) learning_rate = 2e-5 number_of_epochs = 1 model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08) loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) bert_history = model.fit(train_dataset, epochs=number_of_epochs, validation_data=val_dataset) model.save_pretrained(export_dir) return model
def test_TFBertForSequenceClassification(self): from transformers import BertTokenizer, TFBertForSequenceClassification pretrained_weights = 'bert-base-uncased' tokenizer = BertTokenizer.from_pretrained(pretrained_weights) text, inputs, inputs_onnx = self._prepare_inputs(tokenizer) model = TFBertForSequenceClassification.from_pretrained( pretrained_weights) predictions = model.predict(inputs) onnx_model = keras2onnx.convert_keras(model, model.name) self.assertTrue( run_onnx_runtime(onnx_model.graph.name, onnx_model, inputs_onnx, predictions, self.model_files))
def run_first_test(): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 print(f"input_ids:{input_ids}") outputs = model(input_ids) logits = outputs[0] print(logits) print("-" * 30) print(f"outputs:{outputs}") print("-" * 30) model.summary()
def get_model(self): """ Build BERT model. Using origin way to build BERT model On the Transformers package. But this way don't custom input and output, so that can't adapt to deepnlp project worker. Need to change deepnlp project parameter. """ model = TFBertForSequenceClassification.from_pretrained( pretrained_model_name_or_path=self.model_name, from_pt=True if self.from_pt_word in self.model_name else False, num_labels=len(self.label_list)) return model
def initialise_models(self): if self.pretrained == "bert": model_name = "bert-base-cased" from_pt = False elif self.pretrained == "scibert": model_name = "allenai/scibert_scivocab_cased" from_pt = True self.config = BertConfig.from_pretrained(model_name, num_labels=2) self.tokenizer = BertTokenizer.from_pretrained(model_name) self.model = TFBertForSequenceClassification.from_pretrained( model_name, config=self.config, from_pt=from_pt) return self.model
def bert_result(sentence): model = TFBertForSequenceClassification.from_pretrained( "bert-base-uncased") tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") model.load_weights("senti_weights.ckpt") tf_batch = tokenizer(sentence, max_length=128, padding=True, truncation=True, return_tensors='tf') tf_outputs = model(tf_batch) tf_predictions = tf.nn.softmax(tf_outputs[0], axis=-1) label = tf_predictions.numpy() return int(label[0][1] * 100)
def __init__(self, weights_path): """ :param weights_path: specifies where load/store weights of the model :type weights_path: str """ super().__init__(weights_path) # A tensorflow model of Bert base (uncased), pre-trained. # More on it on our report. self.__model = TFBertForSequenceClassification.from_pretrained( 'bert-large-uncased') # Instanciating a proper tokenizer for Bert self.__tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')
def __init__(self, model_path="./modelSentiment", tokenizer="bert-base-uncased"): """ Initialize model Downloads the model if not present """ if os.path.exists(model_path): self.model = TFBertForSequenceClassification.from_pretrained( model_path) else: print("Downloading model...") Model.download_file_from_google_drive( "1uthnEb7WYnIR6y0VVX4gMPoqG-X0oKRK", "Modelfile.zip") with zipfile.ZipFile("Modelfile.zip", "r") as zip_ref: zip_ref.extractall("./") self.model = TFBertForSequenceClassification.from_pretrained( model_path) self.tokenizer = BertTokenizer.from_pretrained(tokenizer)
def evaluate(dataset, limit_num_sents: bool): # Split and tokenize dataset split = Split_BERT() tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') X_train, y_train = split.get_X_y(dataset['train'] + dataset['oos_train'], limit_num_sents=limit_num_sents, set_type='train') X_val, y_val = split.get_X_y(dataset['val'] + dataset['oos_val'], limit_num_sents=limit_num_sents, set_type='val') X_test, y_test = split.get_X_y(dataset['test'] + dataset['oos_test'], limit_num_sents=limit_num_sents, set_type='test') train_ids, train_attention_masks, train_labels = tokenize_BERT(X_train, y_train, tokenizer) val_ids, val_attention_masks, val_labels = tokenize_BERT(X_val, y_val, tokenizer) test_ids, test_attention_masks, test_labels = tokenize_BERT(X_test, y_test, tokenizer) num_labels = len(split.intents_dct.keys()) # Train model model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=num_labels) # we have to adjust the number of labels print('\nBert Model', model.summary()) log_dir = 'tensorboard_data/tb_bert' model_save_path = './models/bert_model.h5' callbacks = [ tf.keras.callbacks.ModelCheckpoint(filepath=model_save_path, save_weights_only=True, monitor='val_loss', mode='min', save_best_only=True), tf.keras.callbacks.TensorBoard(log_dir=log_dir)] loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') optimizer = tf.keras.optimizers.Adam(learning_rate=4e-5) model.compile(loss=loss, optimizer=optimizer, metrics=[metric]) history = model.fit([train_ids, train_attention_masks], train_labels, batch_size=32, epochs=5, validation_data=([val_ids, val_attention_masks], val_labels), callbacks=callbacks) # Test testing = Testing(model, {'test_ids': test_ids, 'test_attention_masks': test_attention_masks}, test_labels, 'bert', split.intents_dct['oos']) results_dct = testing.test_train() return results_dct
def __init__(self): self.modified = False # load models self.classifier_tokenizer = BertTokenizer.from_pretrained( CLASSIFIER_PATH) self.classifier_config = AutoConfig.from_pretrained(CLASSIFIER_PATH) self.classifier_model = TFBertForSequenceClassification.from_pretrained( CLASSIFIER_PATH) self.ner_tokenizer = AutoTokenizer.from_pretrained(PARSBERTNER_PATH) self.ner_config = AutoConfig.from_pretrained(PARSBERTNER_PATH) self.ner_model = TFAutoModelForTokenClassification.from_pretrained( PARSBERTNER_PATH) self.weather_api = Weather() self.adhan_api = Adhan() self.time_api = Time() self.calender_api = Calender()
def fit(self, messages, y, epochs=2, validation_percent=0.15, allow_import=True): if self.english: messages = (messages['Translation'].values) else: messages = (messages['Message'].values) y = self.Lab_Encoder.fit_transform( y.to_numpy().astype(str)).astype(float) # preparing the data: # split in train and validation if os.path.exists(self.path) and allow_import: print("Loading Pretrained Model") self.model = bert_class.from_pretrained(self.path) return (self) else: X_train, X_val, y_train, y_val = train_test_split( messages, y, test_size=validation_percent, random_state=0) # tokenizing the data and making tf.dataset X_train_input = self.convert_to_input(X_train.astype(str)) X_val_input = self.convert_to_input(X_val.astype(str)) train_ds = tf.data.Dataset.from_tensor_slices( (X_train_input[0], X_train_input[1], X_train_input[2], y_train)).map( self.example_to_features).shuffle(100).batch(12).repeat(5) val_ds = tf.data.Dataset.from_tensor_slices( (X_val_input[0], X_val_input[1], X_val_input[2], y_val)).map(self.example_to_features).batch(12) self.model.fit(train_ds, epochs=epochs, validation_data=val_ds, verbose=1) os.mkdir(self.path) self.model.save_pretrained(self.path) return self