def build(self, hidden_layers=[16], activations=['relu'], dropout=0.5, learning_rate=0.01, l2_norm=5e-4, p1=1., p2=1., n_power_iterations=1, epsilon=0.03, xi=1e-6): with self.device: x = Input(batch_shape=[self.n_nodes, self.n_features], dtype=tf.float32, name='features') adj = Input(batch_shape=[self.n_nodes, self.n_nodes], dtype=tf.float32, sparse=True, name='adj_matrix') index = Input(batch_shape=[None], dtype=tf.int32, name='index') self.GCN_layers = [GraphConvolution(hidden_layers[0], activation=activations[0], kernel_regularizer=regularizers.l2(l2_norm)), GraphConvolution(self.n_classes)] self.dropout_layer = Dropout(dropout) logit = self.propagation(x, adj) output = tf.gather(logit, index) output = Softmax()(output) model = Model(inputs=[x, adj, index], outputs=output) self.model = model self.train_metric = SparseCategoricalAccuracy() self.test_metric = SparseCategoricalAccuracy() self.optimizer = Adam(lr=learning_rate) self.built = True self.p1 = p1 # Alpha self.p2 = p2 # Beta self.xi = xi # Small constant for finite difference self.epsilon = epsilon # Norm length for (virtual) adversarial training self.n_power_iterations = n_power_iterations # Number of power iterations
def init_loss(self): self.loss_function = SparseCategoricalCrossentropy() self.train_loss = Mean(name="train_loss") self.train_accuracy = SparseCategoricalAccuracy(name="train_accuracy") self.test_loss = Mean(name="test_loss") self.test_accuracy = SparseCategoricalAccuracy(name="test_accuracy")
def build(self, hiddens=[16], activations=['relu'], dropout=0.5, lr=0.01, l2_norm=5e-4, use_bias=False, p1=1., p2=1., n_power_iterations=1, epsilon=0.03, xi=1e-6): with tf.device(self.device): x = Input(batch_shape=[None, self.graph.n_attrs], dtype=self.floatx, name='attr_matrix') adj = Input(batch_shape=[None, None], dtype=self.floatx, sparse=True, name='adj_matrix') index = Input(batch_shape=[None], dtype=self.intx, name='node_index') GCN_layers = [] dropout_layers = [] for hidden, activation in zip(hiddens, activations): GCN_layers.append( GraphConvolution( hidden, activation=activation, use_bias=use_bias, kernel_regularizer=regularizers.l2(l2_norm))) dropout_layers.append(Dropout(rate=dropout)) GCN_layers.append( GraphConvolution(self.graph.n_classes, use_bias=use_bias)) self.GCN_layers = GCN_layers self.dropout_layers = dropout_layers logit = self.forward(x, adj) output = Gather()([logit, index]) model = Model(inputs=[x, adj, index], outputs=output) self.model = model self.train_metric = SparseCategoricalAccuracy() self.test_metric = SparseCategoricalAccuracy() self.loss_fn = SparseCategoricalCrossentropy(from_logits=True) self.optimizer = Adam(lr=lr) self.p1 = p1 # Alpha self.p2 = p2 # Beta self.xi = xi # Small constant for finite difference # Norm length for (virtual) adversarial training self.epsilon = epsilon self.n_power_iterations = n_power_iterations # Number of power iterations
def load_metrics(): global train_loss, train_acc global valid_loss, valid_acc global test_loss, test_acc train_loss = Mean() valid_loss = Mean() test_loss = Mean() train_acc = SparseCategoricalAccuracy() valid_acc = SparseCategoricalAccuracy() test_acc = SparseCategoricalAccuracy()
def compile_model(model, learning_rate, decay): logger.info('Compiling the model...') # Instantiate the optimizer optimizer = Adam(learning_rate=learning_rate, decay=decay) # Instantiate the loss function loss_fn = SparseCategoricalCrossentropy(from_logits=True) # Prepare the metrics train_acc_metric = SparseCategoricalAccuracy() val_acc_metric = SparseCategoricalAccuracy() # Compile the model model.compile(optimizer=optimizer, loss=loss_fn, metrics=[train_acc_metric]) logger.info('Completed compiling the model.') return optimizer, loss_fn, train_acc_metric, val_acc_metric
def length_accuracy(dataset): """ generates the lengthwise accuracy of a dataset :param dataset: dataset to consider :return: """ model = load_model(constants.text_recognition) for x, y in dataset: # create the accuracy evaluation object accuracy = SparseCategoricalAccuracy() # make a prediction and update the state of the accuracy using it prediction = model.predict(x) accuracy.update_state(y, prediction) print(y[0].numpy()) print(np.argmax(prediction, axis=-1)[0]) print("sequences of length", x[1].shape[1] - 1, "had an accuracy of", accuracy.result().numpy())
def evaluate(self): results = [] for x_test, y_test in self.gen.evaluate(): m = SparseCategoricalAccuracy() predictions = [] predictions.append(self.models[0].predict(np.array(x_test))) predictions.append(self.models[0].predict(np.array(x_test))) # calculate average outcomes = average(predictions).numpy() print(outcomes) print(y_test) if len(results) > 0: curr_avg = sum(results) / len(results) print("Accuracy:", curr_avg) m.update_state( # We have changed y_true = [[2], [1], [3]] to the following y_true=y_test, y_pred=outcomes, sample_weight=[1, 1, 1]) results.append(m.result().numpy()) avg = sum(results) / len(results) return avg
def get_classification_metrics(): train_loss = Mean() train_acc = SparseCategoricalAccuracy() valid_loss = Mean() valid_acc = SparseCategoricalAccuracy() test_loss = Mean() test_acc = SparseCategoricalAccuracy() metric_objects = dict() metric_objects['train_loss'] = train_loss metric_objects['train_acc'] = train_acc metric_objects['valid_loss'] = valid_loss metric_objects['valid_acc'] = valid_acc metric_objects['test_loss'] = test_loss metric_objects['test_acc'] = test_acc return metric_objects
def define_loss(self): """Store attributes containing loss and accuracy functions. Training loss is defined by Cross Categorical Cross Entropy and by Dice loss, an intersection-over-union loss function. """ cce = SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE, name='sparse_crossentropy') self.cce_loss = cce self.dice_loss = jaccard self.tst_loss = tf.keras.metrics.Mean(name='test_loss') self.trn_acc = SparseCategoricalAccuracy(name='trn_acc') self.tst_acc = SparseCategoricalAccuracy(name='tst_acc')
def evaluate(self, dataset): accuracy_metric = SparseCategoricalAccuracy() for x, y in dataset: y_hat = self.predict(x) accuracy_metric.update_state(y, y_hat) return accuracy_metric.result().numpy()
def get_classification_metrics(losses=None, accuracy=None): train_loss = Mean() train_acc = SparseCategoricalAccuracy() validation_loss = Mean() validation_acc = SparseCategoricalAccuracy() test_lost = Mean() test_acc = SparseCategoricalAccuracy() metrics_objects = dict() metrics_objects['train_loss'] = train_loss metrics_objects['train_accuracy'] = train_acc metrics_objects['validation_loss'] = validation_loss metrics_objects['validation_accuracy'] = validation_acc metrics_objects['test_loss'] = test_lost metrics_objects['test_accuracy'] = test_acc return metrics_objects
def build_estimator(self): model = TFRobertaForSequenceClassification.from_pretrained( ROBERTA_BASE) optimizer = AdamWeightDecay( learning_rate=LEARNING_RATE, epsilon=EPSILON, weight_decay_rate=DECAY, beta_1=BETA) # we do not have one-hot vectors, we can use sparce categorical cross entropy and accuracy loss = SparseCategoricalCrossentropy(from_logits=True) metric = SparseCategoricalAccuracy('accuracy') model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) self.model = model
def get_model(): model = Sequential([Conv2D(filters=16, input_shape=(32, 32, 3), kernel_size=(3,3), activation='relu'), Conv2D(filters=8, kernel_size=(3,3), activation='relu'), MaxPooling2D((4, 4)), Flatten(), BatchNormalization(), Dense(32, activation='relu'), Dense(10, activation='softmax')]) model.compile(optimizer=Adam(), loss=SparseCategoricalCrossentropy(), metrics=[SparseCategoricalAccuracy()]) return model
def on_epoch_end(self, epoch, logs={}): y_pred = self.model.predict(self.X_val, verbose=0) scce = SparseCategoricalCrossentropy(from_logits=True) score = scce(self.y_val, y_pred).numpy() acc = SparseCategoricalAccuracy() acc.update_state(self.y_val, y_pred) print("\n") print("The loss is : {}, the accuracy is: {}".format( score, acc.result().numpy())) gc.collect() K.clear_session()
def __init__(self, model, train_log_dir, test_log_dir, manager): self._model = model self._loss_fn = tf.nn.sparse_softmax_cross_entropy_with_logits self._manager = manager self._train_loss = Mean(name='train_loss') self._test_loss = Mean(name='test_loss') self._train_acc = SparseCategoricalAccuracy(name='train_acc') self._test_acc = SparseCategoricalAccuracy(name='test_acc') self._train_loss.reset_states() self._test_loss.reset_states() self._train_acc.reset_states() self._test_acc.reset_states() os.makedirs(train_log_dir, exist_ok=True) os.makedirs(test_log_dir, exist_ok=True) self._train_summary_writer = create_file_writer(train_log_dir) self._test_summary_writer = create_file_writer(test_log_dir)
def __init__(self, fbnet, input_shape, initial_temperature=5, temperature_decay_rate=0.956, temperature_decay_steps=1, latency_alpha=0.2, latency_beta=0.6, weight_lr=0.01, weight_momentum=0.9, weight_decay=1e-4, theta_lr=1e-3, theta_beta1=0.9, theta_beta2=0.999, theta_decay=5e-4): self._epoch = 0 self.initial_temperature = initial_temperature self.temperature = initial_temperature self.latency_alpha = latency_alpha self.latency_beta = latency_beta self.exponential_decay = lambda step: exponential_decay( initial_temperature, temperature_decay_rate, temperature_decay_steps, step) fbnet.build(input_shape) self.fbnet = fbnet self.weights = [] self.thetas = [] for trainable_weight in fbnet.trainable_weights: if 'theta' in trainable_weight.name: self.thetas.append(trainable_weight) else: self.weights.append(trainable_weight) self.weight_opt = SGD(learning_rate=weight_lr, momentum=weight_momentum, decay=weight_decay) self.theta_opt = Adam(learning_rate=theta_lr, beta_1=theta_beta1, beta_2=theta_beta2, decay=theta_decay) self.loss_fn = SparseCategoricalCrossentropy(from_logits=True) self.accuracy_metric = SparseCategoricalAccuracy() self.loss_metric = Mean()
def __init__(self, params): self.lr = params.lr self.epochs = params.epochs # Define loss: self.loss_object = SparseCategoricalCrossentropy() # Define optimizer: self.optimizer = Adam() # Define metrics for loss: self.train_loss = Mean() self.train_accuracy = SparseCategoricalAccuracy() self.test_loss = Mean() self.test_accuracy = SparseCategoricalAccuracy() # Define pre processor (params): preprocessor = Process(32, 1) self.train_ds, self.test_ds, encoder_stats = preprocessor.get_datasets( ) # Define model dims d_model = 512 ff_dim = 2048 heads = 8 encoder_dim = 6 decoder_dim = 6 dk = d_model / heads dv = d_model / heads vocab_size = encoder_stats['vocab_size'] max_pos = 10000 # Define model: self.model = Transformer(d_model, ff_dim, dk, dv, heads, encoder_dim, decoder_dim, vocab_size, max_pos) # Define Checkpoints: self.ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) # Define Checkpoint manager: self.ckpt_manager = tf.train.CheckpointManager( self.ckpt, f'checkpoints{params.ckpt_dir}', max_to_keep=3)
def build_model(self): input_layer = Input(shape=784) dense_layer_0 = Dense(512, activation='relu')(input_layer) dense_layer_1 = Dense(256, activation='relu')(dense_layer_0) dense_layer_2 = Dense(128, activation='relu')(dense_layer_1) dense_layer_3 = Dense(64, activation='relu')(dense_layer_2) dense_layer_4 = Dense(32, activation='relu')(dense_layer_3) output_layer = Dense(10, activation='softmax')(dense_layer_4) self.model = Model(input_layer, output_layer) print(self.model.summary()) loss = SparseCategoricalCrossentropy() optimizer = Adam(learning_rate=1e-3) self.model.compile(loss=loss, optimizer=optimizer, metrics=[SparseCategoricalAccuracy()])
def print_bin_predictions_match(y_test, yhat, binary=True): accuracy = BinaryAccuracy() if binary else SparseCategoricalAccuracy() for i in range(y_test.shape[0]): ix = colored(str(f'{i:02d} |'), 'grey') sep = colored('|', 'grey') y = int(y_test[i][0]) pred = f"{yhat[i][0]:.02f}" if binary else f"{np.argmax(yhat[i])}" accuracy.reset_states() accuracy.update_state(y, yhat[i]) true_pred = accuracy.result().numpy() == 1.0 color = "green" if true_pred else "red" pred = colored(pred, color) if i % 9 == 0: print(f"\n{ix} ", end='') print(f"{y} {sep} {pred} {sep} ", end="") print(colored("\n", "white"))
def fit(self, dataset, augment=True): """ Dataset name, intent class """ self._load_fit_parameters() (x_train, y_train), (x_valid, y_valid) = self._load_format_dataset(dataset, augment) self.model = self._load_base_model(self.fit_params['output_length']) opt = Adam(learning_rate=self.fit_params['learning_rate'], epsilon=self.fit_params['epsilon']) if self.fit_params['from_logits']: loss = SparseCategoricalCrossentropy(from_logits=True) else: loss = SparseCategoricalCrossentropy(from_logits=False) metrics = SparseCategoricalAccuracy('accuracy') self.model.compile(optimizer=opt, loss=loss, metrics=metrics) time = datetime.datetime.now() checkpoint_path = os.path.join(self.model_dir, f"classifier") if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) cp_callback = ModelCheckpoint(filepath=os.path.join(checkpoint_path, f"classifier_model"), \ save_weights_only=True, verbose=1, save_best_only=True) if self.fit_params['validate']: history = self.model.fit(x_train, y_train, epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'], validation_data=(x_valid, y_valid), callbacks=cp_callback) else: history = self.model.fit(x_train, y_train, epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'], callbacks=cp_callback) dump(self.fit_params, os.path.join(checkpoint_path, 'params.json')) self.fitted = True # LOGGING print(f"Classifier model fitted in {elapsed_time(time)}s")
def __init__(self, model_name, max_length, curdir): df_train, df_valid, df_test, intent_names, \ self.intent_map, self.slot_map = load_prepare_dataset(curdir) # Y's: self.intent_train = df_train["intent_label"].map( self.intent_map).values self.intent_valid = df_valid["intent_label"].map( self.intent_map).values self.intent_test = df_test["intent_label"].map(self.intent_map).values tokenizer = BertTokenizer.from_pretrained(model_name) self.curdir = curdir # X's: print('Encoding data...') self.encoded_train = encode_dataset(tokenizer, df_train["words"], max_length) self.encoded_valid = encode_dataset(tokenizer, df_valid["words"], max_length) self.encoded_test = encode_dataset(tokenizer, df_test["words"], max_length) self.slot_train = encode_token_labels(df_train["words"], df_train["word_labels"], tokenizer, self.slot_map, max_length) self.slot_valid = encode_token_labels(df_valid["words"], df_valid["word_labels"], tokenizer, self.slot_map, max_length) self.slot_test = encode_token_labels(df_test["words"], df_test["word_labels"], tokenizer, self.slot_map, max_length) self.intent_model = SlotIntentDetectorModelBase( intent_num_labels=len(self.intent_map), slot_num_labels=len(self.slot_map)) opt = Adam(learning_rate=3e-5, epsilon=1e-08) losses = [ SparseCategoricalCrossentropy(from_logits=True), SparseCategoricalCrossentropy(from_logits=True) ] metrics = [SparseCategoricalAccuracy('accuracy')] self.intent_model.compile(optimizer=opt, loss=losses, metrics=metrics)
def fit(self, dataset, intent, merge_intents): """ Dataset name, intent class """ self.intent = intent self._load_fit_parameters() (x_train, y_train), (x_valid, y_valid) = self._load_format_dataset(dataset, merge_intents) self.model = self._load_base_model(self.fit_params['output_length']) opt = Adam(learning_rate=self.fit_params['learning_rate'], epsilon=self.fit_params['epsilon']) if self.fit_params['from_logits']: loss = SparseCategoricalCrossentropy(from_logits=True) else: loss = SparseCategoricalCrossentropy(from_logits=False) metrics = SparseCategoricalAccuracy('accuracy') # metrics = [Precision(), Recall()] self.model.compile(optimizer=opt, loss=loss, metrics=metrics) time = datetime.datetime.now() # if isinstance(self.intent, list): # self.intent = '@'.join(self.intent) checkpoint_path = os.path.join(self.models_dir, f"{self.intent}") if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) cp_callback = ModelCheckpoint(filepath=os.path.join(checkpoint_path, f"{self.intent}_model"), \ save_weights_only=True, verbose=1, save_best_only=True) if self.fit_params['validate']: history = self.model.fit(x_train, y_train, epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'], validation_data=(x_valid, y_valid), callbacks=[cp_callback]) else: history = self.model.fit(x_train, y_train, epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'], callbacks=[cp_callback]) # LOGGING print(f"{self.intent} tagger fitted") dump(self.fit_params, os.path.join(checkpoint_path, 'params.json'))
def __init__(self, in_channels, out_channels, hiddens=[16], activations=['relu'], dropout=0.5, weight_decay=5e-4, lr=0.01, use_bias=True): super().__init__() self.convs = [] inc = in_channels for hidden, activation in zip(hiddens, activations): layer = GraphConv(inc, hidden, bias=use_bias, activation=get(activation)) self.convs.append(layer) inc = hidden layer = GraphConv(inc, out_channels, bias=use_bias) self.convs.append(layer) self.dropout = layers.Dropout(dropout) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy']) self.weight_decay = weight_decay self.metric = SparseCategoricalAccuracy()
def main(cfg: DictConfig): train_ds, val_ds = create_dataset(cfg=cfg, output_dir=os.getenv( "OUTPUT_DIR", "../data/processed")) model = create_model(cfg=cfg) optimizer = SGD(learning_rate=cfg.TRAINING.LEARNING_RATE, momentum=cfg.TRAINING.MOMENTUM) loss_object = SparseCategoricalCrossentropy(from_logits=True) tb_callback = tf.keras.callbacks.TensorBoard(log_dir=os.getcwd()) model.compile( optimizer=optimizer, loss=loss_object, metrics=[ SparseCategoricalAccuracy(), MeanIouWithLogits(num_classes=cfg.DATASET.NUM_CLASSES) ], ) model.fit(train_ds, epochs=cfg.TRAINING.EPOCHS, validation_data=val_ds, callbacks=[tb_callback])
def create_model(args, num_classes=5): def create_regularizer(): if args.l1 == None: return None if args.l2 == None else regularizers.l2(args.l2) else: return regularizers.l1( argsl1) if args.l2 == None else regularizers.l1_l2(l1=args.l1, l2=args.l2) def purgeNones( layers ): # Used to remove layers that have been set to None (e.g. unused Dropout) return [layer for layer in layers if layer is not None] product = Sequential( purgeNones([ Rescaling(1. / 255), Conv2D(32, 3, activation='relu'), MaxPooling2D(), Conv2D(32, 3, activation='relu'), MaxPooling2D(), Conv2D(32, 3, activation='relu'), MaxPooling2D(), Flatten(), Dropout(args.dropout) if args.dropout else None, Dense(128, activation='relu', kernel_regularizer=create_regularizer()), Dropout(args.dropout) if args.dropout else None, Dense(num_classes) ])) product.compile(optimizer='adam', loss=SparseCategoricalCrossentropy(from_logits=True), metrics=[SparseCategoricalAccuracy()]) return product
attack = cgf['ATTACK']['name'] original_data = cgf['DATASET']['arguments']['original_images'] adv_data = cgf['DATASET']['arguments']["adv_images"] adv_labels = cgf['DATASET']['arguments']['adv_labels'] adv_diffs = cgf['DATASET']['arguments']['adv_diffs'] weights_path = join(model_path, filepath) optimizer = cgf['TRAIN']['optim']['type'] loss_type = cgf['TRAIN']['loss']['type'] metric_list = list(cgf['TRAIN']['metrics'].values()) if loss_type == 'SparseCategoricalCrossentropy': loss_type = SparseCategoricalCrossentropy(from_logits=False) metric_list = [SparseCategoricalAccuracy()] output_shape = 2 labels = np.reshape(labels, (-1)) model_name = cgf['MODEL']['name'] model_arguments = cgf['MODEL']['arguments'] #model = mb.model_selector(model_name, input_shape, output_shape, model_arguments) model = tf.keras.models.load_model(weights_path) # Preprocessing if model_name =='resnet': preprocessing = res_prep data = 255*data data = data - 122
from tensorflow.keras.losses import SparseCategoricalCrossentropy from tensorflow.keras.metrics import SparseCategoricalAccuracy from tensorflow.keras.datasets import mnist (train_x, train_y), (test_x, test_y) = mnist.load_data() train_x, test_x = train_x / 255.0, test_x / 255.0 print('train data:', train_x.shape, train_y.shape) print('test data: ', test_x.shape, test_y.shape) model = Sequential() model.add(Flatten()) model.add( Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.L2())) model.add(Dense(10, activation='softmax')) model.compile(optimizer=Adam(), loss=SparseCategoricalCrossentropy(from_logits=False), metrics=[SparseCategoricalAccuracy()]) model.fit(train_x, train_y, batch_size=32, epochs=10, validation_data=(test_x, test_y), validation_freq=5) model.summary()
def call(self, inputs): x, a = inputs x = self.conv1([x, a]) x = self.conv2([x, a]) output = self.flatten(x) output = self.fc1(output) output = self.fc2(output) return output # Create model model = Net() optimizer = Adam() loss_fn = SparseCategoricalCrossentropy() acc_fn = SparseCategoricalAccuracy() # Training function @tf.function def train_on_batch(inputs, target): with tf.GradientTape() as tape: predictions = model(inputs, training=True) loss = loss_fn(target, predictions) + sum(model.losses) acc = acc_fn(target, predictions) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) return loss, acc
# Add classification layer self.fc2 = Dense(18, activation='softmax') def call(self, inputs): x = self.base_model(inputs) x = self.pool(x) x = self.fc2(x) return x optimizer = SGD(learning_rate=params['learning_rate'], momentum=params['momentum']) loss_object = SparseCategoricalCrossentropy() # Define our metrics train_loss = Mean('train_loss', dtype=tf.float32) train_accuracy = SparseCategoricalAccuracy('train_accuracy') eval_loss = Mean('eval_loss', dtype=tf.float32) eval_accuracy = SparseCategoricalAccuracy('eval_accuracy') current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") train_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/train' test_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/test' img_train_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/img_train' img_eval_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/img_eval' @tf.function def train_step(sample, model): with tf.GradientTape() as tape: features, labels = sample logits = model(features, training=True)
def train(self, sequence_length=30, batch_size=32, epochs=50, rate=5e-5, epsilon=1e-8): ################################### # --------- Import data --------- # # Import data from csv data_path = self.config[INTENT_DATA_PATH] data = pd.read_csv(data_path) # Select required columns data = data[['Intent', 'Questions', 'Synonyms ID']] # Train data prepare x = [] y = [] # Split into x and y list from dataset # Load synonyms map f = open('/content/drive/My Drive/synonyms.json', encoding='utf-8') f2 = open('/content/drive/My Drive/global_synonyms.json', encoding='utf-8') intent_synonyms = json.load(f) global_synonyms = json.load(f2) for questions, intent, synonyms in zip(data['Questions'], data['Intent'], data['Synonyms ID']): q = questions.split('#') s = [] if not pd.isnull(synonyms): s = synonyms.split(',') syn_dicts = {} for d in s: syn_dicts[d] = intent_synonyms[d] for d in global_synonyms: syn_dicts[d] = global_synonyms[d] for q1 in q: all_similary_questions = generate_similary_sentences( (q1.strip(), syn_dicts)) x.extend(all_similary_questions) y.extend([intent] * len(all_similary_questions)) f.close() f2.close() # Intent mapping for future uses intents_count = Counter(y) self.intent_to_idx = { intent: i for i, intent in enumerate(intents_count) } self.idx_to_intent = { i: intent for i, intent in enumerate(intents_count) } # Intent list to index number for training y = [self.intent_to_idx[intent] for intent in y] # Shuffle train data temp = list(zip(x, y)) random.shuffle(temp) x, y = zip(*temp) x = list(x) y = tf.constant(list(y)) # Tokenize the input x = batch_word_segmentation(x) x = self.tokenizer( text=x, return_tensors='tf', add_special_tokens=True, # add [CLS], [SEP] max_length= sequence_length, # max length of the text that can go to BERT padding='max_length', # add [PAD] tokens return_attention_mask= True, # add attention mask to not focus on pad tokens truncation=True) # # Split into train and test # data, data_test = train_test_split(data, train_size=20000, test_size=5000, stratify=data[['num']]) ################################### # ------- Build the model ------- # model = self.build_model(len(intents_count)) ################################### # ------- Train the model ------- # # Hyperparameters # Set an optimizer optimizer = Adam(learning_rate=rate, epsilon=epsilon, decay=0.01, clipnorm=1.0) # Set loss and metrics loss = {'intent': SparseCategoricalCrossentropy(from_logits=True)} metric = {'intent': SparseCategoricalAccuracy('accuracy')} # Compile the model model.compile(optimizer=optimizer, loss=loss, metrics=metric) # Fit the model model.fit( x={ 'input_ids': x['input_ids'], # 'token_type_ids': x['token_type_ids'], 'attention_mask': x['attention_mask'] }, y={'intent': y}, batch_size=batch_size, epochs=epochs) self.model = model # Store map datas to file for future uses map_datas = {OBJ2IDX: self.intent_to_idx, IDX2OBJ: self.idx_to_intent} pickle_file(map_datas, self.config[INTENT_MAP_FILE_PATH])