def train(number_of_hidden_lstm_layers, lstm_units, dense_units):
histories = []
public_predictions = []
private_predictions = []
MODEL_NAME = "BiLSTM" + str(number_of_hidden_lstm_layers) + "-" + str(
lstm_units)
print("\nBiLSTM" + str(number_of_hidden_lstm_layers) + "-" +
str(lstm_units))
PATH = 'BiLSTM/' + MODEL_NAME
for i in range(MODEL_NO):
with strategy.scope():
# create model
BiLSTM = create_BiLSTM(embedding_matrix,
number_of_hidden_lstm_layers, lstm_units,
dense_units)
BiLSTM.summary()
n_steps = len(train_df) // BATCH_SIZE
history = BiLSTM.fit(x=train_dataset,
validation_data=validation_dataset,
callbacks=[es],
epochs=EPOCHS,
verbose=1,
steps_per_epoch=n_steps)
histories.append(history)
public_predictions.append(
BiLSTM.predict(public_test_dataset, verbose=1)[0].flatten())
private_predictions.append(
BiLSTM.predict(private_test_dataset, verbose=1)[0].flatten())
#save memmory
tf.tpu.experimental.initialize_tpu_system(tpu)
y_public_pred = np.average(public_predictions, axis=0)
y_private_pred = np.average(private_predictions, axis=0)
# save its graph
tf.keras.utils.plot_model(
BiLSTM,
show_shapes=True,
show_layer_names=False,
to_file=
'/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/models/BiLSTM/'
+ MODEL_NAME + '/BiLSTM.png')
evaluate(y_public_pred, y_private_pred, test_public_df, test_private_df,
PATH, MODEL_NAME)
plot_history_for_accuracy_and_loss(histories, PATH)
# delete to save memmory
del BiLSTM
del y_public_pred
del y_private_pred
del histories
gc.collect()
def train():
histories = []
public_predictions = []
private_predictions = []
MODEL_NAME = "TextCNNBase"
print("\nTextCNNBase")
PATH = 'TextCNN/' + MODEL_NAME
for i in range(MODEL_NO):
with strategy.scope():
# create model
TextCNN = create_TextCNN(embedding_matrix)
TextCNN.summary()
n_steps = train_df_len // BATCH_SIZE
history = TextCNN.fit(x=train_dataset,
validation_data=validation_dataset,
callbacks=[es],
epochs=EPOCHS,
verbose=1,
steps_per_epoch=n_steps)
histories.append(history)
public_predictions.append(
TextCNN.predict(public_test_dataset, verbose=1)[0].flatten())
private_predictions.append(
TextCNN.predict(private_test_dataset, verbose=1)[0].flatten())
#save memmory
tf.tpu.experimental.initialize_tpu_system(tpu)
y_public_pred = np.average(public_predictions, axis=0)
y_private_pred = np.average(private_predictions, axis=0)
# save its graph
tf.keras.utils.plot_model(
TextCNN,
show_shapes=True,
show_layer_names=False,
to_file=
'/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/models/TextCNN/'
+ MODEL_NAME + '/TextCNN.png')
evaluate(y_public_pred, y_private_pred, test_public_df, test_private_df,
PATH, MODEL_NAME)
plot_history_for_accuracy_and_loss(histories, PATH)
# delete to save memmory
del TextCNN
del y_public_pred
del y_private_pred
del histories
gc.collect()
test_private_df = pd.read_csv("/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/data/test_private_cleared.csv")
#test_private_df = test_private_df.loc[:, ['toxicity', 'comment_text'] + IDENTITY_COLUMNS ].dropna()[:1000]
def get_predictions(modelPATH):
y_public_pred = np.load(modelPATH + '/y_public_pred.npy')
y_private_pred = np.load(modelPATH + '/y_private_pred.npy')
return y_public_pred , y_private_pred
"""# BERT"""
PATH = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/models/BERT/BERT-with-max-avg-pool/'
versions = sorted(os.listdir(PATH))
for version in versions[:4]:
y_public_pred ,y_private_pred = get_predictions(os.path.join(PATH,version))
evaluate(y_public_pred,y_private_pred, test_public_df, test_private_df,'BERT/BERT-with-max-avg-pool/' + version, version )
PATH = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/models/RoBERTa/RoBERTa-with-max-avg-pool/'
versions = sorted(os.listdir(PATH))
for version in versions[:4]:
y_public_pred ,y_private_pred = get_predictions(os.path.join(PATH,version))
evaluate(y_public_pred,y_private_pred, test_public_df, test_private_df,'RoBERTa/RoBERTa-with-max-avg-pool/' + version, version )
PATH = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/models/gpt2/gpt2-with-max-avg-pool/'
versions = sorted(os.listdir(PATH))
for version in versions[:4]:
y_public_pred ,y_private_pred = get_predictions(os.path.join(PATH,version))
evaluate(y_public_pred,y_private_pred, test_public_df, test_private_df,'gpt2/gpt2-with-max-avg-pool/' + version, version )
PATH = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/models/BiLSTM/'
versions = sorted(os.listdir(PATH))
BATCH_SIZE = 64
EPOCHS = 4
MAX_LEN = 180
BUFFER_SIZE = np.ceil(1804874 * 0.8)
MODEL = 'roberta-base'
AUTO = tf.data.experimental.AUTOTUNE
train_inputs_ds = get_dataset('train').repeat().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).prefetch(AUTO)
gc.collect()
val_inputs_ds = get_dataset('val').batch(BATCH_SIZE).cache().prefetch(AUTO)
gc.collect()
test_public_inputs_ds = get_dataset('test_public',forTest = True)
gc.collect()
test_private_inputs_ds = get_dataset('test_private',forTest = True)
gc.collect()
with strategy.scope():
roberta_layer = TFAutoModel.from_pretrained(MODEL)
model = createRoBERTa(roberta_layer)
model.summary()
tf.keras.utils.plot_model(model, show_shapes= True ,show_layer_names=False,
to_file='/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/models/RoBERTa/RoBERTa-with-max-avg-pool/RoBERTawPOOL.png')
n_steps = BUFFER_SIZE // BATCH_SIZE
for epoch in range(EPOCHS):
model.fit(x = train_inputs_ds,validation_data=val_inputs_ds , epochs = 1
,verbose = 1 , steps_per_epoch=n_steps )
y_public_pred = model.predict(test_public_inputs_ds, verbose=1 )
y_private_pred = model.predict(test_private_inputs_ds, verbose=1 )
evaluate(y_public_pred[0],y_private_pred[0], test_public_df, test_private_df, 'RoBERTa/RoBERTa-with-max-avg-pool/' + str(epoch+1) + '_epochs' , MODEL)
axis=0)
y_private_pred = np.average(
[y_private_pred1, y_private_pred2, y_private_pred3, y_private_pred4],
weights=weights,
axis=0)
return y_public_pred, y_private_pred
"""# RoBERTawPool - BiLSTM2-64"""
y_public_pred, y_private_pred = get_predictions(
'RoBERTa/RoBERTa-with-max-avg-pool/2_epochs',
'BiLSTM/BiLSTM2-64',
weights=[0.5, 0.5])
evaluate(y_public_pred, y_private_pred, test_public_df, test_private_df,
'Blend/RoBERTawPool - BiLSTM2-64', 'Blend')
"""# RoBERTawPool - GPT2wPool """
y_public_pred, y_private_pred = get_predictions(
'RoBERTa/RoBERTa-with-max-avg-pool/2_epochs',
'gpt2/gpt2-with-max-avg-pool/3_epochs',
weights=[0.6, 0.4])
evaluate(y_public_pred, y_private_pred, test_public_df, test_private_df,
'Blend/RoBERTawPool - GPT2wPool', 'Blend')
"""# RoBERTawPool - BiGRU2-64"""
y_public_pred, y_private_pred = get_predictions(
'RoBERTa/RoBERTa-with-max-avg-pool/2_epochs',
'BiGRU/BiGRU2-64',
weights=[0.5, 0.5])
evaluate(y_public_pred, y_private_pred, test_public_df, test_private_df,