num_labels, use_one_hot_embeddings):

"""Creates a classification model."""

model = modeling.BertModel(

config=bert_config,

is_training=is_training,

input_ids=input_ids,

input_mask=input_mask,

token_type_ids=segment_ids,

use_one_hot_embeddings=use_one_hot_embeddings)

# Use "pooled_output" for classification tasks on an entire sentence.

# Use "sequence_outputs" for token-level output.

output_layer = model.get_sequence_output() # output_layer: [N, max_length, 768]

hidden_size = output_layer.shape[-1].value

# Create our own layer to tune for politeness data. shape:[N, max_length, num_labels]

with tf.variable_scope("softmax_llq", reuse=tf.AUTO_REUSE):

output_weights = tf.get_variable("output_weights", [num_labels, hidden_size],

initializer=tf.truncated_normal_initializer(stddev=0.02))

output_bias = tf.get_variable("output_bias", [num_labels], initializer=tf.zeros_initializer())

with tf.variable_scope("loss"):

# Dropout helps prevent overfitting

output_layer = tf.layers.dropout(output_layer, rate=0.1, training=is_training)

logits = tf.einsum('nth,hl->ntl', output_layer, tf.transpose(output_weights))

# logits = tf.matmul(output_layer, output_weights, transpose_b=True)

logits = tf.nn.bias_add(logits, output_bias)

log_probs = tf.nn.log_softmax(logits, axis=-1)

# Convert labels into one-hot encoding

one_hot_labels = optimization.create_optimizer_labels = tf.one_hot(label_id_list, depth=num_labels, dtype=tf.float32, axis=-1)

# predicted_labels shape: [N, length]

predicted_labels = tf.squeeze(tf.argmax(log_probs, axis=-1, output_type=tf.int32))

# If we're predicting, we want predicted labels and the probabiltiies.

if is_predicting:

return (predicted_labels, log_probs)

# If we're train/eval, compute loss between predicted and actual label

per_example_loss = tf.reduce_sum(tf.reduce_sum(one_hot_labels * log_probs, axis=-1), axis=-1)

loss = -tf.reduce_mean(per_example_loss)

use_one_hot_embeddings, is_training, is_predicting):

input_ids = features["input_ids"]

input_mask = features["input_mask"]

segment_ids = features["segment_ids"]

label_id_list = features["label_id_list"]

sentiment_ids = features["sentiment_id"]

texts = features["texts"]

selected_texts = features["selected_texts"]

# TRAIN and EVAL

if not is_predicting:

(loss, predicted_labels, log_probs) = create_model(

bert_config, is_training, is_predicting, input_ids, input_mask, segment_ids, label_id_list, num_labels,

use_one_hot_embeddings)

train_op = optimization.create_optimizer(

loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu=False)

global_step = tf.train.get_or_create_global_step()

tf.summary.scalar("loss", loss)

tf.summary.scalar("global_step", global_step)

summaries = tf.summary.merge_all()

return loss, train_op, global_step, summaries, predicted_labels, sentiment_ids, texts, selected_texts

else:

(predicted_labels, log_probs) = create_model(

bert_config, is_training, is_predicting, input_ids, input_mask, segment_ids, label_id_list, num_labels,

use_one_hot_embeddings)

decoded_texts = []

for i, text in enumerate(texts):

if type(text) == type(b""):

text = text.decode("utf-8")

# sentiment "neutral" or length < 2

if sentiment_ids[i] == 0 or len(text.split()) < 2:

decoded_texts.append(text)

else:

text_list = text.lower().split()

text_token = tokenizer.tokenize(text)

segment_id = []

# record the segment id

j_text = 0

j_token = 0

while j_text < len(text_list) and j_token < len(text_token):

_j_token = j_token + 1

text_a = "".join(tokenizer.tokenize(text_list[j_text])).replace("##", "")

while True:

segment_id.append(j_text)

if "".join(text_token[j_token:_j_token]).replace("##", "") == text_a:

j_token = _j_token

break

_j_token += 1

j_text += 1

assert len(segment_id) == len(text_token)

# get selected_text

selected_text = []

predicted_label_id = predicted_labels[i]

predicted_label_id.pop(0)

for _ in range(len(predicted_label_id) - len(text_token)):

predicted_label_id.pop()

max_len = len(predicted_label_id)

assert len(text_token) == max_len

j = 0

while j < max_len:

if predicted_label_id[j] == 1:

if j == max_len - 1:

j += 1

else:

a_selected_text = text_list[segment_id[j]]

selected_text.append(a_selected_text)

for new_j in range(j + 1, len(segment_id)):

if segment_id[j] != segment_id[new_j]:

j = new_j

break

elif new_j == len(segment_id) - 1:

j = new_j

else:

j += 1

decoded_texts.append(" ".join(selected_text))