def load_svhn():
svhn_train = loadmat('./data/train_32x32.mat')
svhn_test = loadmat('./data/test_32x32.mat')
svhn_train_im = svhn_train['X']
svhn_train_im = svhn_train_im.transpose(3, 2, 0, 1).astype(np.float32)
svhn_label = dense_to_one_hot(svhn_train['y'])
svhn_test_im = svhn_test['X']
svhn_test_im = svhn_test_im.transpose(3, 2, 0, 1).astype(np.float32)
svhn_label_test = dense_to_one_hot(svhn_test['y'])
return svhn_train_im, svhn_label, svhn_test_im, svhn_label_test
def train():
# 读取record
train_images, train_labels = read_record(record_train_name)
validate_images, validate_labels = read_record(record_validate_name)
# 获取batch
train_image_batch, train_label_batch = get_batch(train_images, train_labels, BATCH_SIZE)
validata_image_batch, validate_label_batch = get_batch(validate_images, validate_labels, BATCH_SIZE)
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.int32, name='y-input')
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))
weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))
y = inference(x, weights1, biases1, weights2, biases2)
# 计算交叉熵及其平均值
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_)
cross_entropy_mean = tf.reduce_mean(cross_entropy)
# 正则化
regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
regularaztion = regularizer(weights1) + regularizer(weights2)
loss = cross_entropy_mean + regularaztion
# 优化器
train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss)
# 计算正确率
correct_prediction = tf.equal(tf.arg_max(y, 1), tf.arg_max(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# 初始化会话，并开始训练过程。
with tf.Session() as sess:
tf.global_variables_initializer().run()
threads = tf.train.start_queue_runners(sess=sess)
for i in range(500):
train_x, train_y = sess.run([train_image_batch, train_label_batch])
train_y = utils.dense_to_one_hot(train_y, 2)
sess.run(train_step, feed_dict={x: train_x, y_: train_y})
# if i % 10 == 0:
# print(sess.run(tf.arg_max(y, 1), feed_dict={x: xs, y_: ys}))
# print(sess.run(tf.arg_max(ys, 1)))
if i % 10 == 0:
print("\n%d steps" % i)
print("loss --> %g " % sess.run(loss, feed_dict={x: train_x, y_: train_y}))
validate_x, validate_y = sess.run([validata_image_batch, validate_label_batch])
validate_y = utils.dense_to_one_hot(validate_y, 2)
print("accuracy --> %g" % sess.run(accuracy, feed_dict={x: validate_x, y_: validate_y}))
def extract_normalize_images(self):
x_train_valid = self.data_df.iloc[:, 1:].values.reshape(
-1, self.img_h, self.img_w, 1) # (42000,28,28,1) array
x_train_valid = x_train_valid.astype(
np.float) # convert from int64 to float32
x_train_valid = utils.normalize_data(x_train_valid)
x_test = self.test_df.iloc[:, 0:].values.reshape(
-1, self.img_h, self.img_w, 1) # (28000,28,28,1) array
x_test = x_test.astype(np.float)
x_test = utils.normalize_data(x_test)
image_size = 784
# extract image labels
y_train_valid_labels = self.data_df.iloc[:,
0].values # (42000,1) array
labels_count = np.unique(y_train_valid_labels).shape[0]
# number of different labels = 10
#plot some images and labels
#plt.figure(figsize=(15,9))
#for i in range(50):
# plt.subplot(5,10,1+i)
# plt.title(y_train_valid_labels[i])
# plt.imshow(x_train_valid[i].reshape(28,28), cmap=cm.binary)
# labels in one hot representation
y_train_valid = utils.dense_to_one_hot(y_train_valid_labels,
labels_count).astype(np.uint8)
return (x_train_valid, y_train_valid, x_test)
def labeling_fuc(data, output1, output2, true_label, threshold=0.9):
id = np.equal(np.argmax(output1, 1), np.argmax(output2, 1))
output1 = output1[id, :]
output2 = output2[id, :]
data = data[id, :]
true_label = true_label[id, :]
max1 = np.max(output1, 1)
max2 = np.max(output2, 1)
id2 = np.max(np.vstack((max1, max2)), 0) > threshold
output1 = output1[id2, :]
data = data[id2, :]
pseudo_label = utils.dense_to_one_hot(np.argmax(output1, 1), 10)
true_label = true_label[id2, :]
return data, pseudo_label, true_label
def pad_sequence(self, word_ids, char_ids, labels=None):
if labels:
labels, _ = pad_sequences(labels, 0)
labels = np.asarray(labels)
labels = dense_to_one_hot(labels, len(self.vocab_tag), nlevels=2)
word_ids, sequence_lengths = pad_sequences(word_ids, 0)
word_ids = np.asarray(word_ids)
if self.char_feature:
char_ids, word_lengths = pad_sequences(char_ids,
pad_tok=0,
nlevels=2)
char_ids = np.asarray(char_ids)
return [word_ids, char_ids], labels
else:
return [word_ids], labels
def _read_labels_csv_from(path, num_classes, one_hot=False):
"""Reads
Args:
Returns:
"""
print('Reading labels')
with open(os.path.join(path), 'r') as dest_f:
data_iter = csv.reader(dest_f)
train_labels = [data for data in data_iter]
train_labels = np.array(train_labels, dtype=np.uint32)
if one_hot:
labels_one_hot = utils.dense_to_one_hot(train_labels, num_classes)
labels_one_hot = np.asarray(labels_one_hot)
return labels_one_hot
return train_labels
def _read_labels_csv_from(path, num_classes, one_hot=False):
"""Reads
Args:
Returns:
"""
print('Reading labels')
with open(os.path.join(path), 'r') as dest_f:
data_iter = csv.reader(dest_f)
train_labels = [data for data in data_iter]
train_labels = np.array(train_labels, dtype=np.uint32)
if one_hot:
labels_one_hot = utils.dense_to_one_hot(train_labels, num_classes)
labels_one_hot = np.asarray(labels_one_hot)
return labels_one_hot
return train_labels
def evaluate():
with tf.Graph().as_default() as g:
model = crnn.CRNN('test')
model._build_model()
#load training data
test_data, test_label, valid_data, valid_label, Valid_label, Test_label, pernums_test, pernums_valid = load_data(
)
# test, valid segment size
test_size = test_data.shape[0]
valid_size = valid_data.shape[0]
# for hole sentence label
test_label = dense_to_one_hot(test_label, 4)
valid_label = dense_to_one_hot(valid_label, 4)
# for segement label
Test_label = dense_to_one_hot(Test_label, 4)
Valid_label = dense_to_one_hot(Valid_label, 4)
# for sgement type : 1 :for hole sentence, 2: for sgement sentecne
tnum = pernums_test.shape[0]
vnum = pernums_valid.shape[0]
pred_test_uw = np.empty((tnum, 4), dtype=np.float32)
pred_test_w = np.empty((tnum, 4), dtype=np.float32)
valid_iter = divmod((valid_size), FLAGS.valid_batch_size)[0]
test_iter = divmod((test_size), FLAGS.test_batch_size)[0]
y_pred_valid = np.empty((valid_size, 4), dtype=np.float32)
y_pred_test = np.empty((test_size, 4), dtype=np.float32)
y_test = np.empty((tnum, 4), dtype=np.float32)
y_valid = np.empty((vnum, 4), dtype=np.float32)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=model.labels, logits=model.logits)
variable_averages = tf.train.ExponentialMovingAverage(FLAGS.momentum)
variable_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variable_to_restore)
#saver = tf.train.Saver()
flag = False
best_valid_uw = 0
best_valid_w = 0
for i in range(5):
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
#for validation data
index = 0
cost_valid = 0
if (valid_size < FLAGS.valid_batch_size):
validate_feed = {
model.inputs: valid_data,
model.labels: Valid_label
}
y_pred_valid, loss = sess.run(
[model.softmax, cross_entropy],
feed_dict=validate_feed)
cost_valid = cost_valid + np.sum(loss)
else:
for v in range(valid_iter):
v_begin = v * FLAGS.valid_batch_size
v_end = (v + 1) * FLAGS.valid_batch_size
if (v == valid_iter - 1):
if (v_end < valid_size):
v_end = valid_size
validate_feed = {
model.inputs: valid_data[v_begin:v_end],
model.labels: Valid_label[v_begin:v_end]
}
loss, y_pred_valid[v_begin:v_end, :] = sess.run(
[cross_entropy, model.softmax],
feed_dict=validate_feed)
cost_valid = cost_valid + np.sum(loss)
cost_valid = cost_valid / valid_size
print(y_pred_valid)
valid_acc_uw = recall(np.argmax(Valid_label, 1),
np.argmax(y_pred_valid, 1),
average='macro')
valid_acc_w = recall(np.argmax(Valid_label, 1),
np.argmax(y_pred_valid, 1),
average='weighted')
valid_conf = confusion(np.argmax(Valid_label, 1),
np.argmax(y_pred_valid, 1))
print('----------segment metrics---------------')
print("Best valid_UA: %3.4g" % best_valid_uw)
print("Best valid_WA: %3.4g" % best_valid_w)
print('Valid Confusion Matrix:["ang","sad","hap","neu"]')
print(valid_conf)
print('----------segment metrics---------------')
for s in range(vnum):
y_valid[s, :] = np.max(
y_pred_valid[index:index + pernums_valid[s], :], 0)
index += pernums_valid[s]
valid_acc_uw = recall(np.argmax(valid_label, 1),
np.argmax(y_valid, 1),
average='macro')
valid_acc_w = recall(np.argmax(valid_label, 1),
np.argmax(y_valid, 1),
average='weighted')
valid_conf = confusion(np.argmax(valid_label, 1),
np.argmax(y_valid, 1))
#for test set
index = 0
for t in range(test_iter):
t_begin = t * FLAGS.test_batch_size
t_end = (t + 1) * FLAGS.test_batch_size
if (t == test_iter - 1):
if (t_end < test_size):
t_end = test_size
#print t_begin,t_end,t,test_iter
test_feed = {
model.inputs: test_data[t_begin:t_end],
model.labels: Test_label[t_begin:t_end]
}
y_pred_test[t_begin:t_end, :] = sess.run(
model.logits, feed_dict=test_feed)
for s in range(tnum):
y_test[s, :] = np.max(
y_pred_test[index:index + pernums_test[s], :], 0)
index = index + pernums_test[s]
if valid_acc_uw > best_valid_uw:
best_valid_uw = valid_acc_uw
pred_test_uw = y_test
test_acc_uw = recall(np.argmax(test_label, 1),
np.argmax(y_test, 1),
average='macro')
test_conf = confusion(np.argmax(test_label, 1),
np.argmax(y_test, 1))
confusion_uw = test_conf
flag = True
if valid_acc_w > best_valid_w:
best_valid_w = valid_acc_w
pred_test_w = y_test
test_acc_w = recall(np.argmax(test_label, 1),
np.argmax(y_test, 1),
average='weighted')
test_conf = confusion(np.argmax(test_label, 1),
np.argmax(y_test, 1))
confusion_w = test_conf
flag = True
print(
"*****************************************************************"
)
print(global_step)
print("Epoch: %s" % global_step)
print("Valid cost: %2.3g" % cost_valid)
print("Valid_UA: %3.4g" % valid_acc_uw)
print("Valid_WA: %3.4g" % valid_acc_w)
print("Best valid_UA: %3.4g" % best_valid_uw)
print("Best valid_WA: %3.4g" % best_valid_w)
print('Valid Confusion Matrix:["ang","sad","hap","neu"]')
print(valid_conf)
print("Test_UA: %3.4g" % test_acc_uw)
print("Test_WA: %3.4g" % test_acc_w)
print('Test Confusion Matrix:["ang","sad","hap","neu"]')
print(confusion_uw)
print(
"*****************************************************************"
)
if (flag):
f = open(FLAGS.pred_name, 'wb')
pickle.dump((
best_valid_uw,
best_valid_w,
pred_test_w,
test_acc_w,
confusion_w,
pred_test_uw,
test_acc_uw,
confusion_uw,
), f)
f.close()
flag = False
def do_test(fname_model, test_file_dir, norm_dir, data_len, is_default,
model_type):
eval_input_dir = test_file_dir
eval_output_dir = test_file_dir + '/Labels'
graph = load_graph(fname_model)
w = 19
u = 9
# [print(n.name) for n in graph.as_graph_def().node]
# for op in graph.get_operations():
# print(op.name)
final_softout = []
final_label = []
if model_type == 0: # acam
import data_reader_bDNN_v2 as dr
print(os.path.abspath('./configure/ACAM'))
sys.path.insert(0, os.path.abspath('./configure/ACAM'))
import config as cg
if is_default:
w = 19
u = 9
valid_batch_size = 4096
else:
w = cg.w
u = cg.u
valid_batch_size = cg.batch_size
valid_data_set = dr.DataReader(eval_input_dir,
eval_output_dir,
norm_dir,
w=w,
u=u,
name="eval")
node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0')
node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0')
node_keep_probability = graph.get_tensor_by_name(
'prefix/model_1/keep_probabilty:0')
node_logits = graph.get_tensor_by_name('prefix/model_1/logits:0')
node_raw_labels = graph.get_tensor_by_name(
'prefix/model_1/raw_labels:0')
while True:
valid_inputs, valid_labels = valid_data_set.next_batch(
valid_batch_size)
feed_dict = {
node_inputs: valid_inputs,
node_labels: valid_labels,
node_keep_probability: 1
}
if valid_data_set.eof_checker():
final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
final_label = np.reshape(np.asarray(final_label), [-1, 1])
valid_data_set.reader_initialize()
# print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
break
with tf.Session(graph=graph) as sess:
logits, raw_labels = sess.run([node_logits, node_raw_labels],
feed_dict=feed_dict)
soft_pred = bdnn_prediction(valid_batch_size,
logits,
threshold=0.6,
w=w,
u=u)[1]
raw_labels = raw_labels.reshape((-1, 1))
final_softout.append(soft_pred)
final_label.append(raw_labels)
# if valid_data_set.eof_checker():
# final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
# final_label = np.reshape(np.asarray(final_label), [-1, 1])
# valid_data_set.reader_initialize()
# # print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
# break
return final_softout[0:data_len, :], final_label[0:data_len, :]
if model_type == 1: # bdnn
import data_reader_bDNN_v2 as dr
print(os.path.abspath('./configure/bDNN'))
sys.path.insert(0, os.path.abspath('./configure/bDNN'))
import config as cg
if is_default:
w = 19
u = 9
valid_batch_size = 4096
else:
w = cg.w
u = cg.u
valid_batch_size = cg.batch_size
valid_data_set = dr.DataReader(
eval_input_dir, eval_output_dir, norm_dir, w=w, u=u,
name="eval") # training data reader initialization
node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0')
node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0')
node_keep_probability = graph.get_tensor_by_name(
'prefix/model_1/keep_probabilty:0')
node_logits = graph.get_tensor_by_name('prefix/model_1/logits:0')
while True:
valid_inputs, valid_labels = valid_data_set.next_batch(
valid_batch_size)
feed_dict = {
node_inputs: valid_inputs,
node_labels: valid_labels,
node_keep_probability: 1
}
if valid_data_set.eof_checker():
final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
final_label = np.reshape(np.asarray(final_label), [-1, 1])
valid_data_set.reader_initialize()
# print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
break
with tf.Session(graph=graph) as sess:
logits, labels = sess.run([node_logits, node_labels],
feed_dict=feed_dict)
soft_pred = bdnn_prediction(valid_batch_size,
logits,
threshold=0.6,
w=w,
u=u)[1]
raw_indx = int(np.floor(labels.shape[1] / 2))
raw_labels = labels[:, raw_indx]
raw_labels = raw_labels.reshape((-1, 1))
final_softout.append(soft_pred)
final_label.append(raw_labels)
return final_softout[0:data_len, :], final_label[0:data_len, :]
if model_type == 2: # dnn
import data_reader_DNN_v2 as dnn_dr
print(os.path.abspath('./configure/DNN'))
sys.path.insert(0, os.path.abspath('./configure/DNN'))
import config as cg
if is_default:
w = 19
u = 9
valid_batch_size = 4096
else:
w = cg.w
u = cg.u
valid_batch_size = cg.batch_size
valid_data_set = dnn_dr.DataReader(eval_input_dir,
eval_output_dir,
norm_dir,
w=w,
u=u,
name="eval")
node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0')
node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0')
node_keep_probability = graph.get_tensor_by_name(
'prefix/model_1/keep_probabilty:0')
node_softpred = graph.get_tensor_by_name('prefix/model_1/soft_pred:0')
node_raw_labels = graph.get_tensor_by_name(
'prefix/model_1/raw_labels:0')
while True:
valid_inputs, valid_labels = valid_data_set.next_batch(
valid_batch_size)
one_hot_labels = valid_labels.reshape((-1, 1))
one_hot_labels = utils.dense_to_one_hot(one_hot_labels,
num_classes=2)
feed_dict = {
node_inputs: valid_inputs,
node_labels: one_hot_labels,
node_keep_probability: 1
}
if valid_data_set.eof_checker():
final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
final_label = np.reshape(np.asarray(final_label), [-1, 1])
valid_data_set.reader_initialize()
# print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
break
print("aa")
session_conf = tf.ConfigProto(device_count={
'CPU': 1,
'GPU': 0
},
allow_soft_placement=True,
log_device_placement=False)
with tf.Session(graph=graph, config=session_conf) as sess:
soft_pred, raw_labels = sess.run(
[node_softpred, node_raw_labels], feed_dict=feed_dict)
raw_labels = raw_labels.reshape((-1, 1))
final_softout.append(soft_pred)
final_label.append(raw_labels)
return final_softout[0:data_len, :], final_label[0:data_len, :]
if model_type == 3: # lstm
import data_reader_RNN as rnn_dr
print(os.path.abspath('./configure/LSTM'))
sys.path.insert(0, os.path.abspath('./configure/LSTM'))
import config as cg
if is_default:
target_delay = 5
seq_size = 20
batch_num = 200
valid_batch_size = seq_size * batch_num
else:
target_delay = cg.target_delay
seq_size = cg.seq_len
batch_num = cg.num_batches
valid_batch_size = seq_size * batch_num
valid_data_set = rnn_dr.DataReader(eval_input_dir,
eval_output_dir,
norm_dir,
target_delay=target_delay,
name="eval")
node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0')
node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0')
node_keep_probability = graph.get_tensor_by_name(
'prefix/model_1/keep_probabilty:0')
node_softpred = graph.get_tensor_by_name('prefix/model_1/soft_pred:0')
node_raw_labels = graph.get_tensor_by_name(
'prefix/model_1/raw_labels:0')
while True:
valid_inputs, valid_labels = valid_data_set.next_batch(
valid_batch_size)
one_hot_labels = valid_labels.reshape((-1, 1))
one_hot_labels = utils.dense_to_one_hot(one_hot_labels,
num_classes=2)
feed_dict = {
node_inputs: valid_inputs,
node_labels: one_hot_labels,
node_keep_probability: 1
}
if valid_data_set.eof_checker():
final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
final_label = np.reshape(np.asarray(final_label), [-1, 1])
valid_data_set.reader_initialize()
# print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
break
with tf.Session(graph=graph) as sess:
soft_pred, raw_labels = sess.run(
[node_softpred, node_raw_labels], feed_dict=feed_dict)
raw_labels = raw_labels.reshape((-1, 1))
final_softout.append(soft_pred)
final_label.append(raw_labels)
# if valid_data_set.eof_checker():
# final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
# final_label = np.reshape(np.asarray(final_label), [-1, 1])
# valid_data_set.reader_initialize()
# # print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
# break
return final_softout[0:data_len, :], final_label[0:data_len, :]
labeled_data_size = 100
num_features = 784
image_size = 28
num_classes = 10
batch_size = 10
lambda_f = 0.04
k_of_knn = 10
m_of_knn = 50
SEED = 666
np.random.seed(SEED)
data_dict = sio.loadmat('MNIST')
# normalization
samples = data_dict['fea'] / 256
labels = dense_to_one_hot(data_dict['gnd'], num_classes)
# 有标记数据
# l_index = np.random.randint(5000, size=labeled_data_size)
l_index = equal_proportion_sampling(data_dict['gnd'][:5000], num_classes,
labeled_data_size)
l_dataset = DataSet(samples[l_index], labels[l_index])
# 无标记数据
# u_index = np.arange(5000, 60000)
u_index = np.random.randint(5000, 60000, size=2000)
uindex_dataset = DataSet(u_index, labels[u_index])
sample_data = np.vstack((samples[l_index], samples[u_index]))
sample_label = np.vstack((labels[l_index], labels[u_index]))
def train(train_dir=None, model_dir=None, mode='train'):
model = crnn.CRNN(mode)
model._build_model()
global_step = tf.Variable(0, trainable=False)
#sess1 = tf.InteractiveSession()
#load training data
train_data, train_label, valid_data, Valid_label = load_data()
train_label = dense_to_one_hot(train_label, 4)
Valid_label = dense_to_one_hot(Valid_label, 4)
training_size = train_data.shape[0]
with tf.name_scope('cross_entropy'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=model.labels, logits=model.logits)
loss = tf.reduce_mean(cross_entropy)
# print model.logits.get_shape()
with tf.name_scope('accuracy'):
correct_pred = tf.equal(tf.argmax(model.softmax, 1),
tf.argmax(model.labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
with tf.name_scope("moving_average"):
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.momentum, global_step)
variable_averages_op = variable_averages.apply(
tf.trainable_variables())
with tf.name_scope("train_step"):
lr = tf.train.exponential_decay(FLAGS.learning_rate,
global_step,
training_size / FLAGS.train_batch_size,
FLAGS.decay_rate,
staircase=True)
#print (lr.eval())
train_step = tf.train.AdamOptimizer(lr).minimize(
loss, global_step=global_step)
with tf.control_dependencies([train_step, variable_averages_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for e in range(FLAGS.num_epochs):
print(type(train_label))
print(train_label.shape)
index = np.arange(training_size)
np.random.shuffle(index)
train_data = train_data[index]
train_label = train_label[index]
for i in range(int(training_size / FLAGS.train_batch_size) + 1):
start = (i * FLAGS.train_batch_size) % training_size
end = min(start + FLAGS.train_batch_size, training_size)
_, loss_value, step, acc, softmax = sess.run(
[train_op, loss, global_step, accuracy, model.softmax],
feed_dict={
model.inputs: train_data[start:end],
model.labels: train_label[start:end]
})
if i % 10 == 0:
print(
"After epoch:%d, step: %d, loss on training batch is %.2f, accuracy is %.3f."
% (e, step, loss_value, acc))
saver.save(sess,
os.path.join(FLAGS.checkpoint,
FLAGS.model_name),
global_step=global_step)
train_acc_uw = recall(np.argmax(softmax, 1),
np.argmax(train_label[start:end], 1),
average='macro')
train_acc_w = recall(np.argmax(softmax, 1),
np.argmax(train_label[start:end], 1),
average='weighted')
train_conf = confusion(
np.argmax(softmax, 1),
np.argmax(train_label[start:end], 1))
print("train_UA: %3.4g" % train_acc_uw)
print("train_WA: %3.4g" % train_acc_w)
print('Confusion Matrix:["ang","sad","hap","neu"]')
print(train_conf)
if i % 20 == 0:
#for validation data
valid_size = len(valid_data)
valid_iter = divmod((valid_size),
FLAGS.valid_batch_size)[0]
y_pred_valid = np.empty((valid_size, 4), dtype=np.float32)
index = 0
cost_valid = 0
if (valid_size < FLAGS.valid_batch_size):
validate_feed = {
model.inputs: valid_data,
model.labels: Valid_label
}
y_pred_valid, p_loss = sess.run(
[model.softmax, cross_entropy],
feed_dict=validate_feed)
cost_valid = cost_valid + np.sum(p_loss)
else:
print(valid_data.shape)
for v in range(valid_iter):
v_begin = v * FLAGS.valid_batch_size
v_end = (v + 1) * FLAGS.valid_batch_size
if (v == valid_iter - 1):
if (v_end < valid_size):
v_end = valid_size
validate_feed = {
model.inputs: valid_data[v_begin:v_end],
model.labels: Valid_label[v_begin:v_end]
}
p_loss, y_pred_valid[v_begin:v_end, :] = sess.run(
[cross_entropy, model.softmax],
feed_dict=validate_feed)
cost_valid = cost_valid + np.sum(p_loss)
cost_valid = cost_valid / valid_size
print(np.argmax(y_pred_valid, 1))
print(np.argmax(Valid_label, 1))
valid_acc_uw = recall(np.argmax(Valid_label, 1),
np.argmax(y_pred_valid, 1),
average='macro')
valid_acc_w = recall(np.argmax(Valid_label, 1),
np.argmax(y_pred_valid, 1),
average='weighted')
valid_conf = confusion(np.argmax(Valid_label, 1),
np.argmax(y_pred_valid, 1))
print('----------segment metrics---------------')
print("valid_UA: %3.4g" % valid_acc_uw)
print("valid_WA: %3.4g" % valid_acc_w)
print('Valid Confusion Matrix:["ang","sad","hap","neu"]')
print(valid_conf)
print('----------segment metrics---------------')
def do_test(fname_model, test_file_dir, norm_dir, prj_dir, is_default,
model_type):
eval_input_dir = test_file_dir
graph = load_graph(fname_model)
w = 19
u = 9
# [print(n.name) for n in graph.as_graph_def().node]
# for op in graph.get_operations():
# print(op.name)
final_softout = []
final_label = []
if model_type == 0: # acam
import data_reader_bDNN_v2 as dr
print(prj_dir + '/configure/ACAM')
sys.path.insert(0, os.path.abspath(prj_dir + 'ACAM'))
# import config as cg
#
# if is_default:
# w = 19
# u = 9
# valid_batch_size = 4096
# else:
# w = cg.w
# u = cg.u
# valid_batch_size = cg.batch_size
valid_data_set = dr.DataReader(eval_input_dir + "/train/feature_mrcg",
eval_input_dir + "/train/label",
norm_dir,
w=w,
u=u,
name="eval")
node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0')
node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0')
node_keep_probability = graph.get_tensor_by_name(
'prefix/model_1/keep_probabilty:0')
node_logits = graph.get_tensor_by_name('prefix/model_1/logits:0')
node_raw_labels = graph.get_tensor_by_name(
'prefix/model_1/raw_labels:0')
acc_sum = 0
file_num = 0
acc_sum = 0
tp_sum = 0
tn_sum = 0
fp_sum = 0
fn_sum = 0
frame_num = 0
while True:
valid_inputs, valid_labels = valid_data_set.next_batch(7)
feed_dict = {
node_inputs: valid_inputs,
node_labels: valid_labels,
node_keep_probability: 1
}
if valid_data_set.eof_checker():
final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
final_label = np.reshape(np.asarray(final_label), [-1, 1])
valid_data_set.reader_initialize()
# print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
break
with tf.Session(graph=graph) as sess:
logits, raw_labels = sess.run([node_logits, node_raw_labels],
feed_dict=feed_dict)
raw_labels = raw_labels.reshape((-1, 1))
soft_pred = bdnn_prediction(len(raw_labels),
logits,
threshold=0.6,
w=w,
u=u)[0]
eql = (raw_labels == soft_pred)
acc = np.mean(eql)
acc_sum += acc
file_num += 1
frame_s = len(raw_labels)
frame_num += len(raw_labels)
tn = 0
tp = 0
fp = 0
fn = 0
for i in range(len(soft_pred)):
if soft_pred[i] == 0 and raw_labels[i] == 0:
tn += 1
elif soft_pred[i] == 0 and raw_labels[i] == 1:
fn += 1
elif soft_pred[i] == 1 and raw_labels[i] == 0:
fp += 1
elif soft_pred[i] == 1 and raw_labels[i] == 1:
tp += 1
final_softout.append(soft_pred)
final_label.append(raw_labels)
tn_sum += tn
tp_sum += tp
fn_sum += fn
fp_sum += fp
# if acc <= 0.7:
# file_name =valid_data_set.get_cur_file_name().split('/')[-1]
# obj_name = file_name.split('.')[0]
# wav_path = "/mnt/E_DRIVE/Lipr_with_label/train/low"
# shutil.copy("/mnt/E_DRIVE/Lipr_with_label/train/" + obj_name + '.wav', wav_path)
# np.save(os.path.join("/mnt/E_DRIVE/Lipr_with_label/train/low", obj_name + '.label.npy'),
# original_label(soft_pred, "/mnt/E_DRIVE/Lipr_with_label/train/" + obj_name + '.wav'))
print(" train_accuracy=%4.4f" % (acc * 100))
# print("path is " + valid_data_set.get_cur_file_name())
print(
"true_positive: %f, false positive: %f, true negative: %f, false negative: %f"
% (tp / frame_s, fp / frame_s, tn / frame_s, fn / frame_s))
# print("accuracy: %f, true_positive: %f, false positive: %f, true negative: %f, false negative: %f" % (
# acc, tp / frame_s, fp / frame_s, tn / frame_s, fn / frame_s))
print("total accuracy: " + str(acc_sum / file_num))
print(
"total: true_positive: %f, false positive: %f, true negative: %f, false negative: %f"
% (tp_sum / frame_num, fp_sum / frame_num, tn_sum / frame_num,
fn_sum / frame_num))
return final_softout[:, :], final_label[:, :]
if model_type == 1: # bdnn
import data_reader_bDNN_v2 as dr
print(prj_dir + '/configure/bDNN')
sys.path.insert(0, os.path.abspath(prj_dir + 'bDNN'))
import config as cg
if is_default:
w = 19
u = 9
valid_batch_size = 4096
else:
w = cg.w
u = cg.u
valid_batch_size = cg.batch_size
valid_data_set = dr.DataReader(
eval_input_dir, eval_output_dir, norm_dir, w=w, u=u,
name="eval") # training data reader initialization
node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0')
node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0')
node_keep_probability = graph.get_tensor_by_name(
'prefix/model_1/keep_probabilty:0')
node_logits = graph.get_tensor_by_name('prefix/model_1/logits:0')
while True:
valid_inputs, valid_labels = valid_data_set.next_batch(
valid_batch_size)
feed_dict = {
node_inputs: valid_inputs,
node_labels: valid_labels,
node_keep_probability: 1
}
if valid_data_set.eof_checker():
final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
final_label = np.reshape(np.asarray(final_label), [-1, 1])
valid_data_set.reader_initialize()
# print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
break
with tf.Session(graph=graph) as sess:
logits, labels = sess.run([node_logits, node_labels],
feed_dict=feed_dict)
soft_pred = bdnn_prediction(valid_batch_size,
logits,
threshold=0.6,
w=w,
u=u)[1]
raw_indx = int(np.floor(labels.shape[1] / 2))
raw_labels = labels[:, raw_indx]
raw_labels = raw_labels.reshape((-1, 1))
final_softout.append(soft_pred)
final_label.append(raw_labels)
return final_softout[0:data_len, :], final_label[0:data_len, :]
if model_type == 2: # dnn
import data_reader_DNN_v2 as dnn_dr
print(prj_dir + '/configure/DNN')
sys.path.insert(0, os.path.abspath(prj_dir + 'DNN'))
import config as cg
if is_default:
w = 19
u = 9
valid_batch_size = 4096
else:
w = cg.w
u = cg.u
valid_batch_size = cg.batch_size
valid_data_set = dnn_dr.DataReader(eval_input_dir,
eval_output_dir,
norm_dir,
w=w,
u=u,
name="eval")
node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0')
node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0')
node_keep_probability = graph.get_tensor_by_name(
'prefix/model_1/keep_probabilty:0')
node_softpred = graph.get_tensor_by_name('prefix/model_1/soft_pred:0')
node_raw_labels = graph.get_tensor_by_name(
'prefix/model_1/raw_labels:0')
while True:
valid_inputs, valid_labels = valid_data_set.next_batch(
valid_batch_size)
one_hot_labels = valid_labels.reshape((-1, 1))
one_hot_labels = utils.dense_to_one_hot(one_hot_labels,
num_classes=2)
feed_dict = {
node_inputs: valid_inputs,
node_labels: one_hot_labels,
node_keep_probability: 1
}
if valid_data_set.eof_checker():
final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
final_label = np.reshape(np.asarray(final_label), [-1, 1])
valid_data_set.reader_initialize()
# print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
break
with tf.Session(graph=graph) as sess:
soft_pred, raw_labels = sess.run(
[node_softpred, node_raw_labels], feed_dict=feed_dict)
raw_labels = raw_labels.reshape((-1, 1))
final_softout.append(soft_pred)
final_label.append(raw_labels)
return final_softout[0:data_len, :], final_label[0:data_len, :]
if model_type == 3: # lstm
import data_reader_RNN as rnn_dr
print(prj_dir + '/configure/LSTM')
sys.path.insert(0, os.path.abspath(prj_dir + 'LSTM'))
import config as cg
if is_default:
target_delay = 5
seq_size = 20
batch_num = 200
valid_batch_size = seq_size * batch_num
else:
target_delay = cg.target_delay
seq_size = cg.seq_len
valid_batch_size = seq_size
valid_data_set = rnn_dr.DataReader(eval_input_dir +
"/train/feature_mrcg",
eval_input_dir + '/train/label',
norm_dir,
target_delay=target_delay,
name="eval")
node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0')
node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0')
node_keep_probability = graph.get_tensor_by_name(
'prefix/model_1/keep_probabilty:0')
node_softpred = graph.get_tensor_by_name('prefix/model_1/soft_pred:0')
node_raw_labels = graph.get_tensor_by_name(
'prefix/model_1/raw_labels:0')
while True:
valid_inputs, valid_labels = valid_data_set.next_batch(
valid_batch_size)
one_hot_labels = valid_labels.reshape((-1, 1))
one_hot_labels = utils.dense_to_one_hot(one_hot_labels,
num_classes=2)
feed_dict = {
node_inputs: valid_inputs,
node_labels: one_hot_labels,
node_keep_probability: 1
}
if valid_data_set.eof_checker():
# final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
# final_label = np.reshape(np.asarray(final_label), [-1, 1])
valid_data_set.reader_initialize()
break
with tf.Session(graph=graph) as sess:
soft_pred, raw_labels = sess.run(
[node_softpred, node_raw_labels], feed_dict=feed_dict)
raw_labels = raw_labels.reshape((-1, 1))
final_softout.append(soft_pred)
final_label.append(raw_labels)
# if valid_data_set.eof_checker():
# final_softout = np.reshape(np.asarray(final_softout), [-1, 1])
# final_label = np.reshape(np.asarray(final_label), [-1, 1])
# valid_data_set.reader_initialize()
# # print('Valid data reader was initialized!') # initialize eof flag & num_file & start index
# break
return final_softout, final_label
def extract_utterance_level_features(idx, path_save):
tf.reset_default_graph()
tf.set_random_seed(371)
filename = "%s/CV_0%s/Frame_CV_0%s.pickle" % (path_save, idx, idx)
with open(filename, 'rb') as handle:
data = pickle.load(handle)
X_Frame_train = data['X_Frame_train'].astype('float32')
y_Frame_train = data['y_Frame_train']
idx_Frame_train = data['idx_Frame_train']
X_Frame_test = data['X_Frame_test'].astype('float32')
y_Frame_test = data['y_Frame_test']
idx_Frame_test = data['idx_Frame_test']
y_train = data['y_Utter_train']
y_test = data['y_Utter_test']
n_classes = y_test.max() + 1
y_train_onehot = dense_to_one_hot(y_Frame_train, n_classes)
y_test_onehot = dense_to_one_hot(y_Frame_test, n_classes)
log('Training Deep Neural Network............................')
for d in ['/gpu:%s' % 0]:
with tf.device(d):
X = tf.placeholder(tf.float32, [None, X_Frame_train.shape[1]])
Y = tf.placeholder(tf.float32, [None, n_classes])
weights = {
#he init : stddev=np.sqrt(2 / dim_input)
'encoder_h1':
tf.Variable(
tf.random_normal([X_Frame_train.shape[1], dim_hidden_1],
stddev=np.sqrt(2 /
X_Frame_train.shape[1]))),
'encoder_h2':
tf.Variable(
tf.random_normal([dim_hidden_1, dim_hidden_2],
stddev=np.sqrt(2 / dim_hidden_1))),
'encoder_h3':
tf.Variable(
tf.random_normal([dim_hidden_2, dim_hidden_3],
stddev=np.sqrt(2 / dim_hidden_2))),
'encoder_output':
tf.Variable(
tf.random_normal([dim_hidden_3, n_classes],
stddev=np.sqrt(2 / dim_hidden_3)))
}
biases = {
'encoder_b1': tf.Variable(tf.zeros([dim_hidden_1])),
'encoder_b2': tf.Variable(tf.zeros([dim_hidden_2])),
'encoder_b3': tf.Variable(tf.zeros([dim_hidden_3])),
'encoder_output': tf.Variable(tf.zeros([n_classes]))
}
Z1 = tf.matmul(X, weights['encoder_h1']) + biases['encoder_b1']
l1 = tf.nn.relu(Z1)
Z2 = tf.matmul(l1, weights['encoder_h2']) + biases['encoder_b2']
l2 = tf.nn.relu(Z2)
Z3 = tf.matmul(l2, weights['encoder_h3']) + biases['encoder_b3']
l3 = tf.nn.relu(Z3)
output = tf.matmul(
l3, weights['encoder_output']) + biases['encoder_output']
pred = tf.nn.softmax(output)
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=Y))
optimizer = tf.train.AdamOptimizer().minimize(cost)
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
device_count={'GPU': 1},
gpu_options=gpu_options)) as sess:
sess.run(tf.global_variables_initializer())
log('Training CV %s Deep Neural Network............................' %
idx)
for epoch in range(training_epoch):
_, c = sess.run([optimizer, cost],
feed_dict={
X: X_Frame_train,
Y: y_train_onehot
})
if (epoch % display_step) == 0:
train_accuracy = sess.run([accuracy],
feed_dict={
X: X_Frame_train,
Y: y_train_onehot
})
test_accuracy = sess.run([accuracy],
feed_dict={
X: X_Frame_test,
Y: y_test_onehot
})
log("Epoch: {:02d} \t\tCost={:.2f} \tTrainAcc: {:.2f} \tTestAcc: {:.2f}"
.format(epoch, c, train_accuracy[0], test_accuracy[0]))
X_Frame_DNN_train = sess.run(pred, feed_dict={X: X_Frame_train})
X_Frame_DNN_test = sess.run(pred, feed_dict={X: X_Frame_test})
X_Frame_DNN_train, X_Frame_DNN_test = normalize_Zscore(
X_Frame_DNN_train, X_Frame_DNN_test)
utterFeatList = []
for i in range(idx_Frame_train.max() + 1):
frames = X_Frame_DNN_train[idx_Frame_train == i, :]
if frames.size != 0:
utter_feat = np.hstack([
np.amax(frames, axis=0),
np.amin(frames, axis=0),
np.mean(frames, axis=0),
np.mean(frames > 0.2, axis=0)
])
utterFeatList.append(utter_feat)
X_train = np.vstack(utterFeatList)
utterFeatList_test = []
for i in range(idx_Frame_test.max() + 1):
frames = X_Frame_DNN_test[idx_Frame_test == i, :]
if frames.size != 0:
utter_feat = np.hstack([
np.amax(frames, axis=0),
np.amin(frames, axis=0),
np.mean(frames, axis=0),
np.mean(frames > 0.2, axis=0)
])
utterFeatList_test.append(utter_feat)
X_test = np.vstack(utterFeatList_test)
log("Utterance-Level-Features are extracted.")
log("Classifying Speech Emotions using Utter-Level features............................"
)
"""Extreme Learning Machine"""
rhl = RBFRandomLayer(n_hidden=200, rbf_width=0.1)
elmr = GenELMClassifier(hidden_layer=rhl)
elmr.fit(X_train, y_train)
y_pred = elmr.predict(X_test)
test_weighted_accuracy = elmr.score(X_test, y_test)
uar = 0
cnf_matrix = confusion_matrix(y_test, y_pred)
diag = np.diagonal(cnf_matrix)
for index, i in enumerate(diag):
uar += i / collections.Counter(y_test)[index]
test_unweighted_accuracy = uar / len(cnf_matrix)
accuracy = []
accuracy.append(test_weighted_accuracy * 100)
accuracy.append(test_unweighted_accuracy * 100)
a = ['Ang', 'Hap', 'Neu', 'Sad']
# Compute confusion matrix
cnf_matrix = np.transpose(cnf_matrix)
cnf_matrix = cnf_matrix * 100 / cnf_matrix.astype(np.int).sum(axis=0)
cnf_matrix = np.transpose(cnf_matrix).astype(float)
cnf_matrix = np.around(cnf_matrix, decimals=1)
#accuracy per class
conf_mat = (cnf_matrix.diagonal() * 100) / cnf_matrix.sum(axis=1)
conf_mat = np.around(conf_mat, decimals=2)
log('==============[ [%s] ]==============' % idx)
log('Feature Dimension: %d' % X_train.shape[1])
log('Confusion Matrix:\n%s' % cnf_matrix)
log('Accuracy per classes:\n%s' % conf_mat)
log("WAR\t\t\t:\t%.2f %%" % (test_weighted_accuracy * 100))
log("UAR\t\t\t:\t%.2f %%" % (test_unweighted_accuracy * 100))
return np.around(np.array(accuracy), decimals=1)