def tf_non_stream_model_accuracy(
flags,
folder,
time_shift_samples=0,
weights_name='best_weights',
accuracy_name='tf_non_stream_model_accuracy.txt'):
"""Compute accuracy of non streamable model using TF.
Args:
flags: model and data settings
folder: folder name where accuracy report will be stored
time_shift_samples: time shift of audio data it will be applied in range:
-time_shift_samples...time_shift_samples
We can use non stream model for processing stream of audio.
By default it will be slow, so to speed it up
we can use non stream model on sampled audio data:
for example instead of computing non stream model
on every 20ms, we can run it on every 200ms of audio stream.
It will reduce total latency by 10 times.
To emulate sampling effect we use time_shift_samples.
weights_name: file name with model weights
accuracy_name: file name for storing accuracy in path + accuracy_name
Returns:
accuracy
"""
tf.reset_default_graph()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
tf.keras.backend.set_session(sess)
audio_processor = input_data.AudioProcessor(flags)
set_size = audio_processor.set_size('testing')
tf.keras.backend.set_learning_phase(0)
flags.batch_size = 100 # set batch size for inference
set_size = int(set_size / flags.batch_size) * flags.batch_size
model = models.MODELS[flags.model_name](flags)
weights_path = os.path.join(flags.train_dir, weights_name)
model.load_weights(weights_path).expect_partial()
total_accuracy = 0.0
count = 0.0
is_distilled = flags.distill_teacher_json
for i in range(0, set_size, flags.batch_size):
test_fingerprints, test_ground_truth = audio_processor.get_data(
flags.batch_size, i, flags, 0.0, 0.0, time_shift_samples,
'testing', 0.0, 0.0, sess)
predictions = model.predict(test_fingerprints)
if is_distilled:
predictions = predictions[
-1] # last output is ensembled prediction
predicted_labels = np.argmax(predictions, axis=1)
total_accuracy = total_accuracy + np.sum(
predicted_labels == test_ground_truth)
count = count + len(test_ground_truth)
total_accuracy = total_accuracy / count
logging.info('TF Final test accuracy on non stream model = %.2f%% (N=%d)',
*(total_accuracy * 100, set_size))
path = os.path.join(flags.train_dir, folder)
if not os.path.exists(path):
os.makedirs(path)
fname_summary = 'model_summary_non_stream'
utils.save_model_summary(model, path, file_name=fname_summary + '.txt')
tf.keras.utils.plot_model(model,
to_file=os.path.join(path,
fname_summary + '.png'),
show_shapes=True,
expand_nested=True)
with open(os.path.join(path, accuracy_name), 'wt') as fd:
fd.write('%f on set_size %d' % (total_accuracy * 100, set_size))
return total_accuracy * 100
def prepare_model1_weights_file():
tf.reset_default_graph()
m1 = Model1()
m1.train(X, Y, testX, testY, 2)
m1.model.save("model1.tfl")
# -*- coding: utf-8 -*- import numpy as np import tensorflow.compat.v1 as tf tf.disable_v2_behavior() # a variable has an initial value. You need to initialize it in the graph execution env # define the graph tf.reset_default_graph() a = tf.constant(np.ones((2, 2)), dtype=tf.float32) b = tf.Variable(tf.ones((2, 2))) c = a @ b # Create a session sess = tf.InteractiveSession() ''' # furious runnig attempt sess = tf.InteractiveSession() sess.run(c) ############# So, yeah! It throws 'Attempting to use uninitialized value variable' ''' # initialise the variables inside the graph sess.run(tf.global_variables_initializer()) print(sess.run(c)) # close the session tf.Session.close(sess)
def _post_processor(self, in_q, out_q, batch, rc):
def reorg_layer(feature_map, anchors):
'''
feature_map: a feature_map from [feature_map_1, feature_map_2, feature_map_3] returned
from `forward` function
anchors: shape: [3, 2]
'''
# NOTE: size in [h, w] format! don't get messed up!
grid_size = feature_map.get_shape().as_list()[1:3]
# the downscale ratio in height and weight
ratio = tf.cast(tf.constant([416, 416]) / grid_size, tf.float32)
# rescale the anchors to the feature_map
# NOTE: the anchor is in [w, h] format!
rescaled_anchors = [(anchor[0] / ratio[1], anchor[1] / ratio[0])
for anchor in anchors]
feature_map = tf.reshape(
feature_map, [-1, grid_size[0], grid_size[1], 3, 5 + 20])
# split the feature_map along the last dimension
# shape info: take 416x416 input image and the 13*13 feature_map for example:
# box_centers: [N, 13, 13, 3, 2] last_dimension: [center_x, center_y]
# box_sizes: [N, 13, 13, 3, 2] last_dimension: [width, height]
# conf_logits: [N, 13, 13, 3, 1]
# prob_logits: [N, 13, 13, 3, class_num]
box_centers, box_sizes, conf_logits, prob_logits = tf.split(
feature_map, [2, 2, 1, 20], axis=-1)
#box_centers = tf.nn.sigmoid(box_centers)
# use some broadcast tricks to get the mesh coordinates
grid_x = tf.range(grid_size[1], dtype=tf.int32)
grid_y = tf.range(grid_size[0], dtype=tf.int32)
grid_x, grid_y = tf.meshgrid(grid_x, grid_y)
x_offset = tf.reshape(grid_x, (-1, 1))
y_offset = tf.reshape(grid_y, (-1, 1))
x_y_offset = tf.concat([x_offset, y_offset], axis=-1)
# shape: [13, 13, 1, 2]
x_y_offset = tf.cast(
tf.reshape(x_y_offset, [grid_size[0], grid_size[1], 1, 2]),
tf.float32)
# get the absolute box coordinates on the feature_map
box_centers = box_centers + x_y_offset
# rescale to the original image scale
box_centers = box_centers * ratio[::-1]
# avoid getting possible nan value with tf.clip_by_value
box_sizes = tf.exp(box_sizes) * rescaled_anchors
# box_sizes = tf.clip_by_value(tf.exp(box_sizes), 1e-9, 100) * rescaled_anchors
# rescale to the original image scale
box_sizes = box_sizes * ratio[::-1]
# shape: [N, 13, 13, 3, 4]
# last dimension: (center_x, center_y, w, h)
boxes = tf.concat([box_centers, box_sizes], axis=-1)
# shape:
# x_y_offset: [13, 13, 1, 2]
# boxes: [N, 13, 13, 3, 4], rescaled to the original image scale
# conf_logits: [N, 13, 13, 3, 1]
# prob_logits: [N, 13, 13, 3, class_num]
return x_y_offset, boxes, conf_logits, prob_logits
def predict(feature_maps):
'''
Receive the returned feature_maps from `forward` function,
the produce the output predictions at the test stage.
'''
anchors = [[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
[59, 119], [116, 90], [156, 198], [373, 326]]
feature_map_1, feature_map_2, feature_map_3 = feature_maps
feature_map_anchors = [(feature_map_1, anchors[6:9]),
(feature_map_2, anchors[3:6]),
(feature_map_3, anchors[0:3])]
reorg_results = [
reorg_layer(feature_map, anchors)
for (feature_map, anchors) in feature_map_anchors
]
def _reshape(result):
x_y_offset, boxes, conf_logits, prob_logits = result
grid_size = x_y_offset.get_shape().as_list()[:2]
boxes = tf.reshape(boxes,
[-1, grid_size[0] * grid_size[1] * 3, 4])
conf_logits = tf.reshape(
conf_logits, [-1, grid_size[0] * grid_size[1] * 3, 1])
prob_logits = tf.reshape(
prob_logits, [-1, grid_size[0] * grid_size[1] * 3, 20])
# shape: (take 416*416 input image and feature_map_1 for example)
# boxes: [N, 13*13*3, 4]
# conf_logits: [N, 13*13*3, 1]
# prob_logits: [N, 13*13*3, class_num]
return boxes, conf_logits, prob_logits
boxes_list, confs_list, probs_list = [], [], []
for result in reorg_results:
boxes, conf_logits, prob_logits = _reshape(result)
confs = conf_logits
probs = prob_logits
boxes_list.append(boxes)
confs_list.append(confs)
probs_list.append(probs)
# collect results on three scales
# take 416*416 input image for example:
# shape: [N, (13*13+26*26+52*52)*3, 4]
boxes = tf.concat(boxes_list, axis=1)
# shape: [N, (13*13+26*26+52*52)*3, 1]
confs = tf.concat(confs_list, axis=1)
# shape: [N, (13*13+26*26+52*52)*3, class_num]
probs = tf.concat(probs_list, axis=1)
center_x, center_y, width, height = tf.split(boxes, [1, 1, 1, 1],
axis=-1)
x_min = center_x - width / 2
y_min = center_y - height / 2
x_max = center_x + width / 2
y_max = center_y + height / 2
boxes = tf.concat([x_min, y_min, x_max, y_max], axis=-1)
return boxes, confs, probs
tf.reset_default_graph()
post_graph = tf.Graph()
with post_graph.as_default() as g2:
with g2.name_scope("post-process_graph") as scope:
feature_maps = [
tf.placeholder(dtype=tf.float32,
shape=(batch, 13, 13, 3 * (20 + 5))),
tf.placeholder(dtype=tf.float32,
shape=(batch, 26, 26, 3 * (20 + 5))),
tf.placeholder(dtype=tf.float32,
shape=(batch, 52, 52, 3 * (20 + 5)))
]
boxes, confs, probs = predict(feature_maps)
init_op = tf.initialize_all_variables()
with tf.Session(graph=post_graph) as sess:
sess.run(init_op)
while True:
inp_batch = in_q.get()
if inp_batch == 'STOP':
out_q.put('STOP')
break
yolo_s, yolo_m, yolo_l = inp_batch
yolo_s = np.transpose(yolo_s, (0, 2, 3, 1))
yolo_m = np.transpose(yolo_m, (0, 2, 3, 1))
yolo_l = np.transpose(yolo_l, (0, 2, 3, 1))
batch_boxes, batch_confs, batch_probs = sess.run(
[boxes, confs, probs],
feed_dict={
feature_maps[0]: yolo_s,
feature_maps[1]: yolo_m,
feature_maps[2]: yolo_l
})
rc.tick()
out_q.put((batch_boxes, batch_confs * batch_probs))
def generate_sequences(int_to_char, char_to_int, num_sequence, seq_length,
rel_freq, f_generation, hidden_units, num_layers,
config):
"""
:param int_to_char: dictionary that maps integer to characters
:param char_to_int: dictionary that maps characters to integer
:param num_sequence: number of sequences to generate
:param seq_length: length of the sequences
:param rel_freq: relative frequencies
:param f_generation: file writer
:param hidden_units: list containing the number of hidden units for each LSTM cell
:param num_layers: number of LSTM cells
:param config: settings of the session
:return sequences: the output sequences
"""
tf.reset_default_graph()
session2 = tf.Session(config=config)
# Use the distribution of the output to generate a new character accordingly the distribution
initial_chars = ''
k = len(int_to_char)
for i in range(num_sequence):
char = np.random.choice(list(rel_freq.keys()),
p=list(rel_freq.values()))
initial_chars += char
# Preprocess the input of the network
encoded_input = [char_to_int[char] for char in initial_chars]
encoded_input = np.expand_dims(encoded_input, axis=1)
# Initialise and restore the network
X, S, Z, dropout, state = net_param_generation(hidden_units, num_layers, k)
Z_flat = tf.squeeze(Z)
Z_indices = tf.random.categorical(Z_flat, num_samples=1)
session2.run(tf.global_variables_initializer())
new_saver = tf.train.Saver()
new_saver.restore(session2, 'train/')
# Generate sequences
print("Starting generating…")
gen_start = time.time()
sequences = np.zeros(shape=[num_sequence, seq_length], dtype=str)
current_state = np.zeros((num_layers, 2, num_sequence, hidden_units[0]))
for j in range(seq_length):
current_state, output = session2.run([state, Z_indices],
feed_dict={
X: encoded_input,
S: current_state,
dropout: 1.0
})
output = [int_to_char[s] for s in output.ravel()]
sequences[:, j] = output
encoded_input = [char_to_int[char] for char in output]
encoded_input = np.expand_dims(encoded_input, axis=1)
gen_end = time.time()
gen_time = gen_end - gen_start
f_generation.write('Generation Time, ' + str(gen_time) + '\n')
print('Generation Time: {} sec.'.format(gen_time))
for idx, seq in enumerate(sequences):
seq = ''.join(seq)
print('Sequence ', str(idx + 1), ': ', seq, '\n')
f_generation.write('Sequence {}, {}\n'.format(idx + 1, seq))
return sequences
def main(_):
mode = Args['mode']
K = Parameters.NNHyperparameters[
"learner_num"] # the number of learner for boost training
params = Parameters.NNHyperparameters
checkpoint_root = Parameters.WorkSpacePath['checkpoint_dir']
if mode == "training":
training_directory_root = Parameters.WorkSpacePath[
'training_dataset_root']
tfrecord_filename = os.path.join(
Parameters.WorkSpacePath['training_dataset_root'],
'training_input_tfrecord_roi_withId')
if not os.path.exists(tfrecord_filename):
TFRecordIOWithROI.createTFRecord(
tfrecord_filename, dataset_root=training_directory_root)
total_record = sum(
1 for _ in tf.python_io.tf_record_iterator(tfrecord_filename))
# total_record = 610201
D = np.ones(
total_record, dtype=np.float32
) # training data weight, it will be modified according to the evalution result to solve the within class imbalance
Alpha = np.zeros(
K, dtype=np.float32
) # the learner weight which indicates how important the learner is
'''Each learner'''
for i in range(K):
'''train network G'''
subdir = "g%d/tensorboard" % i
tensorboard_dir = os.path.join(checkpoint_root, subdir)
checkpoint_dir = os.path.dirname(tensorboard_dir)
os.makedirs(tensorboard_dir, exist_ok=True)
'''save weight D'''
np.savetxt(os.path.join(checkpoint_dir, "data_weight.txt"),
D,
delimiter=' ')
net = JIgsawAbitraryNetROI.JigsawNetWithROI(params)
filename_queue = tf.train.string_input_producer(
[tfrecord_filename], capacity=128)
inputs, targets, roi_boxes, data_ids = TFRecordIOWithROI.readTFRecord(
filename_queue)
weights = tf.constant(D, dtype=tf.float32)
BoostTraining(net=net,
input=inputs,
roi_box=roi_boxes,
target=targets,
weights=weights,
data_ids=data_ids,
tensorboard_dir=tensorboard_dir,
checkpoint_dir=checkpoint_dir,
is_training=True)
tf.reset_default_graph()
'''calculate alpha and update weights'''
filename_queue = tf.train.string_input_producer(
[tfrecord_filename], shuffle=False, num_epochs=1)
inputs, targets, roi_boxes, data_ids = TFRecordIOWithROI.readTFRecord(
filename_queue)
weights = tf.constant(D, dtype=tf.float32)
alpha, new_weights = Evaluation(net=net,
input=inputs,
roi_box=roi_boxes,
target=targets,
weights=weights,
data_ids=data_ids,
checkpoint_dir=checkpoint_dir,
is_training=False)
Alpha[i] = alpha
D = new_weights
tf.reset_default_graph()
'''save alpha'''
with open(os.path.join(checkpoint_dir, "alpha.txt"), 'w') as f:
f.write("%f" % alpha)
'''Save Alpha'''
with open(os.path.join(checkpoint_root, "alpha.txt"), 'w') as f:
alp = ""
for a in Alpha:
alp += "%f " % a
f.write(alp)
elif mode == "batch_testing": # use tfrecord as input to evaluate
'''Batch Testing'''
testing_directory_root = Parameters.WorkSpacePath[
'testing_dataset_root']
testing_tfrecord_filename = os.path.join(
Parameters.WorkSpacePath['testing_dataset_root'],
'input_tfrecord_roi')
if not os.path.exists(testing_tfrecord_filename):
TFRecordIOWithROI.createTFRecord(
testing_tfrecord_filename, dataset_root=testing_directory_root)
net = JIgsawAbitraryNetROI.JigsawNetWithROI(params)
with open(os.path.join(checkpoint_root, "alpha.txt")) as f:
for line in f:
line = line.rstrip()
if line[0] != '#':
line = line.split()
Alpha = [float(x) for x in line]
BatchTest(checkpoint_root=checkpoint_root,
testing_tfrecord_filename=testing_tfrecord_filename,
K=K,
net=net,
Alpha=Alpha,
is_training=False)
elif mode == "single_testing": # use alignment transformation as input to evaluate.
'''Single Testing'''
testing_data_root1 = Parameters.WorkSpacePath["example_testing_root"]
fragments_dirs = glob.glob(os.path.join(testing_data_root1, "*_ex"))
with open(os.path.join(checkpoint_root, "alpha.txt")) as f:
for line in f:
line = line.rstrip()
if line[0] != '#':
line = line.split()
Alpha = [float(x) for x in line]
net = JIgsawAbitraryNetROI.JigsawNetWithROI(
params=Parameters.NNHyperparameters)
evaluator = SingleTest(checkpoint_root=checkpoint_root,
K=5,
net=net,
is_training=False)
for i in range(len(fragments_dirs)):
print("dataset %d/%d:" % (i, len(fragments_dirs)))
if not os.path.exists(
os.path.join(fragments_dirs[i], "alignments.txt")):
continue
bg_color_file = os.path.join(fragments_dirs[i], "bg_color.txt")
with open(bg_color_file) as f:
for line in f:
line = line.split()
if line:
bg_color = [int(i) for i in line]
bg_color = bg_color[::-1]
gt_pose = os.path.join(fragments_dirs[i], "groundTruth.txt")
relative_alignment = os.path.join(fragments_dirs[i],
"alignments.txt")
gt_pose = Utils.GtPose(gt_pose)
alignments = Utils.Alignment2d(relative_alignment)
ValidatePathNet(alignments,
gt_pose,
fragments_dirs[i],
net,
evaluator,
K,
Alpha,
bg_color,
save_all_leaner=False)
print("----------------")
def main(unused_argv):
del unused_argv
# Load Config
config_name = FLAGS.config
config_module = importlib.import_module('configs.%s' % config_name)
config = config_module.config
model_uid = common.get_model_uid(config_name, FLAGS.exp_uid)
n_latent = config['n_latent']
# Load dataset
dataset = common.load_dataset(config)
basepath = dataset.basepath
save_path = dataset.save_path
train_data = dataset.train_data
# Make the directory
save_dir = os.path.join(save_path, model_uid)
best_dir = os.path.join(save_dir, 'best')
tf.gfile.MakeDirs(save_dir)
tf.gfile.MakeDirs(best_dir)
tf.logging.info('Save Dir: %s', save_dir)
# Set random seed
np.random.seed(FLAGS.random_seed)
tf.set_random_seed(FLAGS.random_seed)
# Load Model
tf.reset_default_graph()
sess = tf.Session()
with tf.device(tf.train.replica_device_setter(ps_tasks=0)):
m = model_dataspace.Model(config, name=model_uid)
_ = m() # noqa
# Initialize
sess.run(tf.global_variables_initializer())
# Load
m.vae_saver.restore(
sess, os.path.join(best_dir, 'vae_best_%s.ckpt' % model_uid))
# Sample from prior
sample_count = 64
image_path = os.path.join(basepath, 'sample', model_uid)
tf.gfile.MakeDirs(image_path)
# from prior
z_p = np.random.randn(sample_count, m.n_latent)
x_p = sess.run(m.x_mean, {m.z: z_p})
x_p = common.post_proc(x_p, config)
common.save_image(common.batch_image(x_p),
os.path.join(image_path, 'sample_prior.png'))
# Sample from priro, as Grid
boundary = 2.0
number_grid = 50
blob = common.make_grid(boundary=boundary,
number_grid=number_grid,
dim_latent=n_latent)
z_grid, dim_grid = blob.z_grid, blob.dim_grid
x_grid = sess.run(m.x_mean, {m.z: z_grid})
x_grid = common.post_proc(x_grid, config)
batch_image_grid = common.make_batch_image_grid(dim_grid, number_grid)
common.save_image(batch_image_grid(x_grid),
os.path.join(image_path, 'sample_grid.png'))
# Reconstruction
sample_count = 64
x_real = train_data[:sample_count]
mu, sigma = sess.run([m.mu, m.sigma], {m.x: x_real})
x_rec = sess.run(m.x_mean, {m.mu: mu, m.sigma: sigma})
x_rec = common.post_proc(x_rec, config)
x_real = common.post_proc(x_real, config)
common.save_image(common.batch_image(x_real),
os.path.join(image_path, 'image_real.png'))
common.save_image(common.batch_image(x_rec),
os.path.join(image_path, 'image_rec.png'))
def testMultipleConvMaskAdded(self, pruning_method):
tf.reset_default_graph()
g = tf.Graph()
with g.as_default():
number_of_layers = 5
kernel_size = [3, 3]
base_depth = 4
depth_step = 7
input_tensor = tf.ones((8, self.height, self.width, base_depth))
top_layer = input_tensor
for ix in range(number_of_layers):
units = base_depth + (ix + 1) * depth_step
top_layer = pruning_layers.sparse_conv2d(
x=top_layer,
units=units,
kernel_size=kernel_size,
is_training=False,
sparsity_technique=pruning_method)
if pruning_method == 'variational_dropout':
theta_logsigma2 = tf.get_collection(
vd.layers.THETA_LOGSIGMA2_COLLECTION)
self.assertLen(theta_logsigma2, number_of_layers)
utils.add_vd_pruning_summaries(theta_logsigma2, threshold=3.0)
dkl_loss_1 = utils.variational_dropout_dkl_loss(
reg_scalar=1,
start_reg_ramp_up=0,
end_reg_ramp_up=1000,
warm_up=False,
use_tpu=False)
dkl_loss_1 = tf.reshape(dkl_loss_1, [1])
dkl_loss_2 = utils.variational_dropout_dkl_loss(
reg_scalar=5,
start_reg_ramp_up=0,
end_reg_ramp_up=1000,
warm_up=False,
use_tpu=False)
dkl_loss_2 = tf.reshape(dkl_loss_2, [1])
for ix in range(number_of_layers):
self.assertListEqual(theta_logsigma2[ix][0].get_shape().as_list(), [
kernel_size[0], kernel_size[1], base_depth + ix * depth_step,
base_depth + (ix + 1) * depth_step
])
init_op = tf.global_variables_initializer()
with self.test_session() as sess:
sess.run(init_op)
if pruning_method == 'variational_dropout':
loss_1, loss_2 = sess.run([dkl_loss_1, dkl_loss_2])
self.assertGreater(loss_2, loss_1)
elif pruning_method == 'l0_regularization':
theta_logalpha = tf.get_collection(
l0.layers.THETA_LOGALPHA_COLLECTION)
self.assertLen(theta_logalpha, number_of_layers)
utils.add_l0_summaries(theta_logalpha)
l0_norm_loss_1 = utils.l0_regularization_loss(
reg_scalar=1,
start_reg_ramp_up=0,
end_reg_ramp_up=1000,
warm_up=False,
use_tpu=False)
l0_norm_loss_1 = tf.reshape(l0_norm_loss_1, [1])
l0_norm_loss_2 = utils.l0_regularization_loss(
reg_scalar=5,
start_reg_ramp_up=0,
end_reg_ramp_up=1000,
warm_up=False,
use_tpu=False)
l0_norm_loss_2 = tf.reshape(l0_norm_loss_2, [1])
for ix in range(number_of_layers):
self.assertListEqual(theta_logalpha[ix][0].get_shape().as_list(), [
kernel_size[0], kernel_size[1], base_depth + ix * depth_step,
base_depth + (ix + 1) * depth_step
])
init_op = tf.global_variables_initializer()
with self.test_session() as sess:
sess.run(init_op)
loss_1, loss_2 = sess.run([l0_norm_loss_1, l0_norm_loss_2])
self.assertGreater(loss_2, loss_1)
else:
mask = tf.get_collection(core.MASK_COLLECTION)
for ix in range(number_of_layers):
self.assertListEqual(mask[ix].get_shape().as_list(), [
kernel_size[0], kernel_size[1], base_depth + ix * depth_step,
base_depth + (ix + 1) * depth_step
])
def piecewise_system(n_epochs0, learning_rate0, trainlog, evallog, allTheData,
n):
KBs_test, KBs_train, X_train, X_test, y_train, y_test, decodeNonProperty, decodeProperty = allTheData
trainlog.write(
"Piecewise LSTM Part One\nEpoch,Mean Squared Error,Root Mean Squared Error\n"
)
evallog.write("Piecewise LSTM Part One\n")
print("")
n_neurons0 = X_train.shape[2]
X0 = tf.compat.v1.placeholder(
tf.float32, shape=[None, KBs_train.shape[1], KBs_train.shape[2]])
y0 = tf.compat.v1.placeholder(
tf.float32, shape=[None, X_train.shape[1], X_train.shape[2]])
outputs0, states0 = tf.nn.dynamic_rnn(
tf.nn.rnn_cell.LSTMCell(num_units=n_neurons0), X0, dtype=tf.float32)
loss0 = tf.losses.mean_squared_error(y0, outputs0)
optimizer0 = tf.train.AdamOptimizer(learning_rate=learning_rate0)
training_op0 = optimizer0.minimize(loss0)
init0 = tf.global_variables_initializer()
with tf.Session() as sess:
init0.run()
mse0 = 0
mseL = 0
for epoch in range(n_epochs0):
print("Piecewise System\tEpoch: {}".format(epoch))
ynew, a = sess.run([outputs0, training_op0],
feed_dict={
X0: KBs_train,
y0: X_train
})
mse = loss0.eval(feed_dict={outputs0: ynew, y0: X_train})
if epoch == 0: mse0 = mse
if epoch == n_epochs0 - 1: mseL = mse
trainlog.write("{},{},{}\n".format(epoch, mse, math.sqrt(mse)))
if mse < 0.00001:
mseL = mse
break
print("\nTesting first half\n")
y_pred = sess.run(outputs0, feed_dict={X0: KBs_test, y0: X_test})
mseNew = loss0.eval(feed_dict={outputs0: y_pred, y0: X_test})
training_stats(evallog, mseNew, mse0, mseL)
# write_vector_file("crossValidationFolds/output/learnedSupportsP[{}].txt".format(n),newStrIRI)
numpy.savez("crossValidationFolds/saves/halfwayData[{}]".format(n),
y_pred)
tf.reset_default_graph()
trainlog.write(
"Piecewise LSTM Part Two\nEpoch,Mean Squared Error,Root Mean Squared Error\n"
)
evallog.write("\nPiecewise LSTM Part Two\n")
n_neurons1 = y_train.shape[2]
X1 = tf.placeholder(tf.float32,
shape=[None, X_train.shape[1], X_train.shape[2]])
y1 = tf.placeholder(tf.float32,
shape=[None, y_train.shape[1], y_train.shape[2]])
outputs1, states1 = tf.nn.dynamic_rnn(
tf.nn.rnn_cell.LSTMCell(num_units=n_neurons1), X1, dtype=tf.float32)
loss1 = tf.losses.mean_squared_error(y1, outputs1)
optimizer1 = tf.train.AdamOptimizer(learning_rate=learning_rate0)
training_op1 = optimizer1.minimize(loss1)
init1 = tf.global_variables_initializer()
with tf.Session() as sess:
init1.run()
mse0 = 0
mseL = 0
for epoch in range(n_epochs0):
print("Piecewise System\tEpoch: {}".format(epoch + n_epochs0))
ynew, a = sess.run([outputs1, training_op1],
feed_dict={
X1: X_train,
y1: y_train
})
mse = loss1.eval(feed_dict={outputs1: ynew, y1: y_train})
if epoch == 0: mse0 = mse
if epoch == n_epochs0 - 1: mseL = mse
trainlog.write("{},{},{}\n".format(epoch, mse, math.sqrt(mse)))
if mse < 0.00001:
mseL = mse
break
print("\nTesting second half")
y_pred = sess.run(outputs1, feed_dict={X1: X_test})
mseNew = loss1.eval(feed_dict={outputs1: y_pred, y1: y_test})
training_stats(evallog, mseNew, mse0, mseL)
print("\nEvaluating Result")
evallog.write("\nReasoner Support Test Data Evaluation\n")
# write_vector_file("crossValidationFolds/output/predictedOutLeftOverSupportTest[{}].txt".format(n), newStrIRI)
data = numpy.load(
"crossValidationFolds/saves/halfwayData[{}].npz".format(n),
allow_pickle=True)
data = data['arr_0']
evallog.write("\nSaved Test Data From Previous LSTM Evaluation\n")
y_pred = sess.run(outputs1, feed_dict={X1: data})
mseNew = loss1.eval(feed_dict={outputs1: y_pred, y1: y_test})
evallog.write("\nTesting Statistic\nIncrease MSE on Saved,{}\n".format(
numpy.float32(mseNew) - mseL))
trueLabels, labelPredictions = get_labels_from_encoding(
y_test, y_pred, decodeNonProperty, decodeProperty)
# write_vector_file("crossValidationFolds/output/predictionSavedKBPipeline[{}].txt".format(n), newStrIRI)
return distance_Evaluations(evallog, y_pred.shape, labelPredictions,
trueLabels)
def tflite_non_stream_model_accuracy(
flags,
folder,
tflite_model_name='non_stream.tflite',
accuracy_name='tflite_non_stream_model_accuracy.txt'):
"""Compute accuracy of non streamable model with TFLite.
Model has to be converted to TFLite and stored in path+tflite_model_name
Args:
flags: model and data settings
folder: folder name where model is located
tflite_model_name: file name with tflite model
accuracy_name: file name for storing accuracy in path + accuracy_name
Returns:
accuracy
"""
tf.reset_default_graph()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
tf.keras.backend.set_session(sess)
path = os.path.join(flags.train_dir, folder)
audio_processor = input_data.AudioProcessor(flags)
set_size = audio_processor.set_size('testing')
interpreter = tf.lite.Interpreter(
model_path=os.path.join(path, tflite_model_name))
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
inputs = []
for s in range(len(input_details)):
inputs.append(np.zeros(input_details[s]['shape'], dtype=np.float32))
total_accuracy = 0.0
count = 0.0
inference_batch_size = 1
for i in range(0, set_size, inference_batch_size):
test_fingerprints, test_ground_truth = audio_processor.get_data(
inference_batch_size, i, flags, 0.0, 0.0, 0, 'testing', 0.0, 0.0,
sess)
# set input audio data (by default input data at index 0)
interpreter.set_tensor(input_details[0]['index'],
test_fingerprints.astype(np.float32))
# run inference
interpreter.invoke()
# get output: classification
out_tflite = interpreter.get_tensor(output_details[0]['index'])
out_tflite_argmax = np.argmax(out_tflite)
total_accuracy = total_accuracy + (test_ground_truth[0]
== out_tflite_argmax)
count = count + 1
if i % 200 == 0 and i:
logging.info(
'tflite test accuracy, non stream model = %.2f%% %d out of %d',
*(total_accuracy * 100 / count, i, set_size))
total_accuracy = total_accuracy / count
logging.info(
'tflite Final test accuracy, non stream model = %.2f%% (N=%d)',
*(total_accuracy * 100, set_size))
with open(os.path.join(path, accuracy_name), 'wt') as fd:
fd.write('%f on set_size %d' % (total_accuracy * 100, set_size))
return total_accuracy * 100
def analysis(config_dict, synonym_filepath,
model_location, batch_size, batch_offset=0,
total_num_batches=0, datasplit='test', delta=3.0,
num_perturbations=5, max_padded_length=0):
"""Run analysis."""
tf.reset_default_graph()
if datasplit not in ['train', 'dev', 'test']:
raise ValueError('Invalid datasplit: %s' % datasplit)
logging.info('model_location: %s', model_location)
logging.info('num_perturbations: %d', num_perturbations)
logging.info('delta: %f', delta)
logging.info('Run analysis, datasplit: %s, batch %d', datasplit, batch_offset)
synonym_keys, max_synoynm_counts, synonym_value_lens_cum, \
synonym_values_list = construct_synonyms(synonym_filepath)
graph_tensor_producer = robust_model.RobustModel(**config_dict)
# Use new batch size.
graph_tensor_producer.batch_size = batch_size
# Overwrite the config originally in the saved checkpoint.
logging.info('old delta %f, old num_perturbations: %d',
graph_tensor_producer.config['delta'],
graph_tensor_producer.config['num_perturbations'])
graph_tensor_producer.config['delta'] = delta
graph_tensor_producer.config['num_perturbations'] = num_perturbations
if max_padded_length > 0:
graph_tensor_producer.config['max_padded_length'] = max_padded_length
logging.info('new delta %f, num_perturbations: %d, max_padded_length: %d',
graph_tensor_producer.config['delta'],
graph_tensor_producer.config['num_perturbations'],
graph_tensor_producer.config['max_padded_length'])
logging.info('graph_tensors.config: %s', graph_tensor_producer.config)
graph_tensors = graph_tensor_producer()
network_saver = tf.train.Saver(graph_tensor_producer.variables)
with tf.train.SingularMonitoredSession() as session:
network_saver.restore(session.raw_session(), model_location)
for _ in range(batch_offset):
# Seek to the correct batch.
session.run(graph_tensors[datasplit]['sentiment'])
input_feed = {}
# Tokenize synonyms.
tokenize_synonyms = [[] for _ in range(graph_tensors['vocab_size'])]
lookup_indices_keys = session.run(graph_tensors['indices'],
feed_dict={graph_tensors['lookup_token']:
synonym_keys})
lookup_indices_values = session.run(graph_tensors['indices'],
feed_dict={
graph_tensors['lookup_token']:
synonym_values_list})
for i, key_index in enumerate(lookup_indices_keys):
tokenize_synonyms[key_index] = lookup_indices_values[
synonym_value_lens_cum[i]:synonym_value_lens_cum[i+1]].tolist()
synonym_values_np = np.zeros([graph_tensors['vocab_size'],
max_synoynm_counts])
for i in range(graph_tensors['vocab_size']):
# False-safe case. No perturbations. Set it as itself.
synonym_values_np[i][0] = i
for j in range(len(tokenize_synonyms[i])):
synonym_values_np[i][j] = tokenize_synonyms[i][j]
synonym_counts_np = [len(s) for s in tokenize_synonyms]
input_feed[graph_tensors['synonym_values']] = synonym_values_np
input_feed[graph_tensors['synonym_counts']] = synonym_counts_np
total_num_batches = (
graph_tensors['%s_num_examples' % datasplit] //
batch_size) if total_num_batches == 0 else total_num_batches
total_num_examples = total_num_batches * batch_size
logging.info('total number of examples %d', total_num_examples)
logging.info('total number of batches %d', total_num_batches)
total_correct, total_verified = 0.0, 0.0
for ibatch in range(total_num_batches):
results = session.run(graph_tensors[datasplit], input_feed)
logging.info('batch: %d, %s bound = %.05f, verified: %.03f,'
' nominally correct: %.03f',
ibatch, datasplit, np.mean(results['bound']),
np.mean(results['verified']),
np.mean(results['correct']))
total_correct += sum(results['correct'])
total_verified += sum(results['verified'])
total_correct /= total_num_examples
total_verified /= total_num_examples
logging.info('%s final correct: %.03f, verified: %.03f',
datasplit, total_correct, total_verified)
logging.info({
'datasplit': datasplit,
'nominal': total_correct,
'verify': total_verified,
'delta': delta,
'num_perturbations': num_perturbations,
'model_location': model_location,
'final': True
})
def tflite_stream_state_external_model_accuracy(
flags,
folder,
tflite_model_name='stream_state_external.tflite',
accuracy_name='tflite_stream_state_external_model_accuracy.txt',
reset_state=False):
"""Compute accuracy of streamable model with external state using TFLite.
Args:
flags: model and data settings
folder: folder name where model is located
tflite_model_name: file name with tflite model
accuracy_name: file name for storing accuracy in path + accuracy_name
reset_state: reset state between testing sequences.
If True - then it is non streaming testing environment: state will be
reseted in the beginning of every test sequence and will not be
transferred to another one (as it is done in real streaming).
Returns:
accuracy
"""
tf.reset_default_graph()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
tf.keras.backend.set_session(sess)
path = os.path.join(flags.train_dir, folder)
logging.info('tflite stream model state external with reset_state %d',
reset_state)
audio_processor = input_data.AudioProcessor(flags)
set_size = audio_processor.set_size('testing')
interpreter = tf.lite.Interpreter(
model_path=os.path.join(path, tflite_model_name))
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
inputs = []
for s in range(len(input_details)):
inputs.append(np.zeros(input_details[s]['shape'], dtype=np.float32))
total_accuracy = 0.0
count = 0.0
inference_batch_size = 1
for i in range(0, set_size, inference_batch_size):
test_fingerprints, test_ground_truth = audio_processor.get_data(
inference_batch_size, i, flags, 0.0, 0.0, 0, 'testing', 0.0, 0.0,
sess)
# before processing new test sequence we can reset model state
# if we reset model state then it is not real streaming mode
if reset_state:
for s in range(len(input_details)):
inputs[s] = np.zeros(input_details[s]['shape'],
dtype=np.float32)
if flags.preprocess == 'raw':
out_tflite = run_stream_inference_classification_tflite(
flags, interpreter, test_fingerprints, inputs)
out_tflite_argmax = np.argmax(out_tflite)
else:
for t in range(test_fingerprints.shape[1]):
# get new frame from stream of data
stream_update = test_fingerprints[:, t, :]
stream_update = np.expand_dims(stream_update, axis=1)
# [batch, time=1, feature]
stream_update = stream_update.astype(np.float32)
# set input audio data (by default input data at index 0)
interpreter.set_tensor(input_details[0]['index'],
stream_update)
# set input states (index 1...)
for s in range(1, len(input_details)):
interpreter.set_tensor(input_details[s]['index'],
inputs[s])
# run inference
interpreter.invoke()
# get output: classification
out_tflite = interpreter.get_tensor(output_details[0]['index'])
# get output states and set it back to input states
# which will be fed in the next inference cycle
for s in range(1, len(input_details)):
# The function `get_tensor()` returns a copy of the tensor data.
# Use `tensor()` in order to get a pointer to the tensor.
inputs[s] = interpreter.get_tensor(
output_details[s]['index'])
out_tflite_argmax = np.argmax(out_tflite)
total_accuracy = total_accuracy + (test_ground_truth[0]
== out_tflite_argmax)
count = count + 1
if i % 200 == 0 and i:
logging.info(
'tflite test accuracy, stream model state external = %f %d out of %d',
*(total_accuracy * 100 / count, i, set_size))
total_accuracy = total_accuracy / count
logging.info(
'tflite Final test accuracy, stream model state external = %.2f%% (N=%d)',
*(total_accuracy * 100, set_size))
with open(os.path.join(path, accuracy_name), 'wt') as fd:
fd.write('%f on set_size %d' % (total_accuracy * 100, set_size))
return total_accuracy * 100
def tf_stream_state_external_model_accuracy(
flags,
folder,
weights_name='best_weights',
accuracy_name='stream_state_external_model_accuracy_sub_set.txt',
reset_state=False,
max_test_samples=1000):
"""Compute accuracy of streamable model with external state using TF.
Args:
flags: model and data settings
folder: folder name where accuracy report will be stored
weights_name: file name with model weights
accuracy_name: file name for storing accuracy in path + accuracy_name
reset_state: reset state between testing sequences.
If True - then it is non streaming testing environment: state will be
reseted on every test and will not be transferred to another one (as
it is done in real streaming).
max_test_samples: max number of test samples. In this mode model is slow
with TF because of batch size 1, so accuracy is computed on subset of
testing data
Returns:
accuracy
"""
tf.reset_default_graph()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
tf.keras.backend.set_session(sess)
audio_processor = input_data.AudioProcessor(flags)
set_size = audio_processor.set_size('testing')
set_size = np.minimum(max_test_samples, set_size)
inference_batch_size = 1
tf.keras.backend.set_learning_phase(0)
flags.batch_size = inference_batch_size # set batch size
model = models.MODELS[flags.model_name](flags)
weights_path = os.path.join(flags.train_dir, weights_name)
model.load_weights(weights_path).expect_partial()
model_stream = utils.to_streaming_inference(
model, flags, modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE)
logging.info('tf stream model state external with reset_state %d',
reset_state)
inputs = []
for s in range(len(model_stream.inputs)):
inputs.append(np.zeros(model_stream.inputs[s].shape, dtype=np.float32))
total_accuracy = 0.0
count = 0.0
inference_batch_size = 1
for i in range(0, set_size, inference_batch_size):
test_fingerprints, test_ground_truth = audio_processor.get_data(
inference_batch_size, i, flags, 0.0, 0.0, 0, 'testing', 0.0, 0.0,
sess)
if reset_state:
for s in range(len(model_stream.inputs)):
inputs[s] = np.zeros(model_stream.inputs[s].shape,
dtype=np.float32)
if flags.preprocess == 'raw':
start = 0
end = flags.window_stride_samples
# iterate over time samples with stride = window_stride_samples
while end <= test_fingerprints.shape[1]:
# get new frame from stream of data
stream_update = test_fingerprints[:, start:end]
# update indexes of streamed updates
start = end
end = start + flags.window_stride_samples
# set input audio data (by default input data at index 0)
inputs[0] = stream_update
# run inference
outputs = model_stream.predict(inputs)
# get output states and set it back to input states
# which will be fed in the next inference cycle
for s in range(1, len(model_stream.inputs)):
inputs[s] = outputs[s]
stream_output_arg = np.argmax(outputs[0])
else:
# iterate over frames
for t in range(test_fingerprints.shape[1]):
# get new frame from stream of data
stream_update = test_fingerprints[:, t, :]
# [batch, time=1, feature]
stream_update = np.expand_dims(stream_update, axis=1)
# set input audio data (by default input data at index 0)
inputs[0] = stream_update
# run inference
outputs = model_stream.predict(inputs)
# get output states and set it back to input states
# which will be fed in the next inference cycle
for s in range(1, len(model_stream.inputs)):
inputs[s] = outputs[s]
stream_output_arg = np.argmax(outputs[0])
total_accuracy = total_accuracy + (test_ground_truth[0]
== stream_output_arg)
count = count + 1
if i % 200 == 0 and i:
logging.info(
'tf test accuracy, stream model state external = %.2f%% %d out of %d',
*(total_accuracy * 100 / count, i, set_size))
total_accuracy = total_accuracy / count
logging.info(
'TF Final test accuracy of stream model state external = %.2f%% (N=%d)',
*(total_accuracy * 100, set_size))
path = os.path.join(flags.train_dir, folder)
if not os.path.exists(path):
os.makedirs(path)
fname_summary = 'model_summary_stream_state_external'
utils.save_model_summary(model_stream,
path,
file_name=fname_summary + '.txt')
tf.keras.utils.plot_model(model_stream,
to_file=os.path.join(path,
fname_summary + '.png'),
show_shapes=True,
expand_nested=True)
with open(os.path.join(path, accuracy_name), 'wt') as fd:
fd.write('%f on set_size %d' % (total_accuracy * 100, set_size))
return total_accuracy * 100
def tf_stream_state_internal_model_accuracy(
flags,
folder,
weights_name='best_weights',
accuracy_name='tf_stream_state_internal_model_accuracy_sub_set.txt',
max_test_samples=1000):
"""Compute accuracy of streamable model with internal state using TF.
Testign model with batch size 1 can be slow, so accuracy is evaluated
on subset of data with size max_test_samples
Args:
flags: model and data settings
folder: folder name where accuracy report will be stored
weights_name: file name with model weights
accuracy_name: file name for storing accuracy in path + accuracy_name
max_test_samples: max number of test samples. In this mode model is slow
with TF because of batch size 1, so accuracy is computed on subset of
testing data
Returns:
accuracy
"""
tf.reset_default_graph()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
tf.keras.backend.set_session(sess)
logging.info('tf stream model state internal without state resetting'
'between testing sequences')
audio_processor = input_data.AudioProcessor(flags)
set_size = audio_processor.set_size('testing')
set_size = np.minimum(max_test_samples, set_size)
inference_batch_size = 1
tf.keras.backend.set_learning_phase(0)
flags.batch_size = inference_batch_size # set batch size
model = models.MODELS[flags.model_name](flags)
weights_path = os.path.join(flags.train_dir, weights_name)
model.load_weights(weights_path).expect_partial()
model_stream = utils.to_streaming_inference(
model, flags, modes.Modes.STREAM_INTERNAL_STATE_INFERENCE)
total_accuracy = 0.0
count = 0.0
for i in range(0, set_size, inference_batch_size):
test_fingerprints, test_ground_truth = audio_processor.get_data(
inference_batch_size, i, flags, 0.0, 0.0, 0, 'testing', 0.0, 0.0,
sess)
if flags.preprocess == 'raw':
stream_output_prediction = run_stream_inference_classification(
flags, model_stream, test_fingerprints)
stream_output_arg = np.argmax(stream_output_prediction)
else:
# iterate over frames
for t in range(test_fingerprints.shape[1]):
# get new frame from stream of data
stream_update = test_fingerprints[:, t, :]
# [batch, time=1, feature]
stream_update = np.expand_dims(stream_update, axis=1)
# classification result of a current frame
stream_output_prediction = model_stream.predict(stream_update)
stream_output_arg = np.argmax(stream_output_prediction)
total_accuracy = total_accuracy + (test_ground_truth[0]
== stream_output_arg)
count = count + 1
if i % 200 == 0 and i:
logging.info(
'tf test accuracy, stream model state internal = %.2f%% %d out of %d',
*(total_accuracy * 100 / count, i, set_size))
total_accuracy = total_accuracy / count
logging.info(
'TF Final test accuracy of stream model state internal = %.2f%% (N=%d)',
*(total_accuracy * 100, set_size))
path = os.path.join(flags.train_dir, folder)
if not os.path.exists(path):
os.makedirs(path)
fname_summary = 'model_summary_stream_state_internal'
utils.save_model_summary(model_stream,
path,
file_name=fname_summary + '.txt')
tf.keras.utils.plot_model(model_stream,
to_file=os.path.join(path,
fname_summary + '.png'),
show_shapes=True,
expand_nested=True)
with open(os.path.join(path, accuracy_name), 'wt') as fd:
fd.write('%f on set_size %d' % (total_accuracy * 100, set_size))
return total_accuracy * 100
def main(args):
global logger
tf1.reset_default_graph()
if not os.path.exists(args.exp_dir):
os.makedirs(args.exp_dir)
model_path = os.path.join(args.exp_dir, 'model')
if not os.path.exists(model_path):
os.makedirs(model_path)
logger = vae_util.setup_logger(os.path.join(args.exp_dir, 'training.log'),
args.debug)
logger.info("Experiment at {}".format(args.exp_dir))
logger.info(vars(args))
# dataset
xs, dim = load_dataset(args.dataset, 'train', args.root_dir,
normalize=True)
ys, n_class = load_dataset(args.dataset, 'train', args.root_dir,
label=True)
if args.limit:
xs, ys = xs[:6400], ys[:6400]
logger.info('Train data len: {}, dim: {}, classes: {}'.format(len(xs),
dim, n_class))
xs_val, _ = load_dataset(args.dataset, 'test', args.root_dir,
normalize=True)
ys_val, _ = load_dataset(args.dataset, 'test', args.root_dir, label=True)
if args.drop >= 0:
logger.info("Dropping class {}".format(args.drop))
xs, ys = drop_class(xs, ys, args.drop)
xs_val, ys_val = drop_class(xs_val, ys_val, args.drop)
cs = utility.one_hot(ys)
n_sample = np.shape(xs)[0]
logger.info('Num Sample = {}.'.format(n_sample))
# Load from configuration
config_filename = os.path.join(args.exp_dir, 'config.yml')
load_configuration(args, config_filename)
pprinter = pprint.PrettyPrinter(indent=4)
logger.info("Configuration: {}".format(pprinter.pformat(vars(args))))
# Save/update the config only when any of the VAE gets trained
if args.train1 or args.train2:
logger.info("Saving configuration to " + config_filename)
save_configuration(args, config_filename, n_sample)
# session
config = tf1.ConfigProto(device_count={'GPU': 0}) if args.cpu else None
sess = tf1.Session(config=config)
# model
outer_vae = vae_util.get_outer_vae(args, sess)
outer_params = vae_util.get_trainable_parameters('outer')
logger.info("Created VAE models:")
logger.info("{}, {} params".format(outer_vae, outer_params))
# train model
if args.train1:
if args.wandb:
wandb.init(project=args.dataset, name=args.exp_name,
sync_tensorboard=True, config=args)
mae = outer_vae.train(lambda: (xs, cs), args.epochs1, args.lr1,
os.path.join(model_path, 'stage1'),
log_epoch=log_epoch)
logger.info("Finished training stage 1 VAE. Mae: {:.2%}".format(mae))
if args.train2:
inner_vae = vae_util.get_inner_vae(args, sess, outer_vae)
sess.run(tf1.global_variables_initializer())
outer_vae.restore(os.path.join(model_path, 'stage1'))
mu_z, sd_z = outer_vae.extract_posterior(xs, cs)
def get_data():
zs = mu_z + sd_z * np.random.normal(0, 1,
[len(mu_z), args.latent_dim1])
return zs, cs
mae = inner_vae.train(get_data, args.epochs2, args.lr2,
os.path.join(model_path, 'stage2'),
log_epoch=log_epoch)
logger.info("Finished training stage 2 VAE. Mae: {:.2%}".format(mae))
# load
if not (args.train1 or args.train2):
# saver.restore(sess, os.path.join(model_path, 'stage1'))
if os.path.exists(os.path.join(model_path, 'stage2.index')):
inner_vae = vae_util.get_inner_vae(args, sess, outer_vae)
inner_vae.restore(os.path.join(model_path, 'stage2'))
logger.info("Loaded Stage 2 VAE")
elif os.path.exists(os.path.join(model_path, 'stage1.index')):
outer_vae.restore(os.path.join(model_path, 'stage1'))
logger.info("Loaded Stage 1 VAE")
else:
raise Exception("No checkpoint found!")
if args.eval:
logger.info("Evaluating...")
evaluate(args, outer_vae, inner_vae, sess)
if args.interpolate:
interpolate(VaeWrapper(outer_vae, inner_vae), sess, args.exp_dir, xs, ys, 20)
if args.manifold:
logger.info("Analyze manifold")
vae_util.analyze_manifold(sess, VaeWrapper(outer_vae, inner_vae), args.dataset)
def session():
tf1.reset_default_graph()
with tf1.Session() as s:
yield s
def evaluate(args, model1: OuterVaeModel, model2: InnerVaeModel,
sess: tf1.Session):
maes = vae_util.evaluate_models(sess, model1, model2, args.dataset,
args.root_dir)
logger.info(maes)
total_params = vae_util.get_trainable_parameters('outer')
logger.info("stage1 trainable params: {}".format(total_params))
total_params = vae_util.get_trainable_parameters('inner')
logger.info("stage2 trainable params: {}".format(total_params))
# test dataset
x, dim = load_dataset(args.dataset, 'test', args.root_dir,
normalize=True)
y, n_class = load_dataset(args.dataset, 'test', args.root_dir,
label=True)
inds = np.array(list(range(len(x))))
np.random.shuffle(inds)
x = x[inds][0:args.fid_cnt]
y = y[inds][0:args.fid_cnt]
y_encoded = utility.one_hot(y, n_class) if args.conditional else None
# reconstruction and generation
def generate_label(cnt):
return utility.one_hot(np.random.randint(0, n_class, cnt), n_class)
def decode(_v):
return np.array([np.where(__v == 1)[0][0] for __v in _v])
img_recons = model1.reconstruct(x, c=y_encoded)
print('recon.shape', img_recons.shape)
y, y1, y2 = None, None, None
img_gens1, y1 = model1.generate(args.fid_cnt, generate_label)
img_gens2, y2 = model2.generate(args.fid_cnt, generate_label)
logger.debug('recon.shape: {}, img1.shape: {}, img2.shape: {}'
''.format(img_recons.shape, img_gens1.shape, img_gens2.shape))
y1 = decode(y1) if y1 is not None else None
y2 = decode(y2) if y2 is not None else None
col = 5 if args.dataset == 'taxinet' else 10
img_recons_sample, recon_inds = vae_util.stitch_imgs(img_recons, None,
row_size=n_class,
col_size=col)
print('img_recons_sample: {}, recon_inds: {}'.format(
img_recons_sample.shape, recon_inds))
# x = np.rint(x[recon_inds] * 255.0)
img_originals, _ = vae_util.stitch_imgs(x[recon_inds], y,
row_size=n_class, col_size=col)
print('img_originals', img_originals.shape)
img_originals = cv2.cvtColor(img_originals.astype(np.uint8),
cv2.COLOR_BGR2RGB)
# y1, y2
img_gens1_sample, _ = vae_util.stitch_imgs(img_gens1, y1,
row_size=n_class,
col_size=col)
img_gens2_sample, _ = vae_util.stitch_imgs(img_gens2, y2,
row_size=n_class,
col_size=col)
cv2.imwrite(os.path.join(args.exp_dir, 'recon_original.png'),
img_originals)
cv2.imwrite(os.path.join(args.exp_dir, 'recon_sample.png'),
vae_util.scale_up(img_recons_sample))
cv2.imwrite(os.path.join(args.exp_dir, 'gen1_sample.png'),
vae_util.scale_up(img_gens1_sample))
cv2.imwrite(os.path.join(args.exp_dir, 'gen2_sample.png'),
vae_util.scale_up(img_gens2_sample))
# calculating FID score
batches, parallel = 100, 4
tf1.reset_default_graph()
fid_recon = get_fid(img_recons, args.dataset, args.root_dir,
args.fid_cnt, num_batches=batches, parallel=parallel)
logger.info('FID = {:.2f}\n'.format(fid_recon))
fid_gen1 = get_fid(img_gens1, args.dataset, args.root_dir, args.fid_cnt,
num_batches=batches, parallel=parallel)
logger.info('FID = {:.2f}\n'.format(fid_gen1))
fid_gen2 = get_fid(img_gens2, args.dataset, args.root_dir, args.fid_cnt,
num_batches=batches, parallel=parallel)
logger.info('FID = {:.2f}\n'.format(fid_gen2))
logger.info('Reconstruction Results: FID = {:.2f}'.format(fid_recon))
logger.info('Generation Results (Stage 1): FID = {:.2f}'.format(fid_gen1))
logger.info('Generation Results (Stage 2): FID = {:.2f}'.format(fid_gen2))
with open(os.path.join(args.exp_dir, 'fid.txt'), 'w') as f:
f.write("recon: {:.2f}, 1st: {:.2f}, 2nd: {:.2f}\n".format(
fid_recon, fid_gen1, fid_gen2))
if args.train1 and args.wandb:
# wandb is initialized only when train1 is True
wandb.log({
'fid_recon': fid_recon,
'fid_gen1': fid_gen1,
'fid_gen2': fid_gen2,
})
kfold = StratifiedKFold(n_splits=10, random_state=222, shuffle=True)
test_auc, test_pr, preds = list(), list(), np.zeros((x.shape[0],))
for fold, (train_index, test_index) in enumerate(kfold.split(x, y)):
print("-" * 10 + "No. :", fold + 1)
y_train, y_test = y[train_index], y[test_index]
x_train, x_test = x[train_index], x[test_index]
# print(y_test.sum(), y_train.sum(), class_weight)
model = get_model()
if fold == 0:
model.summary()
pred = train_fold(model, x_train, y_train, x_test, y_test, x_test, y_test)
preds[test_index] = pred[:, 0]
test_auc.append(roc_auc_score(y_test, pred))
precision, recall, thresholds = precision_recall_curve(y_test, pred)
test_pr.append(auc(recall, precision))
print("-" * 10 + "No. :", fold + 1)
print("auroc = {:.4f} ".format(roc_auc_score(y_test, pred)) + "aupr = {:.4f} ".format(auc(recall, precision)))
# del model
keras.backend.clear_session()
v1.reset_default_graph()
print("10 fold auc :", test_auc)
print("10 fold mean auroc {:.5f}".format(np.mean(test_auc)))
print("10 fold mean aupr {:.5f}".format(np.mean(test_pr)))
def invert(
settings,
samples,
para_path,
g_tolerance=None,
e_tolerance=0.1,
n_iter=None,
max_iter=10000,
heuristic_sigma=None,
):
"""
Return the latent space points corresponding to a set of a samples
( from gradient descent )
Note: this function is designed for ONE sample generation
"""
# num_samples = samples.shape[0]
# cast samples to float32
samples = np.float32(samples)
# get the model
# if settings is a string, assume it's an identifier and load
if type(settings) == str:
settings = json.load(open("./experiments/settings/" + settings + ".txt", "r"))
# print('Inverting', 1, 'samples using model', settings['identifier'], 'at epoch', epoch,)
# if not g_tolerance is None:
# print('until gradient norm is below', g_tolerance)
# else:
# print('until error is below', e_tolerance)
# get parameters
parameters = model.load_parameters(para_path)
# # assertions
# assert samples.shape[2] == settings['num_generated_features']
# create VARIABLE Z
Z = tf.get_variable(
name="Z",
shape=[1, settings["seq_length"], settings["latent_dim"]],
initializer=tf.random_normal_initializer(),
)
# create outputs
G_samples = generator_o(
Z,
settings["hidden_units_g"],
settings["seq_length"],
1,
settings["num_generated_features"],
reuse=False,
parameters=parameters,
)
# generator_vars = ['hidden_units_g', 'seq_length', 'batch_size', 'num_generated_features', 'cond_dim', 'learn_scale']
# generator_settings = dict((k, settings[k]) for k in generator_vars)
# G_samples = model.generator(Z, **generator_settings, reuse=True)
fd = None
# define loss mmd-based loss
if heuristic_sigma is None:
heuristic_sigma = mmd.median_pairwise_distance_o(samples) # this is noisy
print("heuristic_sigma:", heuristic_sigma)
samples = tf.reshape(
samples, [1, settings["seq_length"], settings["num_generated_features"]]
)
Kxx, Kxy, Kyy, wts = mmd._mix_rbf_kernel(
G_samples, samples, sigmas=tf.constant(value=heuristic_sigma, shape=(1, 1))
)
similarity_per_sample = tf.diag_part(Kxy)
reconstruction_error_per_sample = 1 - similarity_per_sample
# reconstruction_error_per_sample = tf.reduce_sum((tf.nn.l2_normalize(G_samples, dim=1) - tf.nn.l2_normalize(samples, dim=1))**2, axis=[1,2])
similarity = tf.reduce_mean(similarity_per_sample)
reconstruction_error = 1 - similarity
# updater
# solver = tf.train.AdamOptimizer().minimize(reconstruction_error_per_sample, var_list=[Z])
# solver = tf.train.RMSPropOptimizer(learning_rate=500).minimize(reconstruction_error, var_list=[Z])
solver = tf.train.RMSPropOptimizer(learning_rate=0.1).minimize(
reconstruction_error_per_sample, var_list=[Z]
)
# solver = tf.train.MomentumOptimizer(learning_rate=0.1, momentum=0.9).minimize(reconstruction_error_per_sample, var_list=[Z])
grad_Z = tf.gradients(reconstruction_error_per_sample, Z)[0]
grad_per_Z = tf.norm(grad_Z, axis=(1, 2))
grad_norm = tf.reduce_mean(grad_per_Z)
# solver = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(reconstruction_error, var_list=[Z])
print("Finding latent state corresponding to samples...")
sess = tf.Session()
sess.run(tf.global_variables_initializer())
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
error = sess.run(reconstruction_error, feed_dict=fd)
g_n = sess.run(grad_norm, feed_dict=fd)
# print(g_n)
i = 0
if not n_iter is None:
while i < n_iter:
_ = sess.run(solver, feed_dict=fd)
error = sess.run(reconstruction_error, feed_dict=fd)
i += 1
else:
if not g_tolerance is None:
while g_n > g_tolerance:
_ = sess.run(solver, feed_dict=fd)
error, g_n = sess.run(
[reconstruction_error, grad_norm], feed_dict=fd
)
i += 1
print(error, g_n)
if i > max_iter:
break
else:
while np.abs(error) > e_tolerance:
_ = sess.run(solver, feed_dict=fd)
error = sess.run(reconstruction_error, feed_dict=fd)
i += 1
# print(error)
if i > max_iter:
break
Zs = sess.run(Z, feed_dict=fd)
Gs = sess.run(G_samples, feed_dict={Z: Zs})
error_per_sample = sess.run(reconstruction_error_per_sample, feed_dict=fd)
print("Z found in", i, "iterations with final reconstruction error of", error)
tf.reset_default_graph()
return Gs, Zs, error_per_sample, heuristic_sigma
def train_collect_eval(collect_env,
eval_env,
test_env,
policy_class,
run_agent_fn=run_env.run_env,
train_fn=train_q.train_q,
do_collect_eval=True,
file_patterns='',
get_data_fn=input_data.get_data,
onpolicy=True,
num_collect=100,
num_eval=100,
num_test=100,
data_format='tfrecord',
eval_frequency=5,
root_dir=None,
task=0,
master='',
ps_tasks=0):
"""Runs synchronous train, collect, eval loop.
This loop instantiates the policy instance from policy_class. The policy
manages its own tf.Session. The train function may create its own session for
the purpose of updating its variables.
train_fn reuses graph created by policy, to avoid having to
configure the same neural net twice (one for policy and one for training.)
Args:
collect_env: (gym.Env) Gym environment to collect data from (and train the
policy on).
eval_env: (gym.Env) Gym environment to evaluate the policy on. Can be
another instance of collect_env, or a different environment if one wishes
to evaluate generalization capability. The only constraint is that the
action and observation spaces have to be equivalent. If None, eval_env
is not evaluated.
test_env: (gym.Env) Another environment to evaluate on. Either another
instance of collect_env, or a different environment to evaluate
generalization.
policy_class: Policy class that we want to train.
run_agent_fn: (Optional) Python function that executes the interaction of
the policy with the environment. Defaults to run_env.run_env.
train_fn: (Optional) Python function that trains the policy. Defaults to
train_q.train_q.
do_collect_eval: If True, performs data collection using the trained policy.
file_patterns: (str) Comma-separated regex of file patterns to train on.
This is used to instantiate the file-backed "replay buffer".
get_data_fn: (Optional) Python function that fetches data from files.
onpolicy: (bool) If True, appends data from policy_collect directory.
num_collect: (int) Number of episodes to collect & evaluate from
collect_env.
num_eval: (int) Number of episodes to collect & evaluate from eval_env.
num_test: (int) Number of episodes to collect & evaluate from test_env.
data_format: (string) File extension of input data files.
eval_frequency: (int) How many times we run eval/test vs. collect.
Evaluating is costly compared to training, so we can speed up iteration
time by not evaluating every time we collect.
root_dir: (str) Root directory for this training trial. Training directory,
eval directory are subdirectories of root_dir.
task: (int) Optional worker task for distributed training. Defaults to solo
master task on a single machine
master: (int) Optional address of master worker. Specify this when doing
distributed training.
ps_tasks: (int) Optional number of parameter-server tasks. Used only for
distributed TF training jobs.
Raises:
ValueError: If ps_tasks > 0 (implies distributed training) while
do_collect_eval is set to True.
"""
# Spaces do not implement `==` operator. Convert to strings to check
# compatibility between training & eval env representation.
if ((collect_env and eval_env) and
(str(collect_env.observation_space), str(collect_env.action_space)) !=
(str(eval_env.observation_space), str(eval_env.action_space))):
raise ValueError('Collect and Eval environments have incompatible '
'observation or action dimensions.')
if ps_tasks > 0 and do_collect_eval:
raise ValueError(
'Collecting data not supported by distributed training jobs')
if onpolicy:
file_patterns += ',' + os.path.join(root_dir, 'policy_collect',
'*.%s' % data_format)
train_dir = os.path.join(root_dir, 'train')
it = 0
while True:
tf.reset_default_graph()
# Re-fresh the source of data.
with tf.Graph().as_default():
with tf.device(tf.train.replica_device_setter(ps_tasks)):
policy = policy_class()
if train_fn:
dataset = get_data_fn(file_patterns=file_patterns)
step, done = train_fn(dataset,
policy,
log_dir=train_dir,
reuse=True,
task=task,
master=master)
else:
step, done = 0, True
if train_fn:
tf.logging.info('Evaluating policy at step %d' % step)
ckpt = tf.train.latest_checkpoint(train_dir)
tf.logging.info('Restoring model variables from %s' % ckpt)
policy.restore(ckpt)
if ckpt:
step = int(ckpt.split('.ckpt-')[-1])
if onpolicy:
run_agent_fn(collect_env,
policy=policy,
global_step=step,
root_dir=root_dir,
task=task,
num_episodes=num_collect,
tag='collect')
if it % eval_frequency == 0:
if eval_env:
run_agent_fn(eval_env,
policy=policy,
global_step=step,
root_dir=root_dir,
task=task,
explore_schedule=None,
num_episodes=num_eval,
tag='eval')
if test_env:
run_agent_fn(test_env,
policy=policy,
global_step=step,
root_dir=root_dir,
task=task,
explore_schedule=None,
num_episodes=num_test,
tag='test')
it += 1
if done:
tf.logging.info('Train-Collect-Eval completed.')
break
def train_a_model(input_seq,
label_seq,
d_model,
head,
init_weights,
print_output=False):
# Clear all stuffs in default graph, so we can start fresh
tf.reset_default_graph()
with tf.device(USED_DEVICE):
# We want each session to have different random seed, but we need each run to have the same random sequence
tf.set_random_seed(random.randint(0, 65535))
batch_size = len(input_seq[0])
seq_len = len(input_seq[0][0])
sess = setup_tensorflow_session()
(input_tensor, output_tensor,
disagreement_cost) = build_model(batch=batch_size,
seq_len=seq_len,
d_model=d_model,
head=head)
(label_tensor, train_op, loss,
classification_loss) = build_train_graph(
output_tensor=output_tensor,
batch=batch_size,
seq_len=seq_len,
d_model=d_model,
additional_costs=[disagreement_cost])
sess.run(tf.global_variables_initializer())
if init_weights is not None:
set_all_variables(sess, init_weights)
for i in range(LOCAL_TRAIN_EPOCH):
avg_loss = 0.0
avg_disagreement_loss = 0.0
avg_classification_loss = 0.0
for input_sample, label_sample in zip(input_seq, label_seq):
[
output_vals, loss_vals, disagreement_cost_vals,
classification_loss_vals, _
] = sess.run([
output_tensor, loss, disagreement_cost,
classification_loss, train_op
],
feed_dict={
input_tensor: input_sample,
label_tensor: label_sample
})
avg_loss = avg_loss + loss_vals
avg_disagreement_loss = avg_disagreement_loss + disagreement_cost_vals
avg_classification_loss = avg_classification_loss + classification_loss_vals
avg_loss = avg_loss / len(input_seq)
avg_disagreement_loss = avg_disagreement_loss / len(input_seq)
avg_classification_loss = avg_classification_loss / len(
input_seq)
if print_output:
print('EPOCH: ' + str(i))
if print_output:
print('=== Input Values ===')
print(input_seq)
print('=== Label Values ===')
print(label_seq)
print('=== Output Values ===')
print(output_vals)
print('=== Loss Values ===')
print(avg_loss)
print('=== Classification Loss Values ===')
print(avg_classification_loss)
print('=== Disagreement Loss Values ===')
print(avg_disagreement_loss)
trained_weights = get_all_variables(sess)
return [
avg_loss, avg_disagreement_loss, avg_classification_loss,
trained_weights
]
def _test_tf_model(
self,
graph,
input_shapes,
output_node_names,
data_mode='random',
input_refs=None,
delta=1e-2,
use_cpu_only=False,
graph_optimizations="freeze", # one of ["freeze", "convert_variables_to_constants", None]
quantize_tf_model=False,
quantize_mlmodel=False,
quantize_config={}):
"""
Common entry to testing routine.
graph - defined TensorFlow graph.
input_shapes - dict str:shape for each input op (placeholder)
output_node_names - output_node_names, a list of strings
data_mode - auto-generated input vectors, can be 'random', 'zeros', 'ones', 'linear', etc.
input_refs - a dictionary of reference input in tensorFlow axis order, each entry is str:shape.
When using auto-generated input vectors, set input_refs to None.
delta - maximum difference of normalized TensorFlow and CoreML outputs
use_cpu_only - If True, instantiate and run CoreML model with CPU only
graph_optimizations == "freeze" - Force TensorFlow graph to be frozen before converting.
quantize_tf_model - If True, try to quantize TensorFlow model before converting
quantize_mlmodel - If True, quantize the mlmodel after converting.
quantize_config - Dictionary with test quantization parameters
"""
# Some file processing
model_dir = tempfile.mkdtemp()
graph_def_file = os.path.join(model_dir, 'tf_graph.pb')
checkpoint_file = os.path.join(model_dir, 'tf_model.ckpt')
static_model_file = os.path.join(model_dir, 'tf_static.pb')
coreml_model_file = os.path.join(model_dir, 'coreml_model.mlmodel')
# add a saver
tf.reset_default_graph()
if graph_optimizations == "freeze":
with graph.as_default() as g:
saver = tf.train.Saver()
if input_refs is None:
feed_dict = {
self._get_tf_tensor_name(graph, name):
generate_data(input_shapes[name], data_mode)
for name in input_shapes
}
else:
feed_dict = {
self._get_tf_tensor_name(graph, name): input_refs[name]
for name in list(input_refs.keys())
}
with tf.Session(graph=graph) as sess:
# initialize
initializer_op = tf.global_variables_initializer()
sess.run(initializer_op)
# run the result
fetches = [
graph.get_operation_by_name(name).outputs[0]
for name in output_node_names
]
result = sess.run(fetches, feed_dict=feed_dict)
# save graph definition somewhere
tf.train.write_graph(sess.graph,
model_dir,
graph_def_file,
as_text=False)
# save the weights if freezing is needed
if not graph_optimizations:
static_model_file = graph_def_file
elif graph_optimizations == "freeze":
saver.save(sess, checkpoint_file)
self._simple_freeze(
input_graph=graph_def_file,
input_checkpoint=checkpoint_file,
output_graph=static_model_file,
output_node_names=",".join(output_node_names))
else:
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, graph.as_graph_def(), output_node_names)
with tf.gfile.GFile(static_model_file, "wb") as f:
f.write(output_graph_def.SerializeToString())
# if TF needs to be quantized, quantize the graph
if quantize_tf_model:
static_model_file = self._quantize_static_tf_model(
model_dir, static_model_file, output_node_names)
# convert to CoreML
mlmodel = coremltools.converters.tensorflow.convert(
static_model_file,
inputs=input_shapes,
outputs=output_node_names,
use_cpu_only=use_cpu_only)
# Quantize MLModel if needed
if quantize_mlmodel:
from coremltools.models.neural_network.quantization_utils import quantize_weights
nbits = quantize_config['nbits']
mode = quantize_config['mode']
mlmodel = quantize_weights(mlmodel, nbits, quantization_mode=mode)
if DEBUG:
print('\n mlmodel description: \n')
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(mlmodel.get_spec(), style='coding')
mlmodel.save(coreml_model_file)
print('\n mlmodel saved at %s' % coreml_model_file)
# Transpose input data as CoreML requires
coreml_inputs = {
name: tf_transpose(feed_dict[self._get_tf_tensor_name(graph,
name)])
for name in input_shapes
}
# Run predict in CoreML
coreml_output = mlmodel.predict(coreml_inputs, useCPUOnly=use_cpu_only)
for idx, out_name in enumerate(output_node_names):
tf_out = result[idx]
if len(tf_out.shape) == 0:
tf_out = np.array([tf_out])
tp = tf_out.flatten()
coreml_out = coreml_output[out_name]
cp = coreml_out.flatten()
self.assertTrue(tf_out.shape == coreml_out.shape)
for i in range(len(tp)):
max_den = max(1.0, tp[i], cp[i])
self.assertAlmostEqual(tp[i] / max_den,
cp[i] / max_den,
delta=delta)
# Cleanup files - models on disk no longer useful
if os.path.exists(model_dir):
shutil.rmtree(model_dir)
def test_pors_model_pretrain_methods(self, pretrain_as_autoencoder,
lm_pretrain_dec, nsp_pretrain,
cpp_pretrain_scheme, use_tpu,
pretrain_order):
FLAGS.out_domain_pretrain_steps = 1
FLAGS.in_domain_pretrain_steps = 1
FLAGS.pretrain_as_autoencoder = pretrain_as_autoencoder
self.params.update({
'add_critic': False,
'vocab_size': 11,
'pretrain_order': pretrain_order,
'first_pretrain_steps': 1,
'lm_pretrain_dec': lm_pretrain_dec,
'nsp_pretrain': nsp_pretrain,
'cpp_pretrain_scheme': cpp_pretrain_scheme,
'encoder_type': 'transformer'
})
FLAGS.use_tpu = use_tpu
tf.reset_default_graph()
# Batch of 4 paragraphs each with 5 sentences.
sids = tf.constant([[[3, 2, 1, 0, 0], [4, 2, 4, 1, 0], [2, 1, 0, 0, 0],
[4, 2, 4, 1, 0], [2, 1, 0, 0, 0]],
[[5, 2, 3, 1, 0], [4, 2, 5, 1, 0], [5, 1, 0, 0, 0],
[4, 2, 5, 1, 0], [5, 1, 0, 0, 0]],
[[3, 2, 1, 0, 0], [4, 2, 4, 1, 0], [2, 1, 0, 0, 0],
[4, 2, 4, 1, 0], [2, 1, 0, 0, 0]],
[[5, 2, 3, 1, 0], [4, 2, 5, 1, 0], [5, 1, 0, 0, 0],
[4, 2, 5, 1, 0], [5, 1, 0, 0, 0]]],
dtype=tf.int64)
pids = tf.constant([[3, 2, 4, 2, 4, 2, 4, 2, 4, 2, 1, 0],
[5, 2, 3, 4, 2, 5, 5, 4, 2, 5, 5, 1],
[3, 2, 4, 2, 4, 2, 4, 2, 4, 2, 1, 0],
[5, 2, 3, 4, 2, 5, 5, 4, 2, 5, 5, 1]],
dtype=tf.int64)
features = {'sentences': sids, 'paragraphs': pids}
ret_tensors = pors.pors_model(features,
self.params,
True,
spid_dict={pors.MASK: 10})
# Verify that ae_vars and d_vars is all vars.
ae_vars = tf.trainable_variables('ae_')
d_vars = tf.trainable_variables('disc_')
self.assertEqual(set(tf.trainable_variables()), set(ae_vars + d_vars))
with self.session() as ss:
ss.run(tf.initializers.global_variables())
ss.run(tf.initializers.local_variables())
ss.run(tf.tables_initializer())
# pre-train the first component on out-domain data for 1 step
# If simultaneous or autoencoder then encoder and decoder are jointly
# pre-trained.
loss, _, _, _ = ss.run(ret_tensors[:-1])
self.assertGreater(loss, 0)
# pre-train the second component on in-domain data for 1 step
loss, _, _, _ = ss.run(ret_tensors[:-1])
self.assertGreater(loss, 0)
# 1 regular training step
loss, _, _, _ = ss.run(ret_tensors[:-1])
self.assertGreater(loss, 0)
def run(logdir, session_id, hparams, group_name):
"""Runs a temperature simulation.
This will simulate an object at temperature `initial_temperature`
sitting at rest in a large room at temperature `ambient_temperature`.
The object has some intrinsic `heat_coefficient`, which indicates
how much thermal conductivity it has: for instance, metals have high
thermal conductivity, while the thermal conductivity of water is low.
Over time, the object's temperature will adjust to match the
temperature of its environment. We'll track the object's temperature,
how far it is from the room's temperature, and how much it changes at
each time step.
Arguments:
logdir: the top-level directory into which to write summary data
session_id: an id for the session.
hparams: A dictionary mapping a hyperparameter name to its value.
group_name: an id for the session group this session belongs to.
"""
tf.reset_default_graph()
tf.set_random_seed(0)
initial_temperature = hparams['initial_temperature']
ambient_temperature = hparams['ambient_temperature']
heat_coefficient = HEAT_COEFFICIENTS[hparams['material']]
session_dir = os.path.join(logdir, session_id)
writer = tf.summary.FileWriter(session_dir)
writer.add_summary(summary.session_start_pb(hparams=hparams,
group_name=group_name))
writer.flush()
with tf.name_scope('temperature'):
# Create a mutable variable to hold the object's temperature, and
# create a scalar summary to track its value over time. The name of
# the summary will appear as 'temperature/current' due to the
# name-scope above.
temperature = tf.Variable(
tf.constant(initial_temperature),
name='temperature')
scalar_summary.op('current', temperature,
display_name='Temperature',
description='The temperature of the object under '
'simulation, in Kelvins.')
# Compute how much the object's temperature differs from that of its
# environment, and track this, too: likewise, as
# 'temperature/difference_to_ambient'.
ambient_difference = temperature - ambient_temperature
scalar_summary.op('difference_to_ambient', ambient_difference,
display_name='Difference to ambient temperature',
description=('The difference between the ambient '
'temperature and the temperature of the '
'object under simulation, in Kelvins.'))
# Newton suggested that the rate of change of the temperature of an
# object is directly proportional to this `ambient_difference` above,
# where the proportionality constant is what we called the heat
# coefficient. But in real life, not everything is quite so clean, so
# we'll add in some noise. (The value of 50 is arbitrary, chosen to
# make the data look somewhat interesting. :-) )
noise = 50 * tf.random.normal([])
delta = -heat_coefficient * (ambient_difference + noise)
scalar_summary.op('delta', delta,
description='The change in temperature from the previous '
'step, in Kelvins.')
# Collect all the scalars that we want to keep track of.
summ = tf.summary.merge_all()
# Now, augment the current temperature by this delta that we computed,
# blocking the assignment on summary collection to avoid race conditions
# and ensure that the summary always reports the pre-update value.
with tf.control_dependencies([summ]):
update_step = temperature.assign_add(delta)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for step in xrange(FLAGS.num_steps):
# By asking TensorFlow to compute the update step, we force it to
# change the value of the temperature variable. We don't actually
# care about this value, so we discard it; instead, we grab the
# summary data computed along the way.
(s, _) = sess.run([summ, update_step])
if (step % FLAGS.summary_freq) == 0:
writer.add_summary(s, global_step=step)
writer.add_summary(summary.session_end_pb(api_pb2.STATUS_SUCCESS))
writer.close()
def train_model(X_batches, Y_batches, batch_size, seq_length, k, epochs,
hidden_units, learning_rate, d, mask, num_layers, config,
model):
"""
:param X_batches: input batches
:param Y_batches: target batches
:param batch_size: batch size
:param seq_length: length of the sequences
:param k: vocab size
:param epochs: number of epochs for the training
:param hidden_units: list containing the number of hidden units for each LSTM cell
:param learning_rate: learning rate
:param d: output_keep_prob of the dropout applied after all the LSTM cells
:param mask: mask of the valid characters
:param num_layers: number of LSTM cells
:param config: settings of the session
:param model: name of the model
"""
# Create session
tf.reset_default_graph()
session = tf.Session(config=config)
# Create model and set parameter
X, Y, S, M, Z, dropout, state, loss, train = net_param(
hidden_units, learning_rate, num_layers, batch_size, seq_length, k)
session.run(tf.global_variables_initializer())
f_train = open('out/' + model + '/train.txt', "w")
for e in range(0, epochs):
print("Starting train…")
train_start = time.time()
print('Epoch: {}.'.format(e))
cum_loss = 0
cum_sum = 0
current_state = np.zeros((num_layers, 2, batch_size, hidden_units[0]))
for i in range(X_batches.shape[0]):
batch_loss, _, current_state, output = session.run(
[loss, train, state, Z],
feed_dict={
X: X_batches[i],
Y: Y_batches[i],
S: current_state,
dropout: d,
M: mask[i]
})
cum_sum += np.sum(mask[i])
cum_loss += batch_loss * np.sum(mask[i])
print('Batch: ' + str(i) + '\tLoss: ' + str(batch_loss))
epoch_loss = cum_loss / cum_sum
train_end = time.time()
train_time = train_end - train_start
print('Train Loss: {:.2f}. Train Time: {} sec.'.format(
epoch_loss, train_time))
f_train.write(
str(e) + ', ' + str(epoch_loss) + ',' + str(train_time) + '\n')
f_train.close()
saver = tf.train.Saver()
saver.save(session, 'train/')
def export_pytorch_checkpoint_to_tf(model,
ckpt_dir,
bert_output_prefix="bert",
appended_val_map=(),
appended_tensors_to_transpose=()):
""" Export PyTorch BERT model to TF Checkpoint
Args:
model (`nn.Module`) : The PyTorch model you want to save
ckpt_dir (`str) : The directory of exporting checkpoint
bert_output_prefix (`str`) : The prefix of BERT module, e.g. bert_pre_trained_model for EasyTransfer
appended_val_map (`tuple`): A tuple of tuples, ( (PyTorch_var_name, Tensorflow_var_name), ...) )
appended_tensors_to_transpose (`tuple`): A tuple of PyTorch tensor names you need to transpose
"""
try:
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import re
import numpy as np
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
except ImportError:
logger.info(
"Export a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see "
"https://www.tensorflow.org/install/ for installation instructions."
)
raise RuntimeError
def to_tf_var_name(name):
for patt, repl in iter(var_map):
name = name.replace(patt, repl)
return name
def create_tf_var(tensor, name, session):
tf_dtype = tf.dtypes.as_dtype(tensor.dtype)
tf_var = tf.get_variable(dtype=tf_dtype,
shape=tensor.shape,
name=name,
initializer=tf.zeros_initializer())
session.run(tf.variables_initializer([tf_var]))
session.run(tf_var)
return tf_var
var_map = appended_val_map + (
("layer.", "layer_"), ("word_embeddings.weight", "word_embeddings"),
("position_embeddings.weight", "position_embeddings"),
("token_type_embeddings.weight", "token_type_embeddings"), (".", "/"),
("LayerNorm/weight", "LayerNorm/gamma"),
("LayerNorm/bias", "LayerNorm/beta"), ("/weight", "/kernel"))
tensors_to_transpose = (
"dense.weight", "attention.self.query", "attention.self.key",
"attention.self.value") + appended_tensors_to_transpose
if not os.path.isdir(ckpt_dir):
io.makedirs(ckpt_dir)
state_dict = model.state_dict()
have_cls_predictions = False
have_cls_seq_relationship = False
for key in state_dict.keys():
if key.startswith("cls.predictions"):
have_cls_predictions = True
if key.startswith("cls.seq_relationship"):
have_cls_seq_relationship = True
if not have_cls_predictions:
state_dict["cls.predictions.output_bias"] = torch.zeros(
model.config.vocab_size)
state_dict["cls.predictions.transform.LayerNorm.beta"] = torch.zeros(
model.config.hidden_size)
state_dict["cls.predictions.transform.LayerNorm.gamma"] = torch.zeros(
model.config.hidden_size)
state_dict["cls.predictions.transform.dense.bias"] = torch.zeros(
model.config.hidden_size)
state_dict["cls.predictions.transform.dense.kernel"] = torch.zeros(
(model.config.hidden_size, model.config.hidden_size))
if not have_cls_seq_relationship:
state_dict["cls.seq_relationship.output_weights"] = torch.zeros(
(2, model.config.hidden_size))
state_dict["cls.seq_relationship.output_bias"] = torch.zeros(2)
tf.reset_default_graph()
with tf.Session() as session:
for var_name in state_dict:
tf_name = to_tf_var_name(var_name)
torch_tensor = state_dict[var_name].cpu().numpy()
if var_name.startswith("bert.") or var_name.startswith("cls."):
prefix = bert_output_prefix + "/" if bert_output_prefix else ""
else:
prefix = ""
tf_name = prefix + tf_name
if any([x in var_name for x in tensors_to_transpose]):
torch_tensor = torch_tensor.T
tf_var = create_tf_var(tensor=torch_tensor,
name=tf_name,
session=session)
tf.keras.backend.set_value(tf_var, torch_tensor)
# tf_weight = session.run(tf_var)
# print("Successfully created {}: {}".format(tf_name, np.allclose(tf_weight, torch_tensor)))
create_tf_var(tensor=np.array(1), name="global_step", session=session)
saver = tf.train.Saver(tf.trainable_variables())
if "oss://" in ckpt_dir:
saver.save(session, "model.ckpt")
for fname in io.listdir("./"):
if fname.startswith("model.ckpt"):
local_file = fname
oss_file = os.path.join(ckpt_dir, fname)
logger.info("uploading %s" % oss_file)
io.upload(local_file, oss_file)
else:
saver.save(session, os.path.join(ckpt_dir, "model.ckpt"))
def prepare_model2_weights_file():
tf.reset_default_graph()
m2 = Model2()
m2.train(X, Y, testX, testY, 1)
m2.model.save("model2.tfl")
y_test = np.concatenate((y_test,y_test_temp), axis=0)
y_val = np.concatenate((y_val,y_val_temp), axis=0)
# In[6]: AE용 train, test
# RNN Input이므로 맨뒤 1 제거
X_train = np.reshape(X_train, [np.shape(X_train)[0], np.shape(X_train)[1], np.shape(X_train)[2]])
#X_val = np.reshape(X_val, [np.shape(X_val)[0], np.shape(X_val)[1], np.shape(X_val)[2]])
X_test = np.reshape(X_test, [np.shape(X_test)[0], np.shape(X_test)[1], np.shape(X_test)[2]])
# In[7]: AE용 train, test
tf.reset_default_graph() # 텐서들 전부 flush
MODEL_PATH = './model/'
new_saver = tf.train.import_meta_graph(os.path.join(MODEL_PATH, 'trained_Atlas_v3.ckpt.meta')) # 저장된 tensor들 load
class Model_BILSTM_AE:
def __init__(self, sess, name):
self.sess = sess
self.name = name
def _build_net(self,num_hidden, num_rnn_layer, num_hidden_rnn, m_idx):
with tf.variable_scope(self.name):
def main(unused_argv=None):
tf.logging.set_verbosity(FLAGS.log)
if FLAGS.checkpoint_path:
checkpoint_path = utils.shell_path(FLAGS.checkpoint_path)
else:
expdir = utils.shell_path(FLAGS.expdir)
tf.logging.info("Will load latest checkpoint from %s.", expdir)
while not tf.gfile.Exists(expdir):
tf.logging.fatal("\tExperiment save dir '%s' does not exist!",
expdir)
sys.exit(1)
try:
checkpoint_path = tf.train.latest_checkpoint(expdir)
except tf.errors.NotFoundError:
tf.logging.fatal(
"There was a problem determining the latest checkpoint.")
sys.exit(1)
if not tf.train.checkpoint_exists(checkpoint_path):
tf.logging.fatal("Invalid checkpoint path: %s", checkpoint_path)
sys.exit(1)
tf.logging.info("Will restore from checkpoint: %s", checkpoint_path)
source_path = utils.shell_path(FLAGS.source_path)
tf.logging.info("Will load Wavs from %s." % source_path)
save_path = utils.shell_path(FLAGS.save_path)
tf.logging.info("Will save embeddings to %s." % save_path)
if not tf.gfile.Exists(save_path):
tf.logging.info("Creating save directory...")
tf.gfile.MakeDirs(save_path)
sample_length = FLAGS.sample_length
batch_size = FLAGS.batch_size
def is_wav(f):
return f.lower().endswith(".wav")
wavfiles = sorted([
os.path.join(source_path, fname)
for fname in tf.gfile.ListDirectory(source_path) if is_wav(fname)
])
for start_file in xrange(0, len(wavfiles), batch_size):
batch_number = (start_file / batch_size) + 1
tf.logging.info("On file number %s (batch %d).", start_file,
batch_number)
end_file = start_file + batch_size
wavefiles_batch = wavfiles[start_file:end_file]
# Ensure that files has batch_size elements.
batch_filler = batch_size - len(wavefiles_batch)
wavefiles_batch.extend(batch_filler * [wavefiles_batch[-1]])
wav_data = [
utils.load_audio(f, sample_length) for f in wavefiles_batch
]
min_len = min([x.shape[0] for x in wav_data])
wav_data = np.array([x[:min_len] for x in wav_data])
try:
tf.reset_default_graph()
# Load up the model for encoding and find the encoding
encoding = encode(wav_data,
checkpoint_path,
sample_length=sample_length)
if encoding.ndim == 2:
encoding = np.expand_dims(encoding, 0)
tf.logging.info("Encoding:")
tf.logging.info(encoding.shape)
tf.logging.info("Sample length: %d" % sample_length)
for num, (wavfile, enc) in enumerate(zip(wavefiles_batch,
encoding)):
filename = "%s_embeddings.npy" % wavfile.split("/")[-1].strip(
".wav")
with tf.gfile.Open(os.path.join(save_path, filename),
"w") as f:
np.save(f, enc)
if num + batch_filler + 1 == batch_size:
break
except Exception as e:
tf.logging.info("Unexpected error happened: %s.", e)
raise
def tf_execute_helper(data_cache, epoch, partitions, checkpoint_path, input_paths,
input_fn_string, model_fn_string, train_fn_string, mst, train=True):
"""
:param data_cache:
:param epoch:
:param partitions:
:param checkpoint_path:
:param input_paths:
:param input_fn_string:
:param model_fn_string:
:param train_fn_string:
:param mst:
:param train:
:return:
"""
begin_time = time.time()
tf.reset_default_graph()
input_fn = dill.loads(base64.b64decode(input_fn_string))
model_fn = dill.loads(base64.b64decode(model_fn_string))
train_fn = dill.loads(base64.b64decode(train_fn_string))
losses = []
errors = []
message = ""
for partition, input_path in zip(partitions, input_paths):
tf.reset_default_graph()
if input_path in data_cache:
# already cached
data = data_cache[input_path]
else:
data = input_fn(input_path)
data_cache[input_path] = data
opt, loss, error = model_fn(data, mst)
train_step = opt.minimize(loss, colocate_gradients_with_ops=True)
saver = tf.train.Saver()
gpu_options = tf.GPUOptions(allow_growth = True)
sess = tf.Session(
config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options))
if os.path.exists(checkpoint_path + ".index"):
saver.restore(sess, checkpoint_path)
else:
sess.run(tf.global_variables_initializer())
message += "\nEPOCH: %d, PARTITION: %d, Model Building and Session Initialization Time: %d\n" % (
epoch, partition, time.time() - begin_time)
time_begin = time.time()
loss_val, error_val = train_fn(sess, train_step, loss, error, train=train)
losses.append(loss_val)
errors.append(error_val)
elapsed_time = time.time() - time_begin
mode = "TRAIN" if train else "VALID"
message += "EPOCH: %d, PARTITION: %d, %s LOSS: %f, ERROR: %f, Time: %f\n" % (epoch, partition, mode, loss_val, error_val, elapsed_time)
if train:
begin_time = time.time()
saver.save(sess, checkpoint_path)
message += "EPOCH: %d, PARTITION: %d, Checkpoint Save Time: %d\n" % (epoch, partition, time.time() - begin_time)
sess.close()
return {'loss': losses, 'error': errors, 'message': message[1:]}
