def train_step(step, boards, supers):
boards = pentago.unpack_boards(boards)
supers = pentago.unpack_supers(supers)
with tf.GradientTape() as tape:
logits = M(boards)
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=supers))
accuracy = tf.equal(tf.argmax(logits, axis=-1, output_type=tf.int32), supers)
accuracy = tf.reduce_mean(tf.cast(accuracy, tf.float32))
grads = tape.gradient(loss, M.trainable_variables)
optimizer.apply_gradients(zip(grads, M.trainable_variables))
tf.print(tf.strings.format('step {}: loss {}, accuracy {}', (step, loss, accuracy)))
with summary_writer.as_default(), tf.contrib.summary.always_record_summaries():
tf.contrib.summary.scalar('loss', loss, step=step)
tf.contrib.summary.scalar('accuracy', accuracy, step=step)
def testPrint(self):
# tf.print() cannot be parsed unless we import print_function
text = """from __future__ import print_function
tf.print()
tf.print('abc')
"""
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, text) # Text should stay the same
from core.yolov3 import YOLOv3, decode, compute_loss
from core.config import cfg
# gpus = tf.config.experimental.list_physical_devices('GPU') # return all GPUs
# tf.config.experimental.set_memory_growth(device=gpus[0], enable=True) # dynamically use GPU memory
# os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
"""get training images and labels"""
trainset = Dataset('train') # creat a Dataset object which will be used for generating batch_image, (batch_smaller_label, batch_medium_label, batch_larger_label)
logdir = "C:/PycharmProjects/YOLOV3_Github/data/log" # file for saving logs
steps_per_epoch = len(trainset) # 250
global_steps = tf.Variable(1, trainable=False, dtype=tf.int64)
warmup_steps = cfg.TRAIN.WARMUP_EPOCHS * steps_per_epoch # 2 epochs
total_steps = cfg.TRAIN.EPOCHS * steps_per_epoch # 30 train epochs
tf.print('steps_per_epoch:%d' %steps_per_epoch)
"""build model and load weights"""
input_tensor = tf.keras.layers.Input([416, 416, 3]) # create an input object with tensor shape (416,416,3), no batch is needed
conv_tensors = YOLOv3(input_tensor) # chain input layer and hidden layers, get output [[b,52,52,3*(5+c)], [b,26,26,3*(5+c)], [b,13,13,3*(5+c)]]
output_tensors = []
for i, conv_tensor in enumerate(conv_tensors):
pred_tensor = decode(conv_tensor, i)
output_tensors.append(conv_tensor) # output layer [batch, grid, grid, 3*(5+c)]
output_tensors.append(pred_tensor) # decoded output layer [batch, grid, grid, 3*(5+c)]
model = tf.keras.Model(input_tensor, output_tensors) # generate a model object based on input layer and output layer
utils.load_weights(model, "./yolov3.weights") # load weights
model.summary()
# compute output
W_eff = W * w_hat
y = tf.matmul(x, W_eff) + b
y = activation(y)
return y, sparse_reg_term + l2_reg_term
# Instantiate network
y = R
reg_losses = None
for i in range(n_layers):
y, reg_loss = kernel_layer(y, n_hid, name=str(i))
reg_losses = reg_loss if reg_losses is None else reg_losses + reg_loss
prediction, reg_loss = kernel_layer(y, n_u, activation=tf.identity, name='out')
tf.print(prediction)
reg_losses = reg_losses + reg_loss
# Compute loss (symbolic)
diff = tm * (R - prediction)
sqE = tf.nn.l2_loss(diff)
loss = sqE + reg_losses
# Instantiate L-BFGS Optimizer
optimizer = tf.contrib.opt.ScipyOptimizerInterface(loss,
options={
'maxiter': output_every,
'disp': verbose_bfgs,
'maxcor': 10
},
method='L-BFGS-B')
import tensorflow as tf
mammal = tf.Variable("Elephant", tf.string)
ignition = tf.Variable(451, tf.int16)
floating = tf.Variable(3.14159265359, tf.float64)
its_complicated = tf.Variable(12.3 - 4.85j, tf.complex64)
mystr = tf.Variable(["Hello"], tf.string)
cool_numbers = tf.Variable([3.14159, 2.71828], tf.float32)
first_primes = tf.Variable([2, 3, 5, 7, 11], tf.int32)
its_very_complicated = tf.Variable([12.3 - 4.85j, 7.5 - 6.23j], tf.complex64)
mymat = tf.Variable([[7],[11]], tf.int16)
myxor = tf.Variable([[False, True],[True, False]], tf.bool)
linear_squares = tf.Variable([[4], [9], [16], [25]], tf.int32)
squarish_squares = tf.Variable([ [4, 9], [16, 25] ], tf.int32)
rank_of_squares = tf.rank(squarish_squares)
mymatC = tf.Variable([[7],[11]], tf.int32)
my_image = tf.zeros([10, 299, 299, 3]) # batch x height x width x color
r = tf.rank(my_image)
# After the graph runs, r will hold the value 4.
print(my_image)
print(r)
tf.print(my_image)
def call(self, inputs, cps=None, isFirstLayer=False):
if isFirstLayer:
tf.print("computed cp for the last input", cps[-1])
bOnA = inputs - self.firstLayerTA0 - cps / (self.firstLayerK1M *
self.E0)
tf.print("computed bOna for the last input", bOnA[-1])
tf.print("second parts",
(4 * inputs * cps / (self.firstLayerK1M * self.E0))[-1])
tf.assert_rank(
self.firstLayerK1M * self.E0 * self.firstLayerTA0 / cps,
tf.rank(inputs))
olderInput = tf.where(
tf.equal(
self.firstLayerK1M * self.E0 * self.firstLayerTA0 / cps,
0), inputs,
0.5 * (bOnA + (tf.pow(bOnA, 2) + 4 * inputs * cps /
(self.firstLayerK1M * self.E0))**0.5))
tf.print("solution for the last inputs of first layer: ",
olderInput[-1])
olderInput = self.firstLayerk2 * self.firstLayerK1M / self.firstLayerkdT * self.firstLayerTA0 * self.E0 / cps * olderInput / (
1 + self.firstLayerK1M * self.E0 / cps * olderInput)
else:
olderInput = inputs
#For batch computation: k1m * tf.transpose(inputs) doesn't work and we need to add a 1 axis in the end and use only *
#Just above we use rank1 vector so should keep rank2 batches of input
cpsExpand = tf.expand_dims(cps, -1)
tf.assert_rank(cpsExpand, 3)
tf.assert_rank(cps, 2)
olderInputExpand = tf.expand_dims(olderInput, -1)
tf.assert_rank(olderInputExpand, 3)
olderInputMidExpand = tf.expand_dims(olderInput, 1)
Cactivs = tf.where(
self.mask > 0, self.Cactiv /
(1 + self.k1M * self.E0 * olderInputExpand / cpsExpand), 0)
Cinhibs = tf.where(
self.mask < 0, self.Cinhib /
(1 + self.k3M * self.E0 * olderInputExpand / cpsExpand), 0)
tf.assert_rank(Cactivs, 3)
tf.assert_rank(Cinhibs, 3)
Inhib = tf.squeeze(tf.matmul(olderInputMidExpand, Cinhibs),
axis=1) / self.kdT
x_eq_clipped = tf.squeeze(tf.matmul(olderInputMidExpand, Cactivs),
axis=1) / (self.kdI * cps + Inhib / cps)
#unclipped version:
Cactivs_unclipped = tf.where(
self.kernel > 0, self.Cactiv * self.kernel /
(1 + self.k1M * self.E0 * olderInputExpand / cpsExpand), 0)
Cinhibs_unclipped = tf.where(
self.kernel < 0, (-1) * self.Cinhib * self.kernel /
(1 + self.k3M * self.E0 * olderInputExpand / cpsExpand), 0)
tf.assert_rank(Cactivs_unclipped, 3)
tf.assert_rank(Cinhibs_unclipped, 3)
#CAREFUL: now the cactivs has taken the batch size, it is of rank 3 : [None,inputdims,outputdims]
# THUS WE NEED: [None,1,inputdims] to use the matmul, and then squeeze the result!
Inhib_unclipped = tf.squeeze(tf.matmul(olderInputMidExpand,
Cinhibs_unclipped),
axis=1) / self.kdT
x_eq_unclipped = tf.squeeze(
tf.matmul(olderInputMidExpand, Cactivs_unclipped),
axis=1) / (self.kdI * cps + Inhib_unclipped / cps)
tf.assert_rank(tf.squeeze(tf.matmul(olderInputMidExpand,
Cinhibs_unclipped),
axis=1),
2,
message="compute not good dims")
tf.print("solution from an unknown layer:", x_eq_clipped[-1])
outputs = tf.stop_gradient(x_eq_clipped -
x_eq_unclipped) + x_eq_unclipped
tf.assert_rank(outputs, 2, message="outputs not good dims")
return outputs
def tryBreak():
for idx in tf.range(100):
tf.print(idx)
if (tf.equal(idx, 10)):
break
def main(_):
tf.random.set_seed(FLAGS.seed)
logdir = FLAGS.logdir
os.makedirs(logdir, exist_ok=True)
genmo_lr = tf.constant(FLAGS.genmo_lr)
infnet_lr = tf.constant(FLAGS.infnet_lr)
prior_lr = tf.constant(FLAGS.prior_lr)
genmo_optimizer = tf.keras.optimizers.Adam(learning_rate=genmo_lr)
infnet_optimizer = tf.keras.optimizers.Adam(learning_rate=infnet_lr)
prior_optimizer = tf.keras.optimizers.SGD(learning_rate=prior_lr)
theta_optimizer = tf.keras.optimizers.Adam(learning_rate=infnet_lr,
beta_1=0.999)
batch_size = FLAGS.batch_size
if FLAGS.dataset == 'static_mnist':
train_ds, valid_ds, test_ds = dataset.get_static_mnist_batch(
batch_size)
train_size = 50000
elif FLAGS.dataset == 'dynamic_mnist':
train_ds, valid_ds, test_ds = dataset.get_dynamic_mnist_batch(
batch_size)
train_size = 50000
elif FLAGS.dataset == 'fashion_mnist':
train_ds, valid_ds, test_ds = dataset.get_dynamic_mnist_batch(
batch_size, fashion_mnist=True)
train_size = 50000
elif FLAGS.dataset == 'omniglot':
train_ds, valid_ds, test_ds = dataset.get_omniglot_batch(batch_size)
train_size = 23000
num_steps_per_epoch = int(train_size / batch_size)
train_ds_mean = dataset.get_mean_from_iterator(train_ds,
dataset_size=train_size,
batch_size=batch_size)
if FLAGS.initialize_with_bias:
bias_value = -tf.math.log(
1. / tf.clip_by_value(train_ds_mean, 0.001, 0.999) - 1.)
bias_initializer = tf.keras.initializers.Constant(bias_value)
else:
bias_initializer = 'zeros'
encoder = [
networks.BinaryNetwork([200], ['linear'],
mean_xs=train_ds_mean,
demean_input=FLAGS.demean_input,
name='bvae_encoder1'),
networks.BinaryNetwork([200], ['linear'], name='bvae_encoder2'),
networks.BinaryNetwork([200], ['linear'], name='bvae_encoder3'),
networks.BinaryNetwork([200], ['linear'], name='bvae_encoder4')
]
decoder = [
networks.BinaryNetwork([200], ['linear'], name='bvae_decoder1'),
networks.BinaryNetwork([200], ['linear'], name='bvae_decoder2'),
networks.BinaryNetwork([200], ['linear'], name='bvae_decoder3'),
networks.BinaryNetwork([784], ['linear'],
demean_input=FLAGS.demean_input,
final_layer_bias_initializer=bias_initializer,
name='bvae_decoder4')
]
prior_logit = tf.Variable(tf.zeros([200], tf.float32))
if FLAGS.grad_type == 'relax':
control_network = []
for _ in range(len(encoder)):
control_network_i = tf.keras.Sequential()
control_network_i.add(
layers.Dense(137, activation=layers.LeakyReLU(alpha=0.3)))
control_network_i.add(layers.Dense(1))
control_network.append(control_network_i)
else:
control_network = None
bvae_model = networks.DiscreteVAE(
encoder,
decoder,
prior_logit,
grad_type=FLAGS.grad_type,
control_nn=control_network,
shared_randomness=FLAGS.shared_randomness)
bvae_model.build(input_shape=(None, 784))
tensorboard_file_writer = tf.summary.create_file_writer(logdir)
# In order to use `tf.train.ExponentialMovingAverage`, one has to
# use `tf.Variable`.
encoder_grad_variable = initialize_grad_variables(bvae_model.encoder_vars)
encoder_grad_sq_variable = initialize_grad_variables(
bvae_model.encoder_vars)
if FLAGS.estimate_grad_basket:
if FLAGS.grad_type == 'reinforce_loo':
grad_basket = ['arm', 'disarm', 'reinforce_loo', 'relax']
else:
raise NotImplementedError
grad_variable_dict = {
k: initialize_grad_variables(bvae_model.encoder_vars)
for k in grad_basket
}
grad_sq_variable_dict = {
k: initialize_grad_variables(bvae_model.encoder_vars)
for k in grad_basket
}
ckpt = tf.train.Checkpoint(
genmo_optimizer=genmo_optimizer,
infnet_optimizer=infnet_optimizer,
theta_optimizer=theta_optimizer,
encoder_grad_variable=encoder_grad_variable,
encoder_grad_sq_variable=encoder_grad_sq_variable,
grad_variable_dict=grad_variable_dict,
grad_sq_variable_dict=grad_sq_variable_dict,
bvae_model=bvae_model)
else:
grad_variable_dict = None
grad_sq_variable_dict = None
ckpt = tf.train.Checkpoint(
genmo_optimizer=genmo_optimizer,
infnet_optimizer=infnet_optimizer,
theta_optimizer=theta_optimizer,
encoder_grad_variable=encoder_grad_variable,
encoder_grad_sq_variable=encoder_grad_sq_variable,
bvae_model=bvae_model)
ckpt_manager = tf.train.CheckpointManager(ckpt, logdir, max_to_keep=5)
if not FLAGS.debug and ckpt_manager.latest_checkpoint:
ckpt.restore(ckpt_manager.latest_checkpoint)
logging.info('Last checkpoint was restored: %s.',
ckpt_manager.latest_checkpoint)
else:
tf.print('No checkpoint to load.')
logging.info('No checkpoint to load.')
start_step = infnet_optimizer.iterations.numpy()
logging.info('Training start from step: %s', start_step)
train_iter = train_ds.__iter__()
for step_i in range(start_step, FLAGS.num_steps):
(encoder_grad_var, variance_dict, genmo_loss,
metrics) = train_one_step(train_iter.next(), bvae_model,
genmo_optimizer, infnet_optimizer,
prior_optimizer, theta_optimizer,
encoder_grad_variable,
encoder_grad_sq_variable,
grad_variable_dict, grad_sq_variable_dict)
train_loss = tf.reduce_mean(genmo_loss)
# Summarize
if step_i % 1000 == 0:
metrics.update({
'train_objective':
train_loss,
'eval_metric/train':
evaluate(bvae_model,
train_ds,
max_step=num_steps_per_epoch,
num_eval_samples=FLAGS.num_train_samples),
'eval_metric/valid':
evaluate(bvae_model,
valid_ds,
num_eval_samples=FLAGS.num_eval_samples),
'eval_metric/test':
evaluate(bvae_model,
test_ds,
num_eval_samples=FLAGS.num_eval_samples),
'var/grad':
encoder_grad_var
})
if FLAGS.grad_type == 'relax':
if FLAGS.temperature is None:
metrics['relax/temperature'] = tf.math.exp(
bvae_model.log_temperature_variable)
if FLAGS.scaling_factor is None:
metrics['relax/scaling'] = bvae_model.scaling_variable
tf.print(step_i, metrics)
with tensorboard_file_writer.as_default():
for k, v in metrics.items():
tf.summary.scalar(k, v, step=step_i)
if variance_dict is not None:
tf.print(variance_dict)
for k, v in variance_dict.items():
tf.summary.scalar(k, v, step=step_i)
# Checkpoint
if step_i % 10000 == 0:
ckpt_save_path = ckpt_manager.save()
logging.info('Saving checkpoint for step %d at %s.', step_i,
ckpt_save_path)
if FLAGS.bias_check:
if FLAGS.grad_type == 'reinforce_loo':
baseline_type = 'disarm'
else:
baseline_type = 'reinforce_loo'
run_bias_check(bvae_model, train_iter.next(), FLAGS.grad_type,
baseline_type)
def test_print(x):
op = tf.print(x)
with tf.control_dependencies([op]):
return tf.identity(op)
# Configure the training
train = train.batch(32).prefetch(1)
validation = validation.batch(32).prefetch(1)
test = test.batch(32).prefetch(1)
num_epochs = 10
for epoch in range(num_epochs):
start = time()
for example in train:
image, label = example["image"], example["label"]
# print('size of image:', np.shape(image))
loss_value = train_step(image, label)
mean_loss.update_state(loss_value)
if tf.equal(tf.math.mod(step, 10), 0):
tf.print(step, "loss: ", mean_loss.result(), "accuracy: ",
accuracy.result())
mean_loss.reset_states()
accuracy.reset_states()
end = time()
print("Time per epoch: ", end - start)
# Validation
tf.print("## Validation - ", epoch)
accuracy.reset_states()
for example in validation:
image, label = example["image"], example["label"]
logits = model(image)
accuracy.update_state(label, tf.argmax(logits, -1))
tf.print("accuracy: ", accuracy.result())
accuracy.reset_states()
def save_teacher(self):
self.teacher_ckpt.step.assign_add(1)
save_path = self.teacher_manager.save()
tf.print("Saved teacher checkpoint", save_path)
def build_training_graph(self, training_data) -> List[tf.Tensor]:
# model parameters and initial values
Wconv1 = weight_variable(
[self.KERNEL, self.KERNEL, self.IN_CHANNELS, self.HIDDEN_CHANNELS],
1.)
bconv1 = bias_variable([1, 1, self.HIDDEN_CHANNELS])
Wconv2 = weight_variable([
self.KERNEL, self.KERNEL, self.HIDDEN_CHANNELS,
self.HIDDEN_CHANNELS
], 1.)
bconv2 = bias_variable([1, 1, self.HIDDEN_CHANNELS])
Wfc1 = weight_variable([self.HIDDEN_FC1, self.HIDDEN_FC2], 1.)
bfc1 = bias_variable([self.HIDDEN_FC2])
Wfc2 = weight_variable([self.HIDDEN_FC2, self.OUT_N], 1.)
bfc2 = bias_variable([self.OUT_N])
params = [Wconv1, bconv1, Wconv2, bconv2, Wfc1, bfc1, Wfc2, bfc2]
# optimizer and data pipeline
optimizer = tf.train.AdamOptimizer(learning_rate=self.LEARNING_RATE)
# training loop
def loop_body(
i: tf.Tensor, max_iter: tf.Tensor, nb_epochs: tf.Tensor,
avg_loss: tf.Tensor
) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor]:
# get next batch
x, y = training_data.get_next()
# model construction
x = tf.reshape(x, [-1, self.IN_DIM, self.IN_DIM, 1])
layer1 = pooling(
tf.nn.relu(conv2d(x, Wconv1, self.STRIDE) + bconv1))
layer2 = pooling(
tf.nn.relu(conv2d(layer1, Wconv2, self.STRIDE) + bconv2))
layer2 = tf.reshape(layer2, [-1, self.HIDDEN_FC1])
layer3 = tf.nn.relu(tf.matmul(layer2, Wfc1) + bfc1)
logits = tf.matmul(layer3, Wfc2) + bfc2
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits,
labels=y))
is_end_epoch = tf.equal(i % max_iter, 0)
def true_fn() -> tf.Tensor:
return loss
def false_fn() -> tf.Tensor:
return (tf.cast(i - 1, tf.float32) * avg_loss +
loss) / tf.cast(i, tf.float32)
with tf.control_dependencies([optimizer.minimize(loss)]):
return i + 1, max_iter, nb_epochs, tf.cond(
is_end_epoch, true_fn, false_fn)
loop, _, _, _ = tf.while_loop(self.cond, loop_body,
[0, self.ITERATIONS, self.EPOCHS, 0.])
# return model parameters after training
loop = tf.print("Training complete", loop)
with tf.control_dependencies([loop]):
return [param.read_value() for param in params]
def on_train_end(self, logs=None):
tf.print(f'Best val_loss was on epoch #{self.best_epoch}')
def true_fn() -> tf.Tensor:
to_continue = tf.print("avg_loss: ", avg_loss)
return to_continue
def test_production_step():
stack = new_stack(((STACK_SIZE, TOKEN_DIM)))
output = new_stack(((STACK_SIZE, TOKEN_DIM)))
stack = safe_push(stack, tf.constant(S, dtype=tf.float32), is_phi)
production = tf.one_hot(2, PRODUCTION_DIM)
phi = tf.one_hot(0, TOKEN_DIM, dtype=tf.float32)
with tf.GradientTape(persistent=True) as tape:
tape.watch(grammar)
tape.watch(production)
tape.watch(stack)
tape.watch(output)
new_s, new_o = production_step(grammar, production, stack, output, phi,
is_phi)
tf.print(tokens_pretty_print(new_s[0]))
tf.print(tape.gradient(new_o, output))
tf.print(tape.gradient(new_s, stack))
tf.print(tape.gradient(new_s[0], grammar[0]).shape)
tf.print(tape.gradient(new_s[1], grammar[0]).shape)
tf.print(tape.gradient(new_o[0], grammar[1]).shape)
tf.print(tape.gradient(new_o[1], grammar[1]).shape)
tf.print(tape.gradient(new_s, production))
def test_generate():
tf.config.experimental_run_functions_eagerly(True)
productions = tf.one_hot([2, 3, 0, 1, 0], PRODUCTION_DIM)
stack_shape = (STACK_SIZE, TOKEN_DIM)
d_S = tf.constant(S, dtype=tf.float32)
d_phi = tf.constant(tf.one_hot([0], TOKEN_DIM))
with tf.GradientTape(persistent=True) as tape:
tape.watch(productions)
output, final_stack = generate(grammar, productions, stack_shape, d_S,
d_phi, is_phi, True)
tf.config.experimental_run_functions_eagerly(False)
tf.print('Final result:')
tf.print(tokens_pretty_print(output))
tf.print('-' * 80)
tf.print('Final stack:')
tf.print(tokens_pretty_print(final_stack))
tf.print('-' * 80)
tf.print(tape.gradient(output, productions))
#세션 학습
import tensorflow as tf
# tensorflow 라이브러리를 tf로 쓴다
# tf.Session()경우 텐서플로우 버전 1.x.x에서 사용하는 표현방식이므로
# 확인결과, 2.X 버전 경우 Session을 정의하고 Run해주는 과정이 생략된다.
# 버전 확인 방법print(tf.__version__)
node = tf.constant('Hello , My name is Jaewon')
node1 = tf.constant(3.0)
node2 = tf.constant(4.0)
tf.print(node, node1, node2)
#constant는 한 번 정해지면 변하지 않는 , 상수항을 의미.
############################################### show ###############################################
# 显示训练集和验证集的acc和loss曲线
acc = history.history['sparse_categorical_accuracy']
loss = history.history['loss']
plt.subplot(1, 2, 1)
plt.plot(acc, label='Training Accuracy')
plt.title('Training Accuracy')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(loss, label='Training Loss')
plt.title('Training Loss')
plt.legend()
plt.show()
############### predict #############
preNum = int(input("input the number of test alphabet:"))
for i in range(preNum):
alphabet1 = input("input test alphabet:")
alphabet = [id_to_onehot[w_to_id[alphabet1]]]
# 使alphabet符合SimpleRNN输入要求：[送入样本数， 循环核时间展开步数， 每个时间步输入特征个数]。此处验证效果送入了1个样本，送入样本数为1；输入1个字母出结果，所以循环核时间展开步数为1; 表示为独热码有5个输入特征，每个时间步输入特征个数为5
alphabet = np.reshape(alphabet, (1, 1, 5))
result = model.predict([alphabet])
pred = tf.argmax(result, axis=1)
pred = int(pred)
tf.print(alphabet1 + '->' + input_word[pred])
def print_examples(x):
if np.random.uniform() < debug_print_examples_rate:
tf.print(x, output_stream=logging.info)
return x
def get_string(x, i):
return ' '.join(id_to_word[id] for id in x[i] if id > 3)
features = [get_string(x, i) for i in range(x.shape[0])]
features = np.array(features)
targets = y
dhtml(features[0], c2, f2, fs2)
dhtml('Data Building')
ds=tf.data.Dataset\
.from_tensor_slices((features,targets))
for ex in ds.take(3):
tf.print(ex[0].numpy()[:50], ex[1])
tf.random.set_seed(123)
ds = ds.shuffle(50000, reshuffle_each_iteration=False)
ds_test = ds.take(10000)
ds_train_valid = ds.skip(10000)
ds_valid = ds_train_valid.take(10000)
ds_train = ds_train_valid.skip(10000)
tokenizer = tfds.features.text.Tokenizer()
token_counts = Counter()
for example in ds_train:
tokens = tokenizer.tokenize(example[0].numpy())
token_counts.update(tokens)
print('vocabulary size:', len(token_counts))
import tensorflow as tf # You can create constants in TF to hold specific values a = tf.constant(1) b = tf.constant(2) c = a + b d = a * b tf.print(c) // output: 3 tf.print(d) // output: 2
# Broadcasting import tensorflow as tf x = tf.constant([1, 2, 3]) y = tf.constant(2) z = tf.constant([2, 2, 2]) tf.print(tf.multiply(x, 2)) #[2 4 6] tf.print(x * y) #[2 4 6] tf.print(x * z) #[2 4 6] x = tf.reshape(x, [3, 1]) y = tf.range(1, 5) print(tf.multiply(x, y)) # [[ 1 2 3 4] # [ 2 4 6 8] # [ 3 6 9 12]] x_stretch = tf.constant([[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]) y_stretch = tf.constant([[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]]) print(x_stretch * y_stretch) print(tf.broadcast_to(tf.constant([1, 2, 3, 4]), [3, 4]))
def loss(self, y_true, y_pred, *args):
positives = tf.reduce_sum(y_true[:, :, 1:-4], axis=1)
N = tf.reduce_sum(positives)
if N == 0:
return tf.convert_to_tensor(0, dtype=tf.float32)
loc = self.loc_loss(y_true, y_pred)
conf_loss_pos, amount_pos = self.conf_loss_pos(y_true, y_pred)
conf_loss_neg = self.conf_loss_neg(y_true, y_pred, amount_pos)
positives = tf.reduce_max(y_true[:, :, 1:-4], axis=-1)
loc_loss = tf.reduce_sum(loc * positives)
loss = (conf_loss_pos + conf_loss_neg + 1 * loc_loss) / N
if verbose > 0 and tf.random.uniform((1, ))[0] > (1 - verbose % 1):
tf.print()
if verbose >= 2:
tf.print('pred: ', y_pred[:, :, -4:])
tf.print('loc_loss: ', loc_loss / N)
tf.print('conf_loss_pos: ', conf_loss_pos / N)
tf.print('conf_loss_neg: ', conf_loss_neg / N)
tf.print('loc_loss: ', loc_loss / N)
tf.print('loss: ', loss)
return tf.convert_to_tensor(loss, dtype=tf.float32)
def create_mnist_dataset(batch_size, split, sess_curr,
assignments_curr) -> Tuple[tf.data.Dataset, int]:
"""
Creates a Dataset for MNIST Data.
This function create the correct tf.data.Dataset for a given split, transforms and
batch inputs.
"""
global assignments
assignments = assignments_curr
images = read_mnist_images(split)
labels = read_mnist_labels(split)
img_ids = np.arange(len(images))
print(len(img_ids))
def gen():
for image, label, img_id in zip(images, labels, img_ids):
# assignment = sess.run(assignments_curr[img_id // 100])
# assignment = assignments_curr[img_id // 100][0]
yield ((img_id, image), label)
if split == 'train':
ds = (tf.data.Dataset.from_generator(
gen,
output_types=((tf.int32, tf.uint8), tf.uint8),
output_shapes=((tf.TensorShape([]), (28, 28, 1)), (1, ))))
temp = ds.apply(split_and_merge)
tf.print("tf.print: ", temp, output_stream=sys.stdout)
batch = (temp.batch(batch_size).map(transform_train).repeat())
return batch, len(labels)
elif split == 'val':
batch = (
tf.data.Dataset.from_generator(
gen,
output_types=((tf.int32, tf.uint8), tf.uint8),
output_shapes=((tf.TensorShape([]), (28, 28, 1)), (1, )))
# .apply(split_and_merge)
.batch(batch_size).map(transform_val).repeat())
return batch, len(labels)
# if __name__ == "__main__":
# global assignments
# sess = tf.InteractiveSession()
# # assignments = {}
# # table = tf.contrib.lookup.MutableHashTable(key_dtype=tf.int64, value_dtype=tf.Variable, default_value=-1, empty_key=0)
# # table = tf.contrib.lookup.MutableDenseHashTable(key_dtype=tf.int64, value_dtype=tf.Variable, default_value=-1, empty_key=0)
# keys = tf.constant([0, 1, 2, 3], dtype=tf.int64)
# # assignments = []
# vals = []
# for i in range(4):
# # assignments.append(tf.Variable(np.random.randint(0, 2), dtype=tf.int32))
# vals.append(tf.Variable(np.random.randint(0, 2), dtype=tf.int32))
# vals = tf.constant([vals], dtype=tf.Variable)
# sess.run(tf.global_variables_initializer())
# table = tf.contrib.lookup.HashTable(tf.contrib.lookup.KeyValueTensorInitializer(keys, vals),-1)
# table.init.run()
# print("run")
# print(sess.run(table.lookup(tf.range(4, dtype=tf.int64))))
# insert_op = table.insert(keys, vals)
# sess.run(insert_op)
# print(sess.run(table.lookup(keys)))
# Can't pass assignments as an argument to gen, otherwise it will be
# evaluated and passed to generator as NumPy-array arguments
# batch = (tf.data.Dataset
# .from_generator(gen,
# output_types=((tf.float32, tf.int32), tf.int32),
# output_shapes=((tf.TensorShape([]), tf.TensorShape([])), tf.TensorShape([])))
# .apply(split_and_merge)
# .batch(2)
# .make_one_shot_iterator()
# .get_next())
# sess = tf.InteractiveSession()
# sess.run(tf.global_variables_initializer())
# for _ in range(5):
# print(sess.run(batch))
def f(x):
tf.print(tf.shape(x)[0])
return tf.shape(x)
def train(ds_train, ds_val, model, optimizer, ckpt, saved_weights_path,
last_epoch, last_step, max_epochs, steps_per_epoch):
train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)
train_loss_heatmap = tf.keras.metrics.Mean('train_loss_heatmap',
dtype=tf.float32)
train_loss_paf = tf.keras.metrics.Mean('train_loss_paf', dtype=tf.float32)
val_loss = tf.keras.metrics.Mean('val_loss', dtype=tf.float32)
val_loss_heatmap = tf.keras.metrics.Mean('val_loss_heatmap',
dtype=tf.float32)
val_loss_paf = tf.keras.metrics.Mean('val_loss_paf', dtype=tf.float32)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = 'logs/gradient_tape/' + current_time + '/train'
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
val_log_dir = 'logs/gradient_tape/' + current_time + '/val'
val_summary_writer = tf.summary.create_file_writer(val_log_dir)
output_paf_idx = 10
output_heatmap_idx = 11
# determine start epoch in case the training has been stopped manually and resumed
resume = last_step != 0 and (steps_per_epoch - last_step) != 0
if resume:
start_epoch = last_epoch
else:
start_epoch = last_epoch + 1
# start processing
for epoch in range(start_epoch, max_epochs + 1, 1):
print("Start processing epoch {}".format(epoch))
# set the initial step index depending on if you resumed the processing
if resume:
step = last_step + 1
data_iter = ds_train.skip(last_step)
print(f"Skipping {last_step} steps (May take a few minutes)...")
resume = False
else:
step = 0
data_iter = ds_train
# process steps
for x, y in data_iter:
step += 1
losses, total_loss = train_one_step(model, optimizer, x, y)
train_loss(total_loss)
train_loss_heatmap(losses[output_heatmap_idx])
train_loss_paf(losses[output_paf_idx])
if step % 10 == 0:
tf.print('Epoch', epoch, f'Step {step}/{steps_per_epoch}',
': Loss paf', losses[10], 'Loss heatmap', losses[11],
'Total loss', total_loss)
with train_summary_writer.as_default():
summary_step = (epoch - 1) * steps_per_epoch + step - 1
tf.summary.scalar('loss',
train_loss.result(),
step=summary_step)
tf.summary.scalar('loss_heatmap',
train_loss_heatmap.result(),
step=summary_step)
tf.summary.scalar('loss_paf',
train_loss_paf.result(),
step=summary_step)
if step % 100 == 0:
figure = probe_model(model)
with train_summary_writer.as_default():
tf.summary.image("Test prediction",
plot_to_image(figure),
step=step)
if step % 1000 == 0:
ckpt.step.assign(step)
ckpt.epoch.assign(epoch)
save_path = manager.save()
print("Saved checkpoint for step {}: {}".format(
step, save_path))
if step >= steps_per_epoch:
break
print("Completed epoch {}. Saving weights...".format(epoch))
model.save_weights(saved_weights_path, overwrite=True)
# save checkpoint at the end of an epoch
ckpt.step.assign(step)
ckpt.epoch.assign(epoch)
manager.save()
# reset metrics every epoch
train_loss.reset_states()
train_loss_heatmap.reset_states()
train_loss_paf.reset_states()
# calculate validation loss
print("Calculating validation losses...")
for val_step, (x_val, y_val_true) in enumerate(ds_val):
if val_step % 1000 == 0:
print(f"Validation step {val_step} ...")
y_val_pred = model(x_val)
losses = [
eucl_loss(y_val_true[0], y_val_pred[0]),
eucl_loss(y_val_true[1], y_val_pred[1]),
eucl_loss(y_val_true[2], y_val_pred[2]),
eucl_loss(y_val_true[3], y_val_pred[3]),
eucl_loss(y_val_true[4], y_val_pred[4]),
eucl_loss(y_val_true[5], y_val_pred[5]),
eucl_loss(y_val_true[6], y_val_pred[6]),
eucl_loss(y_val_true[7], y_val_pred[7]),
eucl_loss(y_val_true[8], y_val_pred[8]),
eucl_loss(y_val_true[9], y_val_pred[9]),
eucl_loss(y_val_true[10], y_val_pred[10]),
eucl_loss(y_val_true[11], y_val_pred[11])
]
total_loss = tf.reduce_sum(losses)
val_loss(total_loss)
val_loss_heatmap(losses[output_heatmap_idx])
val_loss_paf(losses[output_paf_idx])
val_loss_res = val_loss.result()
val_loss_heatmap_res = val_loss_heatmap.result()
val_loss_paf_res = val_loss_paf.result()
print(
f'Validation losses for epoch: {epoch} : Loss paf {val_loss_paf_res}, Loss heatmap '
f'{val_loss_heatmap_res}, Total loss {val_loss_res}')
with val_summary_writer.as_default():
tf.summary.scalar('val_loss', val_loss_res, step=epoch)
tf.summary.scalar('val_loss_heatmap',
val_loss_heatmap_res,
step=epoch)
tf.summary.scalar('val_loss_paf', val_loss_paf_res, step=epoch)
val_loss.reset_states()
val_loss_heatmap.reset_states()
val_loss_paf.reset_states()
def tf_print(*args):
tf.print(*args)
return args[0]
grad = tape.gradient(_loss, train_weights)
optimizer.apply_gradients(zip(grad, train_weights))
# begin training
for idx, data in enumerate(gen):
start_time = time.time()
x_batch = tf.convert_to_tensor(data[0])
y_batch = tf.convert_to_tensor(data[1])
train_step(x_batch, y_batch)
end_time = time.time()
consume_time = end_time - start_time
total_time += consume_time
if idx % monitor_interval == 0:
cur_usage = psutil.Process(os.getpid()).memory_info().rss
max_mem_usage = max(cur_usage, max_mem_usage)
avg_mem_usage += cur_usage
count += 1
tf.print("[*] {} iteration: memory usage {:.2f}MB, consume time {:.4f}s".format(
idx, cur_usage / (1024 * 1024), consume_time))
print('consumed time:', total_time)
avg_mem_usage = avg_mem_usage / count / (1024 * 1024)
max_mem_usage = max_mem_usage / (1024 * 1024)
print('average memory usage: {:.2f}MB'.format(avg_mem_usage))
print('maximum memory usage: {:.2f}MB'.format(max_mem_usage))
preNum = int(input("input the number of test pictures:"))
for i in range(preNum):
image_path = input("the path of test picture:")
img = Image.open(image_path)
image = plt.imread(image_path)
plt.set_cmap('gray')
plt.imshow(image)
img = img.resize((28, 28), Image.ANTIALIAS)
img_arr = np.array(img.convert('L'))
for i in range(28):
for j in range(28):
if img_arr[i][j] < 200:
img_arr[i][j] = 255
else:
img_arr[i][j] = 0
img_arr = img_arr / 255.0
x_predict = img_arr[tf.newaxis, ...]
result = model.predict(x_predict)
pred = tf.argmax(result, axis=1)
print('\n')
tf.print(pred)
plt.pause(1)
plt.close()
def __init__(self, cfg):
strategy = tf.distribute.MirroredStrategy()
self.num_replicas = strategy.num_replicas_in_sync
self.train_batch_size = self.num_replicas * cfg.train.dataset.batch_size
self.val_batch_size = self.num_replicas * cfg.val.dataset.batch_size
train_dataset = build_dataset(dataset=cfg.train.dataset.dataset,
dataset_dir=cfg.train.dataset.dataset_dir,
batch_size=self.train_batch_size,
training=cfg.train.dataset.training,
input_size=cfg.train.dataset.input_size,
augmentation=cfg.train.dataset.augmentation,
assigner=cfg.assigner.as_dict(),
anchor=cfg.anchor.as_dict() if cfg.anchor else None)
val_dataset = build_dataset(dataset=cfg.val.dataset.dataset,
dataset_dir=cfg.val.dataset.dataset_dir,
batch_size=self.val_batch_size,
training=cfg.val.dataset.training,
input_size=cfg.val.dataset.input_size,
augmentation=cfg.val.dataset.augmentation,
assigner=cfg.assigner.as_dict(),
anchor=cfg.anchor.as_dict() if cfg.anchor else None)
with strategy.scope():
self.train_dataset = strategy.experimental_distribute_dataset(train_dataset)
self.val_dataset = strategy.experimental_distribute_dataset(val_dataset)
use_mixed_precision = cfg.dtype == "float16"
if use_mixed_precision:
policy = tf.keras.mixed_precision.experimental.Policy("mixed_float16")
tf.keras.mixed_precision.experimental.set_policy(policy)
self.detector = build_detector(cfg.detector, cfg=cfg)
optimizer = build_optimizer(**cfg.train.optimizer.as_dict())
tf.print(_time_to_string(), "The optimizers is %s." % cfg.train.optimizer.optimizer)
if cfg.train.lookahead:
tf.print(_time_to_string(), "Using Lookahead Optimizer.")
optimizer = LookaheadOptimizer(optimizer, cfg.train.lookahead.steps, cfg.train.lookahead.alpha)
if use_mixed_precision:
loss_scale = (cfg.train.mixed_precision.loss_scale
if cfg.train.mixed_precision.loss_scale is not None and
cfg.train.mixed_precision.loss_scale > 0
else "dynamic")
optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(
optimizer=optimizer, loss_scale=loss_scale)
tf.print(_time_to_string(), "Using mixed precision training, loss scale is {}.".format(loss_scale))
self.optimizer = optimizer
self.strategy = strategy
self.use_mixed_precision = use_mixed_precision
self.cfg = cfg
self.total_train_steps = cfg.train.train_steps
self.warmup_steps = cfg.train.warmup.steps
self.warmup_learning_rate = cfg.train.warmup.warmup_learning_rate
self.initial_learning_rate = cfg.train.learning_rate_scheduler.initial_learning_rate
self._learning_rate_scheduler = build_learning_rate_scheduler(
**cfg.train.learning_rate_scheduler.as_dict(),
steps=self.total_train_steps,
warmup_steps=self.warmup_steps)
tf.print(_time_to_string(),
"The leaning rate scheduler is %s, initial learning rate is %f." % (
cfg.train.learning_rate_scheduler.learning_rate_scheduler,
cfg.train.learning_rate_scheduler.initial_learning_rate))
if self.warmup_steps > 0:
tf.print(_time_to_string(), "Using warm-up learning rate policy, "
"warm-up learning rate %f, training %d steps." % (self.warmup_learning_rate, self.warmup_steps))
tf.print(_time_to_string(), "Training", self.total_train_steps,
"steps, every step has", self.train_batch_size, "images.")
self.global_step = tf.Variable(initial_value=0,
trainable=False,
name="global_step",
dtype=tf.int64)
self.val_steps = tf.Variable(0, trainable=False, name="val_steps", dtype=tf.int64)
self.learning_rate = tf.Variable(initial_value=0,
trainable=False,
name="learning_rate",
dtype=tf.float32)
self.checkpoint = tf.train.Checkpoint(optimizer=self.optimizer, detector=self.detector.model)
self.manager = tf.train.CheckpointManager(checkpoint=self.checkpoint,
directory=cfg.train.checkpoint_dir,
max_to_keep=10)
latest_checkpoint = self.manager.latest_checkpoint
if latest_checkpoint is not None:
try:
steps = int(latest_checkpoint.split("-")[-1])
self.global_step.assign(steps)
except:
self.global_step.assign(0)
self.checkpoint.restore(latest_checkpoint)
tf.print(_time_to_string(), "Restored weights from %s." % latest_checkpoint)
else:
self.global_step.assign(0)
self.summary_writer = tf.summary.create_file_writer(logdir=cfg.train.summary_dir)
self.log_every_n_steps = cfg.train.log_every_n_steps
self.save_ckpt_steps = cfg.train.save_ckpt_steps
self.use_jit = tf.config.optimizer.get_jit() is not None
self.training_loss_metrics = {}
self.val_loss_metrics = {}
self.ap_metric = None
self._add_graph = True
def optimize(
self,
num_steps: int,
datasets: Mapping[str, Dataset] | Dataset,
model_specs: Mapping[str, ModelSpec] | ModelSpec,
acquisition_rule: AcquisitionRule[TensorType
| State[S | None, TensorType], SP]
| None = None,
acquisition_state: S | None = None,
*,
track_state: bool = True,
fit_initial_model: bool = True,
) -> OptimizationResult[S] | OptimizationResult[None]:
"""
Attempt to find the minimizer of the ``observer`` in the ``search_space`` (both specified at
:meth:`__init__`). This is the central implementation of the Bayesian optimization loop.
For each step in ``num_steps``, this method:
- Finds the next points with which to query the ``observer`` using the
``acquisition_rule``'s :meth:`acquire` method, passing it the ``search_space``,
``datasets``, models built from the ``model_specs``, and current acquisition state.
- Queries the ``observer`` *once* at those points.
- Updates the datasets and models with the data from the ``observer``.
If any errors are raised during the optimization loop, this method will catch and return
them instead, along with the history of the optimization process, and print a message (using
`absl` at level `logging.ERROR`).
**Note:** While the :class:`~trieste.models.TrainableProbabilisticModel` interface implies
mutable models, it is *not* guaranteed that the model passed to :meth:`optimize` will
be updated during the optimization process. For example, if ``track_state`` is `True`, a
copied model will be used on each optimization step. Use the models in the return value for
reliable access to the updated models.
**Type hints:**
- The ``acquisition_rule`` must use the same type of
:class:`~trieste.space.SearchSpace` as specified in :meth:`__init__`.
- The ``acquisition_state`` must be of the type expected by the ``acquisition_rule``.
Any acquisition state in the optimization result will also be of this type.
:param num_steps: The number of optimization steps to run.
:param datasets: The known observer query points and observations for each tag.
:param model_specs: The model to use for each :class:`~trieste.data.Dataset` in
``datasets``.
:param acquisition_rule: The acquisition rule, which defines how to search for a new point
on each optimization step. Defaults to
:class:`~trieste.acquisition.rule.EfficientGlobalOptimization` with default
arguments. Note that if the default is used, this implies the tags must be
`OBJECTIVE`, the search space can be any :class:`~trieste.space.SearchSpace`, and the
acquisition state returned in the :class:`OptimizationResult` will be `None`.
:param acquisition_state: The acquisition state to use on the first optimization step.
This argument allows the caller to restore the optimization process from an existing
:class:`Record`.
:param track_state: If `True`, this method saves the optimization state at the start of each
step. Models and acquisition state are copied using `copy.deepcopy`.
:param fit_initial_model: If `False`, this method assumes that the initial models have
already been optimized on the datasets and so do not require optimization before the
first optimization step.
:return: An :class:`OptimizationResult`. The :attr:`final_result` element contains either
the final optimization data, models and acquisition state, or, if an exception was
raised while executing the optimization loop, it contains the exception raised. In
either case, the :attr:`history` element is the history of the data, models and
acquisition state at the *start* of each optimization step (up to and including any step
that fails to complete). The history will never include the final optimization result.
:raise ValueError: If any of the following are true:
- ``num_steps`` is negative.
- the keys in ``datasets`` and ``model_specs`` do not match
- ``datasets`` or ``model_specs`` are empty
- the default `acquisition_rule` is used and the tags are not `OBJECTIVE`.
"""
if isinstance(datasets, Dataset):
datasets = {OBJECTIVE: datasets}
# ignore below is due to MyPy not being able to handle overloads properly
model_specs = {OBJECTIVE: model_specs} # type: ignore
# reassure the type checker that everything is tagged
datasets = cast(Dict[str, Dataset], datasets)
model_specs = cast(Dict[str, ModelSpec], model_specs)
if num_steps < 0:
raise ValueError(f"num_steps must be at least 0, got {num_steps}")
if datasets.keys() != model_specs.keys():
raise ValueError(
f"datasets and model_specs should contain the same keys. Got {datasets.keys()} and"
f" {model_specs.keys()} respectively.")
if not datasets:
raise ValueError(
"dicts of datasets and model_specs must be populated.")
if acquisition_rule is None:
if datasets.keys() != {OBJECTIVE}:
raise ValueError(
f"Default acquisition rule EfficientGlobalOptimization requires tag"
f" {OBJECTIVE!r}, got keys {datasets.keys()}")
acquisition_rule = cast(AcquisitionRule[TensorType, SP],
EfficientGlobalOptimization())
models = map_values(create_model, model_specs)
history: list[Record[S]] = []
for step in range(num_steps):
try:
if track_state:
models_copy = copy.deepcopy(models)
acquisition_state_copy = copy.deepcopy(acquisition_state)
history.append(
Record(datasets, models_copy, acquisition_state_copy))
if step == 0 and fit_initial_model:
for tag, model in models.items():
dataset = datasets[tag]
model.update(dataset)
model.optimize(dataset)
points_or_stateful = acquisition_rule.acquire(
self._search_space, datasets, models)
if callable(points_or_stateful):
acquisition_state, query_points = points_or_stateful(
acquisition_state)
else:
query_points = points_or_stateful
observer_output = self._observer(query_points)
tagged_output = (observer_output if isinstance(
observer_output, Mapping) else {
OBJECTIVE: observer_output
})
datasets = {
tag: datasets[tag] + tagged_output[tag]
for tag in tagged_output
}
for tag, model in models.items():
dataset = datasets[tag]
model.update(dataset)
model.optimize(dataset)
except Exception as error: # pylint: disable=broad-except
tf.print(
f"\nOptimization failed at step {step}, encountered error with traceback:"
f"\n{traceback.format_exc()}"
f"\nTerminating optimization and returning the optimization history. You may "
f"be able to use the history to restart the process from a previous successful "
f"optimization step.\n",
output_stream=logging.ERROR,
)
return OptimizationResult(Err(error), history)
tf.print("Optimization completed without errors",
output_stream=logging.INFO)
record = Record(datasets, models, acquisition_state)
return OptimizationResult(Ok(record), history)
def receive_output(average: tf.Tensor) -> tf.Operation:
# simply print average
return tf.print("Average:", average)
grad2 = tf.function(grad02)
for step in range(training_step):
sW1, sB1 = grad01(X1, X2, Y, w1, w2, b)
sW2, sB2 = grad02(X1, X2, Y, w1, w2, b)
dW1 = sW1 * learning_rate
dW2 = sW2 * learning_rate
dB1 = sB1 * learning_rate
dB2 = sB2 * learning_rate
w1.assign_sub(dW1) # subract dW from w
w2.assign_sub(dW2)
b.assign_sub((dB1 + dB2) / 2) # subract dB from b
los1 = loss(X1, X2, Y, w1, w2, b) #new loss function
tf.summary.scalar("loss", los1, step=step)
if step == 0 or step % display_step == 0:
print("Loss at step {:02d}: {:.6f}".format(step, los1))
plt.cla()
plt.scatter(X1, Y, c='red', label='x1')
plt.scatter(X2, Y, c='blue', label='x2')
plt.plot(TrX1, w1 * TrX1 + w2 * TrX2 + 0.2, 'r-', lw=5)
plt.text(0.5,
0,
'Step=%d\n Loss=%.5f' % (step, los1),
fontdict={
'size': 20,
'color': 'green'
})
plt.pause(0.5)
tf.print(w1) # Should be around 2
tf.print(w2) # Should be around 3
tf.print(b) # Should be around 0.2
def run(self):
with self.strategy.scope():
# `experimental_run_v2` replicates the provided computation and runs it
# with the distributed input.
@tf.function(experimental_relax_shapes=True, input_signature=self.train_dataset.element_spec)
def distributed_train_step(images, labels):
# @tf.function(experimental_relax_shapes=True)
def step_fn(batch_images, batch_labels):
normalized_images = tf.image.convert_image_dtype(batch_images, tf.float32)
with tf.GradientTape() as tape:
outputs = self.detector.model(normalized_images, training=True)
loss_dict = self.detector.losses(outputs, batch_labels)
loss = loss_dict["loss"] * (1. / self.num_replicas)
if self.use_mixed_precision:
scaled_loss = self.optimizer.get_scaled_loss(loss)
else:
scaled_loss = loss
self.optimizer.learning_rate = self.learning_rate.value()
gradients = tape.gradient(scaled_loss, self.detector.model.trainable_variables)
if self.use_mixed_precision:
gradients = self.optimizer.get_unscaled_gradients(gradients)
self.optimizer.apply_gradients(zip(gradients, self.detector.model.trainable_variables))
for key, value in loss_dict.items():
if key not in self.training_loss_metrics:
if key == "l2_loss":
self.training_loss_metrics[key] = metrics.NoOpMetric()
else:
self.training_loss_metrics[key] = tf.keras.metrics.Mean()
if key == "l2_loss":
self.training_loss_metrics[key].update_state(value / self.num_replicas)
else:
self.training_loss_metrics[key].update_state(value)
if self.global_step.value() % self.log_every_n_steps == 0:
# tf.print(self.optimizer.loss_scale._current_loss_scale, self.optimizer.loss_scale._num_good_steps)
matched_boxes, nmsed_boxes, _ = self.detector.summary_boxes(outputs, batch_labels)
batch_gt_boxes = batch_labels["gt_boxes"] * (1. / batch_labels["input_size"])
batch_images = tf.image.draw_bounding_boxes(images=batch_images,
boxes=batch_gt_boxes,
colors=tf.constant([[0., 0., 255.]]))
batch_images = tf.image.draw_bounding_boxes(images=batch_images,
boxes=matched_boxes,
colors=tf.constant([[255., 0., 0.]]))
batch_images = tf.image.draw_bounding_boxes(images=batch_images,
boxes=nmsed_boxes,
colors=tf.constant([[0., 255., 0.]]))
batch_images = tf.cast(batch_images, tf.uint8)
with tf.device("/cpu:0"):
with self.summary_writer.as_default():
tf.summary.image("train/images", batch_images, self.global_step, 5)
return loss
per_replica_losses = self.strategy.experimental_run_v2(step_fn, args=(images, labels))
return self.strategy.reduce(tf.distribute.ReduceOp.SUM,
per_replica_losses,
axis=None)
@tf.function(experimental_relax_shapes=True, input_signature=self.train_dataset.element_spec)
def distributed_valuate_step(images, labels):
def step_fn(batch_images, batch_labels):
normalized_images = tf.image.convert_image_dtype(batch_images, tf.float32)
outputs = self.detector.model(normalized_images, training=False)
loss_dict = self.detector.losses(outputs, batch_labels)
for key, value in loss_dict.items():
if key not in self.val_loss_metrics:
if key == "l2_loss":
self.val_loss_metrics[key] = metrics.NoOpMetric()
else:
self.val_loss_metrics[key] = tf.keras.metrics.Mean()
if key == "l2_loss":
self.val_loss_metrics[key].update_state(value / self.num_replicas)
else:
self.val_loss_metrics[key].update_state(value)
matched_boxes, nmsed_boxes, nmsed_scores = self.detector.summary_boxes(outputs, batch_labels)
batch_gt_boxes = batch_labels["gt_boxes"] * (1. / batch_labels["input_size"])
if self.val_steps.value() % self.log_every_n_steps == 0:
batch_images = tf.image.draw_bounding_boxes(images=batch_images,
boxes=batch_gt_boxes,
colors=tf.constant([[0., 0., 255.]]))
batch_images = tf.image.draw_bounding_boxes(images=batch_images,
boxes=matched_boxes,
colors=tf.constant([[255., 0., 0.]]))
batch_images = tf.image.draw_bounding_boxes(images=batch_images,
boxes=nmsed_boxes,
colors=tf.constant([[0., 255., 0.]]))
batch_images = tf.cast(batch_images, tf.uint8)
with tf.device("/cpu:0"):
with self.summary_writer.as_default():
tf.summary.image("valuate/images", batch_images, self.val_steps.value(), 5)
return batch_gt_boxes, nmsed_boxes, nmsed_scores
return self.strategy.experimental_run_v2(step_fn, args=(images, labels))
def learning_rate_scheduler(global_step):
global_step = tf.cast(global_step, tf.float32)
if tf.less(global_step, self.warmup_steps):
# decayed = 0.5 * (1. - tf.math.cos(math.pi * global_step / self.warmup_steps))
# return self.cfg.train.warmup.learning_rate * decayed
return ((self.initial_learning_rate - self.warmup_learning_rate)
* global_step / self.warmup_steps + self.warmup_learning_rate)
if self.cfg.train.learning_rate_scheduler.learning_rate_scheduler == "piecewise_constant":
return self._learning_rate_scheduler(global_step)
return self._learning_rate_scheduler(global_step - self.warmup_steps)
count = 0
max_ap = 0
self.ap_metric = metrics.AveragePrecision()
# TRAIN LOOP
start = time.time()
for images, image_info in self.train_dataset:
self.global_step.assign_add(1)
lr = learning_rate_scheduler(self.global_step.value())
self.learning_rate.assign(lr)
if self._add_graph:
tf.summary.trace_on(graph=True, profiler=True)
distributed_train_step(images, image_info)
with self.summary_writer.as_default():
tf.summary.trace_export(name=self.cfg.detector,
step=0,
profiler_outdir=self.cfg.train.summary_dir)
self._add_graph = False
else:
distributed_train_step(images, image_info)
count += 1
info = [_time_to_string(), "TRAINING", self.global_step]
if tf.equal(self.global_step % self.log_every_n_steps, 0):
with self.summary_writer.as_default():
for key in self.training_loss_metrics:
value = self.training_loss_metrics[key].result()
self.training_loss_metrics[key].reset_states()
tf.summary.scalar("train/" + key, value, self.global_step)
info.extend([key, "=", value])
tf.summary.scalar("learning_rate", self.learning_rate.value(), self.global_step)
info.extend(["lr", "=", self.learning_rate.value()])
info.append("(%.2fs)" % ((time.time() - start) / count))
tf.print(*info)
start = time.time()
count = 0
if self.global_step >= self.total_train_steps:
tf.print(_time_to_string(), "Train over.")
break
if tf.logical_and(self.global_step % self.cfg.val.val_every_n_steps == 0,
self.global_step > self.total_train_steps // 3):
tf.print("=" * 150)
# EVALUATING LOO
val_start = time.time()
for images, image_info in self.val_dataset:
self.val_steps.assign_add(1)
gt_boxes, pred_boxes, pred_scores = distributed_valuate_step(images, image_info)
gt_boxes = [b for x in tf.nest.flatten(gt_boxes) for b in self.strategy.unwrap(x)]
pred_boxes = [b for x in tf.nest.flatten(pred_boxes) for b in self.strategy.unwrap(x)]
pred_scores = [s for x in tf.nest.flatten(pred_scores) for s in self.strategy.unwrap(x)]
gt_boxes = tf.concat(gt_boxes, 0)
pred_boxes = tf.concat(pred_boxes, 0)
pred_scores = tf.concat(pred_scores, 0)
self.ap_metric.update_state(gt_boxes, pred_boxes, pred_scores)
val_end = time.time()
template = [_time_to_string(), "EVALUATING", self.global_step]
with self.summary_writer.as_default():
for name in self.val_loss_metrics:
value = self.val_loss_metrics[name].result()
tf.summary.scalar("val/" + name, value, self.global_step)
template.extend([name, "=", value])
self.val_loss_metrics[name].reset_states()
ap = self.ap_metric.result()
self.ap_metric.reset_states()
tf.summary.scalar("val/ap", ap, self.global_step)
template.extend(["ap =", ap, "(%.2fs)." % (val_end - val_start)])
tf.print(*template)
if ap > max_ap:
self.manager.save(self.global_step)
tf.print(_time_to_string(), "Saving detector to %s." % self.manager.latest_checkpoint)
max_ap = ap
start = time.time()
count = 0
else:
if tf.equal(self.global_step % self.save_ckpt_steps, 0):
self.manager.save(self.global_step)
tf.print(_time_to_string(), "Saving detector to %s." % self.manager.latest_checkpoint)
self.manager.save(self.global_step)
self.summary_writer.close()