def _enable_tensorboard(self):
if not self._tensor_board_dir:
tb = program.TensorBoard()
tb.configure(argv=[None, "--logdir", str(self._logdir)])
tb.launch()
self._tensor_board_dir = self._logdir
def main(unused_argv):
if FLAGS.clean_model_dir:
shutil.rmtree(FLAGS.model_dir, ignore_errors=True)
params = {
'output_stride': FLAGS.output_stride,
'batch_size': FLAGS.batch_size,
'base_architecture': FLAGS.base_architecture,
'pre_trained_model': FLAGS.pre_trained_model,
'batch_norm_decay': _BATCH_NORM_DECAY,
'num_classes': _NUM_CLASSES,
'tensorboard_images_max_outputs': FLAGS.tensorboard_images_max_outputs,
'weight_decay': FLAGS.weight_decay,
'learning_rate_policy': FLAGS.learning_rate_policy,
'num_train': _NUM_IMAGES['train'],
'initial_learning_rate': FLAGS.initial_learning_rate,
'max_iter': FLAGS.max_iter,
'end_learning_rate': FLAGS.end_learning_rate,
'power': _POWER,
'momentum': _MOMENTUM,
'freeze_batch_norm': FLAGS.freeze_batch_norm,
'initial_global_step': FLAGS.initial_global_step,
'resnet_size': FLAGS.resnet_size,
'kGender':50.0,
'kAge':1.0,
'learning_rate':FLAGS.initial_learning_rate,
'data_format':None,
}
# Set up a RunConfig to only save checkpoints once per training cycle.
run_config = tf.estimator.RunConfig().replace(save_checkpoints_secs=1e9)
model = tf.estimator.Estimator(
model_fn=resnet_model.resnetv2_model_fn,
model_dir=FLAGS.model_dir,
config=run_config,
params=params)
if(FLAGS.saveonly != True):
# Launch tensorboard for training
# Remove http messages
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
tb = program.TensorBoard()
tb.configure(argv=[None, '--logdir', FLAGS.model_dir, '--port', str(FLAGS.tbport), '--bind_all'])
url = tb.launch()
print('TensorBoard at {}'.format(url))
for step in range(FLAGS.train_epochs // FLAGS.epochs_per_eval):
tensors_to_log = {
#'learning_rate': 'learning_rate',
#'cross_entropy': 'cross_entropy',
#'train_px_accuracy': 'train_px_accuracy',
#'train_mean_iou': 'train_mean_iou',
}
logging_hook = tf.estimator.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=10)
train_hooks = [logging_hook]
eval_hooks = None
if FLAGS.debug_hooks:
debug_hook = tf_debug.LocalCLIDebugHook()
train_hooks.append(debug_hook)
eval_hooks = [debug_hook]
train_spec = tf.estimator.TrainSpec(input_fn=lambda: input_fn(True, FLAGS.data_dir, FLAGS.batch_size, FLAGS.epochs_per_eval) , max_steps=30000000)
#train_spec = tf.estimator.TrainSpec(input_fn=lambda: input_fn(True, FLAGS.data_dir, FLAGS.batch_size, FLAGS.epochs_per_eval))
eval_spec = tf.estimator.EvalSpec(input_fn=lambda: input_fn(False, FLAGS.data_dir, 1))
tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
savedmodel = model.export_saved_model('saved_model', serving_input_receiver_fn, experimental_mode=tf.estimator.ModeKeys.PREDICT, as_text=True)
savedmodelpath = savedmodel.decode('utf-8')
print('{} saved'.format(savedmodelpath))
converter = tf.lite.TFLiteConverter.from_saved_model(savedmodelpath)
converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_LATENCY]
samplefiles = get_samples(FLAGS.sample_dir, FLAGS.match)
converter.representative_dataset = lambda:representative_dataset_gen(samplefiles)
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
converter.inference_input_type = tf.uint8 # or tf.uint8
converter.inference_output_type = tf.uint8 # or tf.uint8
tflite_model = converter.convert()
outflite = './tflite/{}_int8.tflite'.format(defaultname)
open(outflite, "wb").write(tflite_model)
edgetpu_compile = 'edgetpu_compiler {} -o ./etpu'.format(outflite)
stream = os.popen(edgetpu_compile)
compileout = stream.read()
print(compileout)
def start_tensorboard(logdir):
tb = tb_program.TensorBoard(plugins=[core_plugin.CorePluginLoader()])
port = int(os.getenv('TB_PORT', 6006))
tb.configure(logdir=logdir, port=port)
tb.launch()
logging.info("Starting TensorBoard with --logdir=%s" % logdir)
digits_mnist = keras.datasets.mnist
(train_images, _), (test_images, _) = digits_mnist.load_data()
train_images = train_images.reshape(train_images.shape[0], 28, 28,
1).astype('float32')
test_images = test_images.reshape(test_images.shape[0], 28, 28,
1).astype('float32')
# Normalizing the images to the range of [0., 1.]
train_images /= 255.
test_images /= 255.
# --------------------------------------------------------------------
# ----VISUALIZATION--TENSORBOARD---------------
import datetime
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
from tensorboard import program
tb = program.TensorBoard()
tb.configure(argv=[None, '--logdir', log_dir])
url = tb.launch()
tb_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
# -----------------
checkpoint_path = "training_1/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# Callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
save_weights_only=True,
verbose=1)
# --TRAINING----
epoch_no, batch_size = 5, 100
train_model = ConvAE.fit(train_images,
train_images,
epochs=epoch_no,
def launch_tb(dir_path: str):
tb = program.TensorBoard()
tb.configure(argv=[None, '--logdir', dir_path])
url = tb.launch()
sys.stdout.write('TensorBoard at %s \n' % url)
input('press enter to quit TensorBoard')
def testConfigure(self):
tb = program.TensorBoard(fake_asset_provider,
plugins=[core_plugin.CorePluginLoader])
tb.configure(logdir="foo")
self.assertEqual(tb.flags.logdir, "foo")
def testConfigure_unknownFlag(self):
tb = program.TensorBoard(fake_asset_provider,
plugins=[core_plugin.CorePlugin])
with self.assertRaisesRegex(ValueError, "Unknown TensorBoard flag"):
tb.configure(foo="bar")
def testPlugins_pluginLoader(self):
loader = core_plugin.CorePluginLoader()
tb = program.TensorBoard(fake_asset_provider, plugins=[loader])
self.assertIs(tb.plugin_loaders[0], loader)
def testPlugins_invalidType(self):
plugin_instance = core_plugin.CorePlugin(base_plugin.TBContext())
with self.assertRaisesRegex(TypeError, "CorePlugin"):
tb = program.TensorBoard(fake_asset_provider,
plugins=[plugin_instance])
def testPlugins_pluginClass(self):
tb = program.TensorBoard(fake_asset_provider,
plugins=[core_plugin.CorePlugin])
self.assertIsInstance(tb.plugin_loaders[0], base_plugin.BasicLoader)
self.assertIs(tb.plugin_loaders[0].plugin_class,
core_plugin.CorePlugin)
def testPlugins_pluginLoaderClass(self):
tb = program.TensorBoard(fake_asset_provider,
plugins=[core_plugin.CorePluginLoader])
self.assertIsInstance(tb.plugin_loaders[0],
core_plugin.CorePluginLoader)
def start_tensorboard(logdir):
tb = tb_program.TensorBoard()
port = int(os.getenv(TB_PORT_ENV_VAR, 6006))
tb.configure(logdir=logdir, port=port)
tb.launch()
logging.info("Starting TensorBoard with --logdir=%s" % logdir)
def launch_tb(output_dir, port):
from tensorboard import program
tb = program.TensorBoard()
tb.configure(
argv=[None, '--bind_all', '--logdir', output_dir, '--port', port])
return tb.launch()
def pretrain(self):
"""
The pretraining function. This starts the pretraining using parameters given in the flags
:return: None
"""
cuda = True if torch.cuda.is_available() else False
if cuda:
self.model.cuda()
# Construct optimizer after the model moved to GPU
self.optm = torch.optim.Adam(self.model.parameters(),
lr=0.01,
weight_decay=1e-3)
self.lr_scheduler = self.make_lr_scheduler()
# Start a tensorboard session for logging loss and training images
tb = program.TensorBoard()
tb.configure(argv=[None, '--logdir', self.ckpt_dir])
url = tb.launch()
print("Starting pre-training process")
pre_train_epoch = 300
for epoch in range(pre_train_epoch):
# print("This is pretrainin Epoch {}".format(epoch))
# Set to Training Mode
train_loss = []
train_loss_eval_mode_list = []
sim_loss_list = []
self.model.train()
for j, (geometry, params_truth) in enumerate(self.train_loader):
# if j == 0 and epoch == 0:
# print(geometry)
if cuda:
geometry = geometry.cuda() # Put data onto GPU
params_truth = params_truth.cuda() # Put data onto GPU
self.optm.zero_grad() # Zero the gradient first
logit, w0, wp, g = self.model(geometry) # Get the output
# print("label size:", params_truth.size())
# print("logit size:", params.size())
pretrain_loss = self.make_MSE_loss(
w0, params_truth[:, :4]) # Get the loss tensor
pretrain_loss += self.make_MSE_loss(
wp, params_truth[:, 4:8]) # Get the loss tensor
pretrain_loss += self.make_MSE_loss(g, params_truth[:, 8:12] *
10) # Get the loss tensor
sim_loss = self.make_MSE_loss(logit, params_truth[:, 12:])
pretrain_loss.backward() # Calculate the backward gradients
# torch.nn.utils.clip_grad_value_(self.model.parameters(), 10)
train_loss.append(np.copy(
pretrain_loss.cpu().data.numpy())) # Aggregate the loss
sim_loss_list.append(np.copy(sim_loss.cpu().data.numpy()))
if epoch < pre_train_epoch - 1:
self.optm.step() # Move one step the optimizer
# Calculate the avg loss of training
train_avg_loss = np.mean(train_loss)
sim_loss = np.mean(sim_loss_list)
self.running_loss.append(sim_loss)
if epoch % 20 == 0: # Evaluate every 20 steps
# Record the training loss to the tensorboard
# train_avg_loss = train_loss.data.numpy() / (j+1)
self.log.add_scalar('Pretrain Loss', train_avg_loss, epoch)
self.log.add_scalar('Simulation Loss', sim_loss, epoch)
for j in range(self.flags.num_plot_compare):
f = compare_Lor_params(
w0=w0[j, :].cpu().data.numpy(),
wp=wp[j, :].cpu().data.numpy(),
g=g[j, :].cpu().data.numpy(),
truth=params_truth[j, :12].cpu().data.numpy())
self.log.add_figure(
tag='Test ' + str(j) +
') e2 Lorentz Parameter Prediction'.format(1),
figure=f,
global_step=epoch)
# Pretraining files contain both Lorentz parameters and simulated model spectra
pretrain_sim_prediction = params_truth[:, 12:]
pretrain_model_prediction = logit
#pretrain_model_prediction = Lorentz_layer(w0,wp,g/10)
for j in range(self.flags.num_plot_compare):
f = compare_spectra(Ypred=pretrain_model_prediction[
j, :].cpu().data.numpy(),
Ytruth=pretrain_sim_prediction[
j, :].cpu().data.numpy())
self.log.add_figure(tag='Test ' + str(j) +
') e2 Model Prediction'.format(1),
figure=f,
global_step=epoch)
print(
"This is Epoch %d, pretrain loss %.5f,, and sim loss is %.5f"
% (epoch, train_avg_loss, sim_loss))
# Model improving, save the model
if train_avg_loss < self.best_pretrain_loss:
self.best_pretrain_loss = train_avg_loss
self.save()
print("Saving the model...")
if self.best_pretrain_loss < self.flags.stop_threshold:
print("Pretraining finished EARLIER at epoch %d, reaching loss of %.5f" % \
(epoch, self.best_pretrain_loss))
return None
# Learning rate decay upon plateau
self.lr_scheduler.step(train_avg_loss)
# Save pretrained model at end
if epoch == pre_train_epoch - 1:
# weights = self.model.linears[-1].weight.cpu().data.numpy()
# # print(weights.shape)
# np.savetxt('Pretrain_Lorentz_Weights.csv', weights, fmt='%.3f', delimiter=',')
torch.save(
self.model,
os.path.join(self.ckpt_dir, 'best_pretrained_model.pt'))
# self.record_weight(name='Pretraining', batch=0, epoch=999)
self.log.close()
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
from tensorboard import default
from tensorboard import program
import tensorflow as tf
from explainer_plugin import explainer_plugin
if __name__ == '__main__':
plugins = default.get_plugins() + [explainer_plugin.ExplainerPlugin]
assets = os.path.join(tf.resource_loader.get_data_files_path(),
'assets.zip')
tensorboard = program.TensorBoard(plugins, lambda: open(assets, 'rb'))
tensorboard.configure(sys.argv)
sys.exit(tensorboard.main())
def run(self):
tb = program.TensorBoard(default.get_plugins()) #,
#default.get_assets_zip_provider())
tb.configure(argv=[None, '--logdir', self.dir_path])
url = tb.launch()
print('TensorBoard at %s \n' % url)
