def __init__(self,
loss,
optimizer,
metric=None,
batch_size=64,
ema=0.,
trainable_vars=None,
shuffle=True,
step_tensor=None,
validation_monitors=None,
validation_batch_size=None,
name=None,
graph=None):
self.graph = tf.get_default_graph()
if graph:
self.graph = graph
self.name = name
self.scope_name = name
# Ops
self.loss = loss
self.optimizer = optimizer
self.metric = metric
self.metric_summ_name = ""
if metric is not None:
self.metric_summ_name = metric.name.split('/')[0]
if isinstance(validation_monitors, tf.Tensor):
validation_monitors = [validation_monitors]
self.validation_monitors = validation_monitors or []
self.grad = None
self.apply_grad = None
self.summ_op = None
self.val_summary_op = None
self.train_vars = trainable_vars
self.shuffle = shuffle
self.batch_size = batch_size
self.validation_batch_size = validation_batch_size or batch_size
self.n_batches = 0
self.ema = ema
self.feed_dict = None
self.val_feed_dict = None
self.loss_value = None
self.val_loss = None
self.acc_value = None
self.val_acc = None
if step_tensor is None:
with self.graph.as_default():
self.training_steps = tf.Variable(0.,
name="Training_step",
trainable=False)
else:
self.training_steps = step_tensor
# Building
if not isinstance(self.loss, tf.Tensor):
raise ValueError("Unknown Loss type")
if not isinstance(self.optimizer, tf_optimizer.Optimizer):
raise ValueError("Unknown Optimizer")
if self.train_vars is None:
self.train_vars = tf.trainable_variables()
else:
self.train_var = to_list(self.train_vars)
self.train = None
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
import keras
from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
from tensorflow.python.keras.backend import set_session
import numpy as np
import os
from std_msgs.msg import Empty
from geometry_msgs.msg import Twist
from tensorflow.keras.models import load_model
sess = tf.Session()
graph = tf.get_default_graph()
# IMPORTANT: models have to be loaded AFTER SETTING THE SESSION for keras!
# Otherwise, their weights will be unavailable in the threads after the session there has been set
set_session(sess)
model = load_model('third_save.h5')
# Instantiate CvBridge
bridge = CvBridge()
def image_callback(msg):
print("Received an image!")
cv2_img = bridge.imgmsg_to_cv2(msg, "bgr8")
# try:
def train():
exception_box = ExceptionBox()
# Create training and validation datasets
train_set = create_dataset(FLAGS.train_files.split(','),
batch_size=FLAGS.train_batch_size,
epochs=FLAGS.epochs,
augmentations=Config.augmentations,
cache_path=FLAGS.feature_cache,
train_phase=True,
exception_box=exception_box,
process_ahead=len(Config.available_devices) * FLAGS.train_batch_size * 2,
buffering=FLAGS.read_buffer)
iterator = tfv1.data.Iterator.from_structure(tfv1.data.get_output_types(train_set),
tfv1.data.get_output_shapes(train_set),
output_classes=tfv1.data.get_output_classes(train_set))
# Make initialization ops for switching between the two sets
train_init_op = iterator.make_initializer(train_set)
if FLAGS.dev_files:
dev_sources = FLAGS.dev_files.split(',')
dev_sets = [create_dataset([source],
batch_size=FLAGS.dev_batch_size,
train_phase=False,
exception_box=exception_box,
process_ahead=len(Config.available_devices) * FLAGS.dev_batch_size * 2,
buffering=FLAGS.read_buffer) for source in dev_sources]
dev_init_ops = [iterator.make_initializer(dev_set) for dev_set in dev_sets]
if FLAGS.metrics_files:
metrics_sources = FLAGS.metrics_files.split(',')
metrics_sets = [create_dataset([source],
batch_size=FLAGS.dev_batch_size,
train_phase=False,
exception_box=exception_box,
process_ahead=len(Config.available_devices) * FLAGS.dev_batch_size * 2,
buffering=FLAGS.read_buffer) for source in metrics_sources]
metrics_init_ops = [iterator.make_initializer(metrics_set) for metrics_set in metrics_sets]
# Dropout
dropout_rates = [tfv1.placeholder(tf.float32, name='dropout_{}'.format(i)) for i in range(6)]
dropout_feed_dict = {
dropout_rates[0]: FLAGS.dropout_rate,
dropout_rates[1]: FLAGS.dropout_rate2,
dropout_rates[2]: FLAGS.dropout_rate3,
dropout_rates[3]: FLAGS.dropout_rate4,
dropout_rates[4]: FLAGS.dropout_rate5,
dropout_rates[5]: FLAGS.dropout_rate6,
}
no_dropout_feed_dict = {
rate: 0. for rate in dropout_rates
}
# Building the graph
learning_rate_var = tfv1.get_variable('learning_rate', initializer=FLAGS.learning_rate, trainable=False)
reduce_learning_rate_op = learning_rate_var.assign(tf.multiply(learning_rate_var, FLAGS.plateau_reduction))
optimizer = create_optimizer(learning_rate_var)
# Enable mixed precision training
if FLAGS.automatic_mixed_precision:
log_info('Enabling automatic mixed precision training.')
optimizer = tfv1.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
gradients, loss, non_finite_files = get_tower_results(iterator, optimizer, dropout_rates)
# Average tower gradients across GPUs
avg_tower_gradients = average_gradients(gradients)
log_grads_and_vars(avg_tower_gradients)
# global_step is automagically incremented by the optimizer
global_step = tfv1.train.get_or_create_global_step()
apply_gradient_op = optimizer.apply_gradients(avg_tower_gradients, global_step=global_step)
# Summaries
step_summaries_op = tfv1.summary.merge_all('step_summaries')
step_summary_writers = {
'train': tfv1.summary.FileWriter(os.path.join(FLAGS.summary_dir, 'train'), max_queue=120),
'dev': tfv1.summary.FileWriter(os.path.join(FLAGS.summary_dir, 'dev'), max_queue=120),
'metrics': tfv1.summary.FileWriter(os.path.join(FLAGS.summary_dir, 'metrics'), max_queue=120),
}
human_readable_set_names = {
'train': 'Training',
'dev': 'Validation',
'metrics': 'Metrics',
}
# Checkpointing
checkpoint_saver = tfv1.train.Saver(max_to_keep=FLAGS.max_to_keep)
checkpoint_path = os.path.join(FLAGS.save_checkpoint_dir, 'train')
best_dev_saver = tfv1.train.Saver(max_to_keep=1)
best_dev_path = os.path.join(FLAGS.save_checkpoint_dir, 'best_dev')
# Save flags next to checkpoints
os.makedirs(FLAGS.save_checkpoint_dir, exist_ok=True)
flags_file = os.path.join(FLAGS.save_checkpoint_dir, 'flags.txt')
with open(flags_file, 'w') as fout:
fout.write(FLAGS.flags_into_string())
with tfv1.Session(config=Config.session_config) as session:
log_debug('Session opened.')
# Prevent further graph changes
tfv1.get_default_graph().finalize()
# Load checkpoint or initialize variables
load_or_init_graph_for_training(session)
def run_set(set_name, epoch, init_op, dataset=None):
is_train = set_name == 'train'
train_op = apply_gradient_op if is_train else []
feed_dict = dropout_feed_dict if is_train else no_dropout_feed_dict
total_loss = 0.0
step_count = 0
step_summary_writer = step_summary_writers.get(set_name)
checkpoint_time = time.time()
if is_train and FLAGS.cache_for_epochs > 0 and FLAGS.feature_cache:
feature_cache_index = FLAGS.feature_cache + '.index'
if epoch % FLAGS.cache_for_epochs == 0 and os.path.isfile(feature_cache_index):
log_info('Invalidating feature cache')
os.remove(feature_cache_index) # this will let TF also overwrite the related cache data files
# Setup progress bar
class LossWidget(progressbar.widgets.FormatLabel):
def __init__(self):
progressbar.widgets.FormatLabel.__init__(self, format='Loss: %(mean_loss)f')
def __call__(self, progress, data, **kwargs):
data['mean_loss'] = total_loss / step_count if step_count else 0.0
return progressbar.widgets.FormatLabel.__call__(self, progress, data, **kwargs)
prefix = 'Epoch {} | {:>10}'.format(epoch, human_readable_set_names[set_name])
widgets = [' | ', progressbar.widgets.Timer(),
' | Steps: ', progressbar.widgets.Counter(),
' | ', LossWidget()]
suffix = ' | Dataset: {}'.format(dataset) if dataset else None
pbar = create_progressbar(prefix=prefix, widgets=widgets, suffix=suffix).start()
# Initialize iterator to the appropriate dataset
session.run(init_op)
# Batch loop
while True:
try:
_, current_step, batch_loss, problem_files, step_summary = \
session.run([train_op, global_step, loss, non_finite_files, step_summaries_op],
feed_dict=feed_dict)
exception_box.raise_if_set()
except tf.errors.OutOfRangeError:
exception_box.raise_if_set()
break
if problem_files.size > 0:
problem_files = [f.decode('utf8') for f in problem_files[..., 0]]
log_error('The following files caused an infinite (or NaN) '
'loss: {}'.format(','.join(problem_files)))
total_loss += batch_loss
step_count += 1
pbar.update(step_count)
step_summary_writer.add_summary(step_summary, current_step)
if is_train and FLAGS.checkpoint_secs > 0 and time.time() - checkpoint_time > FLAGS.checkpoint_secs:
checkpoint_saver.save(session, checkpoint_path, global_step=current_step)
checkpoint_time = time.time()
pbar.finish()
mean_loss = total_loss / step_count if step_count > 0 else 0.0
return mean_loss, step_count
log_info('STARTING Optimization')
train_start_time = datetime.utcnow()
best_dev_loss = float('inf')
dev_losses = []
epochs_without_improvement = 0
try:
for epoch in range(FLAGS.epochs):
# Training
log_progress('Training epoch %d...' % epoch)
train_loss, _ = run_set('train', epoch, train_init_op)
log_progress('Finished training epoch %d - loss: %f' % (epoch, train_loss))
checkpoint_saver.save(session, checkpoint_path, global_step=global_step)
if FLAGS.dev_files:
# Validation
dev_loss = 0.0
total_steps = 0
for source, init_op in zip(dev_sources, dev_init_ops):
log_progress('Validating epoch %d on %s...' % (epoch, source))
set_loss, steps = run_set('dev', epoch, init_op, dataset=source)
dev_loss += set_loss * steps
total_steps += steps
log_progress('Finished validating epoch %d on %s - loss: %f' % (epoch, source, set_loss))
dev_loss = dev_loss / total_steps
dev_losses.append(dev_loss)
# Count epochs without an improvement for early stopping and reduction of learning rate on a plateau
# the improvement has to be greater than FLAGS.es_min_delta
if dev_loss > best_dev_loss - FLAGS.es_min_delta:
epochs_without_improvement += 1
else:
epochs_without_improvement = 0
# Save new best model
if dev_loss < best_dev_loss:
best_dev_loss = dev_loss
save_path = best_dev_saver.save(session, best_dev_path, global_step=global_step, latest_filename='best_dev_checkpoint')
log_info("Saved new best validating model with loss %f to: %s" % (best_dev_loss, save_path))
# Early stopping
if FLAGS.early_stop and epochs_without_improvement == FLAGS.es_epochs:
log_info('Early stop triggered as the loss did not improve the last {} epochs'.format(
epochs_without_improvement))
break
# Reduce learning rate on plateau
if (FLAGS.reduce_lr_on_plateau and
epochs_without_improvement % FLAGS.plateau_epochs == 0 and epochs_without_improvement > 0):
# If the learning rate was reduced and there is still no improvement
# wait FLAGS.plateau_epochs before the learning rate is reduced again
session.run(reduce_learning_rate_op)
current_learning_rate = learning_rate_var.eval()
log_info('Encountered a plateau, reducing learning rate to {}'.format(
current_learning_rate))
if FLAGS.metrics_files:
# Read only metrics, not affecting best validation loss tracking
for source, init_op in zip(metrics_sources, metrics_init_ops):
log_progress('Metrics for epoch %d on %s...' % (epoch, source))
set_loss, _ = run_set('metrics', epoch, init_op, dataset=source)
log_progress('Metrics for epoch %d on %s - loss: %f' % (epoch, source, set_loss))
print('-' * 80)
except KeyboardInterrupt:
pass
log_info('FINISHED optimization in {}'.format(datetime.utcnow() - train_start_time))
log_debug('Session closed.')
def train():
# Create training and validation datasets
train_set = create_dataset(FLAGS.train_files.split(','),
batch_size=FLAGS.train_batch_size,
cache_path=FLAGS.feature_cache)
iterator = tfv1.data.Iterator.from_structure(
tfv1.data.get_output_types(train_set),
tfv1.data.get_output_shapes(train_set),
output_classes=tfv1.data.get_output_classes(train_set))
# Make initialization ops for switching between the two sets
train_init_op = iterator.make_initializer(train_set)
if FLAGS.dev_files:
dev_csvs = FLAGS.dev_files.split(',')
dev_sets = [
create_dataset([csv], batch_size=FLAGS.dev_batch_size)
for csv in dev_csvs
]
dev_init_ops = [
iterator.make_initializer(dev_set) for dev_set in dev_sets
]
# Dropout
dropout_rates = [
tfv1.placeholder(tf.float32, name='dropout_{}'.format(i))
for i in range(6)
]
dropout_feed_dict = {
dropout_rates[0]: FLAGS.dropout_rate,
dropout_rates[1]: FLAGS.dropout_rate2,
dropout_rates[2]: FLAGS.dropout_rate3,
dropout_rates[3]: FLAGS.dropout_rate4,
dropout_rates[4]: FLAGS.dropout_rate5,
dropout_rates[5]: FLAGS.dropout_rate6,
}
no_dropout_feed_dict = {rate: 0. for rate in dropout_rates}
# Building the graph
optimizer = create_optimizer()
gradients, loss = get_tower_results(iterator, optimizer, dropout_rates)
# Average tower gradients across GPUs
avg_tower_gradients = average_gradients(gradients)
log_grads_and_vars(avg_tower_gradients)
# global_step is automagically incremented by the optimizer
global_step = tfv1.train.get_or_create_global_step()
apply_gradient_op = optimizer.apply_gradients(avg_tower_gradients,
global_step=global_step)
# Summaries
step_summaries_op = tfv1.summary.merge_all('step_summaries')
step_summary_writers = {
'train':
tfv1.summary.FileWriter(os.path.join(FLAGS.summary_dir, 'train'),
max_queue=120),
'dev':
tfv1.summary.FileWriter(os.path.join(FLAGS.summary_dir, 'dev'),
max_queue=120)
}
# Checkpointing
checkpoint_saver = tfv1.train.Saver(max_to_keep=FLAGS.max_to_keep)
checkpoint_path = os.path.join(FLAGS.checkpoint_dir, 'train')
checkpoint_filename = 'checkpoint'
best_dev_saver = tfv1.train.Saver(max_to_keep=1)
best_dev_path = os.path.join(FLAGS.checkpoint_dir, 'best_dev')
best_dev_filename = 'best_dev_checkpoint'
initializer = tfv1.global_variables_initializer()
with tfv1.Session() as session:
log_debug('Session opened.')
tfv1.get_default_graph().finalize()
# Loading or initializing
loaded = False
if FLAGS.load in ['auto', 'last']:
loaded = try_loading(session, checkpoint_saver,
checkpoint_filename, 'most recent')
if not loaded and FLAGS.load in ['auto', 'best']:
loaded = try_loading(session, best_dev_saver, best_dev_filename,
'best validation')
if not loaded:
if FLAGS.load in ['auto', 'init']:
log_info('Initializing variables...')
session.run(initializer)
else:
log_error(
'Unable to load %s model from specified checkpoint dir'
' - consider using load option "auto" or "init".' %
FLAGS.load)
sys.exit(1)
def run_set(set_name, epoch, init_op, dataset=None):
is_train = set_name == 'train'
train_op = apply_gradient_op if is_train else []
feed_dict = dropout_feed_dict if is_train else no_dropout_feed_dict
total_loss = 0.0
step_count = 0
step_summary_writer = step_summary_writers.get(set_name)
checkpoint_time = time.time()
# Setup progress bar
class LossWidget(progressbar.widgets.FormatLabel):
def __init__(self):
progressbar.widgets.FormatLabel.__init__(
self, format='Loss: %(mean_loss)f')
def __call__(self, progress, data, **kwargs):
data[
'mean_loss'] = total_loss / step_count if step_count else 0.0
return progressbar.widgets.FormatLabel.__call__(
self, progress, data, **kwargs)
prefix = 'Epoch {} | {:>10}'.format(
epoch, 'Training' if is_train else 'Validation')
widgets = [
' | ',
progressbar.widgets.Timer(), ' | Steps: ',
progressbar.widgets.Counter(), ' | ',
LossWidget()
]
suffix = ' | Dataset: {}'.format(dataset) if dataset else None
pbar = create_progressbar(prefix=prefix,
widgets=widgets,
suffix=suffix).start()
# Initialize iterator to the appropriate dataset
session.run(init_op)
# Batch loop
while True:
try:
_, current_step, batch_loss, step_summary = \
session.run([train_op, global_step, loss, step_summaries_op],
feed_dict=feed_dict)
except tf.errors.OutOfRangeError:
break
total_loss += batch_loss
step_count += 1
pbar.update(step_count)
step_summary_writer.add_summary(step_summary, current_step)
if is_train and FLAGS.checkpoint_secs > 0 and time.time(
) - checkpoint_time > FLAGS.checkpoint_secs:
checkpoint_saver.save(session,
checkpoint_path,
global_step=current_step)
checkpoint_time = time.time()
pbar.finish()
mean_loss = total_loss / step_count if step_count > 0 else 0.0
return mean_loss, step_count
log_info('STARTING Optimization')
train_start_time = datetime.utcnow()
best_dev_loss = float('inf')
dev_losses = []
try:
for epoch in range(FLAGS.epochs):
# Training
log_progress('Training epoch %d...' % epoch)
train_loss, _ = run_set('train', epoch, train_init_op)
log_progress('Finished training epoch %d - loss: %f' %
(epoch, train_loss))
checkpoint_saver.save(session,
checkpoint_path,
global_step=global_step)
if FLAGS.dev_files:
# Validation
dev_loss = 0.0
total_steps = 0
for csv, init_op in zip(dev_csvs, dev_init_ops):
log_progress('Validating epoch %d on %s...' %
(epoch, csv))
set_loss, steps = run_set('dev',
epoch,
init_op,
dataset=csv)
dev_loss += set_loss * steps
total_steps += steps
log_progress(
'Finished validating epoch %d on %s - loss: %f' %
(epoch, csv, set_loss))
dev_loss = dev_loss / total_steps
dev_losses.append(dev_loss)
if dev_loss < best_dev_loss:
best_dev_loss = dev_loss
save_path = best_dev_saver.save(
session,
best_dev_path,
global_step=global_step,
latest_filename=best_dev_filename)
log_info(
"Saved new best validating model with loss %f to: %s"
% (best_dev_loss, save_path))
# Early stopping
if FLAGS.early_stop and len(dev_losses) >= FLAGS.es_steps:
mean_loss = np.mean(dev_losses[-FLAGS.es_steps:-1])
std_loss = np.std(dev_losses[-FLAGS.es_steps:-1])
dev_losses = dev_losses[-FLAGS.es_steps:]
log_debug(
'Checking for early stopping (last %d steps) validation loss: '
'%f, with standard deviation: %f and mean: %f' %
(FLAGS.es_steps, dev_losses[-1], std_loss,
mean_loss))
if dev_losses[-1] > np.max(dev_losses[:-1]) or \
(abs(dev_losses[-1] - mean_loss) < FLAGS.es_mean_th and std_loss < FLAGS.es_std_th):
log_info(
'Early stop triggered as (for last %d steps) validation loss:'
' %f with standard deviation: %f and mean: %f'
% (FLAGS.es_steps, dev_losses[-1], std_loss,
mean_loss))
break
except KeyboardInterrupt:
pass
log_info('FINISHED optimization in {}'.format(datetime.utcnow() -
train_start_time))
log_debug('Session closed.')
def _export_inference_graph(input_type,
detection_model,
use_moving_averages,
trained_checkpoint_prefix,
output_directory,
additional_output_tensor_names=None,
input_shape=None,
output_collection_name='inference_op',
graph_hook_fn=None,
write_inference_graph=False,
temp_checkpoint_prefix='',
use_side_inputs=False,
side_input_shapes=None,
side_input_names=None,
side_input_types=None):
"""Export helper."""
tf.gfile.MakeDirs(output_directory)
frozen_graph_path = os.path.join(output_directory,
'frozen_inference_graph.pb')
saved_model_path = os.path.join(output_directory, 'saved_model')
model_path = os.path.join(output_directory, 'model.ckpt')
outputs, placeholder_tensor_dict = build_detection_graph(
input_type=input_type,
detection_model=detection_model,
input_shape=input_shape,
output_collection_name=output_collection_name,
graph_hook_fn=graph_hook_fn,
use_side_inputs=use_side_inputs,
side_input_shapes=side_input_shapes,
side_input_names=side_input_names,
side_input_types=side_input_types)
profile_inference_graph(tf.get_default_graph())
saver_kwargs = {}
if use_moving_averages:
if not temp_checkpoint_prefix:
# This check is to be compatible with both version of SaverDef.
if os.path.isfile(trained_checkpoint_prefix):
saver_kwargs['write_version'] = saver_pb2.SaverDef.V1
temp_checkpoint_prefix = tempfile.NamedTemporaryFile().name
else:
temp_checkpoint_prefix = tempfile.mkdtemp()
replace_variable_values_with_moving_averages(
tf.get_default_graph(), trained_checkpoint_prefix,
temp_checkpoint_prefix)
checkpoint_to_use = temp_checkpoint_prefix
else:
checkpoint_to_use = trained_checkpoint_prefix
saver = tf.train.Saver(**saver_kwargs)
input_saver_def = saver.as_saver_def()
write_graph_and_checkpoint(
inference_graph_def=tf.get_default_graph().as_graph_def(),
model_path=model_path,
input_saver_def=input_saver_def,
trained_checkpoint_prefix=checkpoint_to_use)
if write_inference_graph:
inference_graph_def = tf.get_default_graph().as_graph_def()
inference_graph_path = os.path.join(output_directory,
'inference_graph.pbtxt')
for node in inference_graph_def.node:
node.device = ''
with tf.gfile.GFile(inference_graph_path, 'wb') as f:
f.write(str(inference_graph_def))
if additional_output_tensor_names is not None:
output_node_names = ','.join(list(outputs.keys())+(
additional_output_tensor_names))
else:
output_node_names = ','.join(outputs.keys())
frozen_graph_def = freeze_graph.freeze_graph_with_def_protos(
input_graph_def=tf.get_default_graph().as_graph_def(),
input_saver_def=input_saver_def,
input_checkpoint=checkpoint_to_use,
output_node_names=output_node_names,
restore_op_name='save/restore_all',
filename_tensor_name='save/Const:0',
output_graph=frozen_graph_path,
clear_devices=True,
initializer_nodes='')
write_saved_model(saved_model_path, frozen_graph_def,
placeholder_tensor_dict, outputs)
def __init__(self):
self.graph = tf.get_default_graph()
self.session = tf.Session()
self.model = None
def singlePatchAttack(args, img, clusters, session, positive_embedding,
adv_filename, grad_input, positive, loss):
"""
Single Patch
:param args: command arguments
:param img: the image
:param clusters: gradient clusters (label, weight, center, cluster_mask)
:param session: session
:param positive_embedding: positive embedding
:param adv_filename: the adversarial image name
:param grad_input: gradient to input image (tensor)
:param positive: positive image (tf.placeholder)
:param loss: loss (tensor)
:return:
"""
img = img.transpose(2, 0, 1)
n_cluster = min(len(clusters), args.max_cluster)
if n_cluster == 0:
print('%s: no cluster ' % adv_filename)
return
image_input = tf.get_default_graph().get_tensor_by_name(
'image_input:0') # (?, 3, 112, 112)
keep_prob = tf.get_default_graph().get_tensor_by_name('keep_prob:0')
is_train = tf.get_default_graph().get_tensor_by_name('training_mode:0')
embedding = tf.get_default_graph().get_tensor_by_name(
'embedding:0') # (?, 512)
moment_val = 0.9
step_val = 1.0 / args.gradient_step
c, w, h = img.shape
grad = np.zeros_like(img)
img_grad = np.stack([grad for i in range(n_cluster)])
cluster_imgs = np.stack([img for i in range(n_cluster)])
r = 1
success = False
success_cluster_no = []
success_loss = []
while not success:
print("radius = %d" % r)
#create patches for all clusters
patch_list = []
for _, _, center, _ in clusters:
patch_mask = get_patch(img, center, radius=r)
patch_list.append(patch_mask)
if len(patch_list) >= n_cluster:
break
patches = np.stack(patch_list)
moments = np.zeros([n_cluster, c, w, h], dtype=np.float32)
for step in range(args.gradient_step):
#try each cluster
grad_mask = img_grad * patches
cluster_imgs = cluster_imgs + grad_mask
fdict = {
image_input: cluster_imgs,
keep_prob: 1.0,
is_train: False,
positive: positive_embedding
}
grads_np, embedding_np, loss_np = session.run(
[grad_input, embedding, loss], feed_dict=fdict)
moments = moments * moment_val + grads_np * (1. - moment_val)
img_grad = step_val * np.sign(moments)
loss_np = -1 * loss_np
print('step %d: %s ' % (step, loss_np))
# test whether success
for i in range(n_cluster):
if loss_np[i] < 0.2:
success = True
success_cluster_no.append(i)
success_loss.append(loss_np[i])
success_img = cluster_imgs[i, :, :, :].transpose(1, 2, 0)
output_img = np.round(
(success_img + 1) * 255 / 2).astype(np.uint8)
filename = adv_filename + '_cluster_%d.jpg' % (i)
io.imsave(filename, output_img)
if success:
break
r = r + 1
write_csv(success_cluster_no, success_loss,
adv_filename + '_r%d.csv' % (r - 1))
def export():
r'''
Restores the trained variables into a simpler graph that will be exported for serving.
'''
log_info('Exporting the model...')
from tensorflow.python.framework.ops import Tensor, Operation
inputs, outputs, _ = create_inference_graph(
batch_size=FLAGS.export_batch_size,
n_steps=FLAGS.n_steps,
tflite=FLAGS.export_tflite)
graph_version = int(file_relative_read('GRAPH_VERSION').strip())
assert graph_version > 0
outputs['metadata_version'] = tf.constant([graph_version],
name='metadata_version')
outputs['metadata_sample_rate'] = tf.constant([FLAGS.audio_sample_rate],
name='metadata_sample_rate')
outputs['metadata_feature_win_len'] = tf.constant(
[FLAGS.feature_win_len], name='metadata_feature_win_len')
outputs['metadata_feature_win_step'] = tf.constant(
[FLAGS.feature_win_step], name='metadata_feature_win_step')
outputs['metadata_beam_width'] = tf.constant([FLAGS.export_beam_width],
name='metadata_beam_width')
outputs['metadata_alphabet'] = tf.constant([Config.alphabet.serialize()],
name='metadata_alphabet')
if FLAGS.export_language:
outputs['metadata_language'] = tf.constant(
[FLAGS.export_language.encode('utf-8')], name='metadata_language')
# Prevent further graph changes
tfv1.get_default_graph().finalize()
output_names_tensors = [
tensor.op.name for tensor in outputs.values()
if isinstance(tensor, Tensor)
]
output_names_ops = [
op.name for op in outputs.values() if isinstance(op, Operation)
]
output_names = output_names_tensors + output_names_ops
with tf.Session() as session:
# Restore variables from checkpoint
if FLAGS.load == 'auto':
method_order = ['best', 'last']
else:
method_order = [FLAGS.load]
load_or_init_graph(session, method_order)
output_filename = FLAGS.export_name + '.pb'
if FLAGS.remove_export:
if os.path.isdir(FLAGS.export_dir):
log_info('Removing old export')
shutil.rmtree(FLAGS.export_dir)
output_graph_path = os.path.join(FLAGS.export_dir, output_filename)
if not os.path.isdir(FLAGS.export_dir):
os.makedirs(FLAGS.export_dir)
frozen_graph = tfv1.graph_util.convert_variables_to_constants(
sess=session,
input_graph_def=tfv1.get_default_graph().as_graph_def(),
output_node_names=output_names)
frozen_graph = tfv1.graph_util.extract_sub_graph(
graph_def=frozen_graph, dest_nodes=output_names)
if not FLAGS.export_tflite:
with open(output_graph_path, 'wb') as fout:
fout.write(frozen_graph.SerializeToString())
else:
output_tflite_path = os.path.join(
FLAGS.export_dir, output_filename.replace('.pb', '.tflite'))
converter = tf.lite.TFLiteConverter(
frozen_graph,
input_tensors=inputs.values(),
output_tensors=outputs.values())
converter.optimizations = [tf.lite.Optimize.DEFAULT]
# AudioSpectrogram and Mfcc ops are custom but have built-in kernels in TFLite
converter.allow_custom_ops = True
tflite_model = converter.convert()
with open(output_tflite_path, 'wb') as fout:
fout.write(tflite_model)
log_info('Models exported at %s' % (FLAGS.export_dir))
def main(args):
""" Main
Given a list of images, save out facial encoding data files and copy
images into folders of face clusters.
"""
from os.path import join, basename, exists
from os import makedirs
import numpy as np
import shutil
import sys
if not exists(args.output):
makedirs(args.output)
with tf.Graph().as_default():
with tf.Session() as sess:
image_paths = get_onedir(args.input)
#image_list, label_list = facenet.get_image_paths_and_labels(train_set)
meta_file, ckpt_file = facenet.get_model_filenames(
os.path.expanduser(args.model_dir))
print('Metagraph file: %s' % meta_file)
print('Checkpoint file: %s' % ckpt_file)
load_model(args.model_dir, meta_file, ckpt_file)
# Get input and output tensors
images_placeholder = tf.get_default_graph().get_tensor_by_name(
"input:0")
embeddings = tf.get_default_graph().get_tensor_by_name(
"embeddings:0")
phase_train_placeholder = tf.get_default_graph(
).get_tensor_by_name("phase_train:0")
image_size = images_placeholder.get_shape()[1]
print("image_size:", image_size)
embedding_size = embeddings.get_shape()[1]
# Run forward pass to calculate embeddings
print('Runnning forward pass on images')
nrof_images = len(image_paths)
nrof_batches = int(math.ceil(1.0 * nrof_images / args.batch_size))
emb_array = np.zeros((nrof_images, embedding_size))
facial_encodings = compute_facial_encodings(
sess, images_placeholder, embeddings, phase_train_placeholder,
image_size, embedding_size, nrof_images, nrof_batches,
emb_array, args.batch_size, image_paths)
sorted_clusters = cluster_facial_encodings(facial_encodings)
num_cluster = len(sorted_clusters)
# Copy image files to cluster folders
for idx, cluster in enumerate(sorted_clusters):
#save all the cluster
cluster_dir = join(args.output, str(idx))
if not exists(cluster_dir):
makedirs(cluster_dir)
for path in cluster:
shutil.copy(path, join(cluster_dir, basename(path)))
def main(ms_file_name, line_freq, ouptut_file):
tf.reset_default_graph()
sess = tf.InteractiveSession()
# load vocabulary
int2word = read_vocab("models/vocabulary_semantic.txt")
# Restore weights
model = "models/semantic_model.meta"
saver = tf.train.import_meta_graph(model)
saver.restore(sess, model[:-5])
graph = tf.get_default_graph()
model_input = graph.get_tensor_by_name("model_input:0")
seq_len = graph.get_tensor_by_name("seq_lengths:0")
rnn_keep_prob = graph.get_tensor_by_name("keep_prob:0")
height_tensor = graph.get_tensor_by_name("input_height:0")
width_reduction_tensor = graph.get_tensor_by_name("width_reduction:0")
logits = tf.get_collection("logits")[0]
# Constants that are saved inside the model itself
WIDTH_REDUCTION, HEIGHT = sess.run([width_reduction_tensor, height_tensor])
decoded, _ = tf.nn.ctc_greedy_decoder(logits, seq_len)
# split the music score into lines
print(f"Process {ms_file_name}\n")
lines = split_score(ms_file_name, line_freq)
output = open(ouptut_file, "w")
# process save file
for idx, line in enumerate(lines):
# write the file to sample directory for sampling
print(f"./samples/sample{idx}.png\n")
cv2.imwrite(f"./samples/sample{idx}.png", line)
gray = cv2.cvtColor(line, cv2.COLOR_BGR2GRAY)
image = ctc_utils.resize(gray, HEIGHT)
image = ctc_utils.normalize(image)
image = np.asarray(image).reshape(1, image.shape[0], -1, 1)
seq_lengths = [image.shape[2] / WIDTH_REDUCTION]
prediction = sess.run(decoded,
feed_dict={
model_input: image,
seq_len: seq_lengths,
rnn_keep_prob: 1.0,
})
str_predictions = ctc_utils.sparse_tensor_to_strs(prediction)
for w in str_predictions[0]:
description = int2word[w]
notation, v1, v2 = parse_description(description)
if v1 != "tie":
if notation == "barline":
output.write("### ----------------\n")
elif notation == "note" or notation == "gracenote":
output.write(f'- ["{notation}", "{v1}", "{v2}"]\n')
elif notation == "rest":
output.write(f'- ["rest", "{v1}"]\n')
output.close()
def __init__(self, variable, mesh_impl):
"""Create a LaidOutVariable.
Args:
variable: a Variable (Operation)
mesh_impl: a MeshImpl
"""
self._variable = variable
self._mesh_impl = mesh_impl
shape = variable.outputs[0].shape
slice_shape = mesh_impl.slice_shape(shape)
base_name = variable.name
slices = []
slices_with_master_dtype = []
with tf.device(
variable.master_device), utils.outside_all_rewrites():
zero_tensor = tf.zeros(slice_shape, dtype=variable.slice_dtype)
# pylint: disable=protected-access
init_device_stack = tf.get_default_graph()._device_function_stack
if not mesh_impl.graph_device_function_stacks:
for pnum in xrange(mesh_impl.size):
tpu_device = mesh_impl.device_assignment.tpu_device(
replica=pnum)
with tf.device(tpu_device):
mesh_impl.graph_device_function_stacks.append(
tf.get_default_graph()._device_function_stack.copy(
))
for physical_pnum in xrange(mesh_impl.size):
slice_var_name = base_name + "_slice_%d" % physical_pnum
# Use tf.Variable instead of tf.get_variable since latter adds lots of
# useless operations to the TF graph. Use tf.get_variable only if
# in a AUTO_REUSE scope.
# Note: Repeatedly 'with tf.device():' slows down the graph
# construction. Therefore we directly use the cached device_stack here.
tf.get_default_graph()._device_function_stack = (
mesh_impl.graph_device_function_stacks[physical_pnum])
if tf.get_variable_scope().reuse == tf.AUTO_REUSE:
slice_var = tf.get_variable(
initializer=zero_tensor,
trainable=self._variable.trainable,
collections=["TPU_VAR"],
dtype=variable.slice_dtype,
name=slice_var_name)
else:
slice_var = tf.Variable(initial_value=zero_tensor,
trainable=self._variable.trainable,
collections=["TPU_VAR"],
dtype=variable.slice_dtype,
name=slice_var_name,
expected_shape=slice_shape)
slices.append(slice_var)
# Restore the initial stack
tf.get_default_graph()._device_function_stack = init_device_stack
# pylint: enable=protected-access
self._laid_out_tensor = mesh_impl.LaidOutTensor(
[tpu_variables.ReplicatedVariable(base_name, slices)])
with tf.device(
variable.master_device), utils.outside_all_rewrites():
if os.environ.get("MTF_SEQUENCE_MODE", "") == "1":
if mesh_impl.copy_master_to_slice_ops:
with tf.control_dependencies(
[mesh_impl.copy_master_to_slice_ops[-1]]):
self._copy_master_to_slices = self._gen_copy_master_to_slices_op(
variable.get_master(), shape, slices,
slice_shape)
else:
self._copy_master_to_slices = self._gen_copy_master_to_slices_op(
variable.get_master(), shape, slices, slice_shape)
mesh_impl.copy_master_to_slice_ops.append(
self._copy_master_to_slices)
else:
self._copy_master_to_slices = self._gen_copy_master_to_slices_op(
variable.get_master(), shape, slices, slice_shape)
slices_with_master_dtype = [
tf.cast(s, variable.master_dtype) for s in slices
]
slices_with_master_dtype = [
slices_with_master_dtype[mesh_impl.l2p(logical_pnum)]
for logical_pnum in range(mesh_impl.size)
]
self._copy_slices_to_master = variable.assign_to_master(
mesh_impl.combine_slices(slices_with_master_dtype,
shape,
device=variable.master_device))
minsize = 20 # minimum size of face
threshold = [0.6, 0.7, 0.7] # three steps's threshold
factor = 0.709 # scale factor
margin = 44
frame_interval = 3
batch_size = 1000
image_size = 182
input_image_size = 160
HumanNames = os.listdir(train_img)
HumanNames.sort()
print('Loading feature extraction model')
facenet.load_model(modeldir)
images_placeholder = tf.get_default_graph().get_tensor_by_name(
"input:0")
embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
phase_train_placeholder = tf.get_default_graph().get_tensor_by_name(
"phase_train:0")
embedding_size = embeddings.get_shape()[1]
classifier_filename_exp = os.path.expanduser(classifier_filename)
with open(classifier_filename_exp, 'rb') as infile:
(model, class_names) = pickle.load(infile)
# video_capture = cv2.VideoCapture("akshay_mov.mp4")
c = 0
print('Start Recognition!')
prevTime = 0
# ret, frame = video_capture.read()
def train():
do_cache_dataset = True
# pylint: disable=too-many-boolean-expressions
if (FLAGS.data_aug_features_multiplicative > 0 or
FLAGS.data_aug_features_additive > 0 or
FLAGS.augmentation_spec_dropout_keeprate < 1 or
FLAGS.augmentation_freq_and_time_masking or
FLAGS.augmentation_pitch_and_tempo_scaling or
FLAGS.augmentation_speed_up_std > 0):
do_cache_dataset = False
# Create training and validation datasets
train_set = create_dataset(FLAGS.train_files.split(','),
batch_size=FLAGS.train_batch_size,
enable_cache=FLAGS.feature_cache and do_cache_dataset,
cache_path=FLAGS.feature_cache,
train_phase=True)
iterator = tfv1.data.Iterator.from_structure(tfv1.data.get_output_types(train_set),
tfv1.data.get_output_shapes(train_set),
output_classes=tfv1.data.get_output_classes(train_set))
# Make initialization ops for switching between the two sets
train_init_op = iterator.make_initializer(train_set)
if FLAGS.dev_files:
dev_csvs = FLAGS.dev_files.split(',')
dev_sets = [create_dataset([csv], batch_size=FLAGS.dev_batch_size, train_phase=False) for csv in dev_csvs]
dev_init_ops = [iterator.make_initializer(dev_set) for dev_set in dev_sets]
# Dropout
dropout_rates = [tfv1.placeholder(tf.float32, name='dropout_{}'.format(i)) for i in range(6)]
dropout_feed_dict = {
dropout_rates[0]: FLAGS.dropout_rate,
dropout_rates[1]: FLAGS.dropout_rate2,
dropout_rates[2]: FLAGS.dropout_rate3,
dropout_rates[3]: FLAGS.dropout_rate4,
dropout_rates[4]: FLAGS.dropout_rate5,
dropout_rates[5]: FLAGS.dropout_rate6,
}
no_dropout_feed_dict = {
rate: 0. for rate in dropout_rates
}
# Building the graph
optimizer = create_optimizer()
# Enable mixed precision training
if FLAGS.automatic_mixed_precision:
log_info('Enabling automatic mixed precision training.')
optimizer = tfv1.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
gradients, loss, non_finite_files = get_tower_results(iterator, optimizer, dropout_rates)
# Average tower gradients across GPUs
avg_tower_gradients = average_gradients(gradients)
log_grads_and_vars(avg_tower_gradients)
# global_step is automagically incremented by the optimizer
global_step = tfv1.train.get_or_create_global_step()
apply_gradient_op = optimizer.apply_gradients(avg_tower_gradients, global_step=global_step)
# Summaries
step_summaries_op = tfv1.summary.merge_all('step_summaries')
step_summary_writers = {
'train': tfv1.summary.FileWriter(os.path.join(FLAGS.summary_dir, 'train'), max_queue=120),
'dev': tfv1.summary.FileWriter(os.path.join(FLAGS.summary_dir, 'dev'), max_queue=120)
}
# Checkpointing
checkpoint_saver = tfv1.train.Saver(max_to_keep=FLAGS.max_to_keep)
checkpoint_path = os.path.join(FLAGS.checkpoint_dir, 'train')
best_dev_saver = tfv1.train.Saver(max_to_keep=1)
best_dev_path = os.path.join(FLAGS.checkpoint_dir, 'best_dev')
# Save flags next to checkpoints
os.makedirs(FLAGS.checkpoint_dir, exist_ok=True)
flags_file = os.path.join(FLAGS.checkpoint_dir, 'flags.txt')
with open(flags_file, 'w') as fout:
fout.write(FLAGS.flags_into_string())
initializer = tfv1.global_variables_initializer()
with tfv1.Session(config=Config.session_config) as session:
log_debug('Session opened.')
# Loading or initializing
loaded = False
# Initialize training from a CuDNN RNN checkpoint
if FLAGS.cudnn_checkpoint:
if FLAGS.use_cudnn_rnn:
log_error('Trying to use --cudnn_checkpoint but --use_cudnn_rnn '
'was specified. The --cudnn_checkpoint flag is only '
'needed when converting a CuDNN RNN checkpoint to '
'a CPU-capable graph. If your system is capable of '
'using CuDNN RNN, you can just specify the CuDNN RNN '
'checkpoint normally with --checkpoint_dir.')
sys.exit(1)
log_info('Converting CuDNN RNN checkpoint from {}'.format(FLAGS.cudnn_checkpoint))
ckpt = tfv1.train.load_checkpoint(FLAGS.cudnn_checkpoint)
missing_variables = []
# Load compatible variables from checkpoint
for v in tfv1.global_variables():
try:
v.load(ckpt.get_tensor(v.op.name), session=session)
except tf.errors.NotFoundError:
missing_variables.append(v)
# Check that the only missing variables are the Adam moment tensors
if any('Adam' not in v.op.name for v in missing_variables):
log_error('Tried to load a CuDNN RNN checkpoint but there were '
'more missing variables than just the Adam moment '
'tensors.')
sys.exit(1)
# Initialize Adam moment tensors from scratch to allow use of CuDNN
# RNN checkpoints.
log_info('Initializing missing Adam moment tensors.')
init_op = tfv1.variables_initializer(missing_variables)
session.run(init_op)
loaded = True
tfv1.get_default_graph().finalize()
if not loaded and FLAGS.load in ['auto', 'last']:
loaded = try_loading(session, checkpoint_saver, 'checkpoint', 'most recent')
if not loaded and FLAGS.load in ['auto', 'best']:
loaded = try_loading(session, best_dev_saver, 'best_dev_checkpoint', 'best validation')
if not loaded:
if FLAGS.load in ['auto', 'init']:
log_info('Initializing variables...')
session.run(initializer)
else:
log_error('Unable to load %s model from specified checkpoint dir'
' - consider using load option "auto" or "init".' % FLAGS.load)
sys.exit(1)
def run_set(set_name, epoch, init_op, dataset=None):
is_train = set_name == 'train'
train_op = apply_gradient_op if is_train else []
feed_dict = dropout_feed_dict if is_train else no_dropout_feed_dict
total_loss = 0.0
step_count = 0
step_summary_writer = step_summary_writers.get(set_name)
checkpoint_time = time.time()
# Setup progress bar
class LossWidget(progressbar.widgets.FormatLabel):
def __init__(self):
progressbar.widgets.FormatLabel.__init__(self, format='Loss: %(mean_loss)f')
def __call__(self, progress, data, **kwargs):
data['mean_loss'] = total_loss / step_count if step_count else 0.0
return progressbar.widgets.FormatLabel.__call__(self, progress, data, **kwargs)
prefix = 'Epoch {} | {:>10}'.format(epoch, 'Training' if is_train else 'Validation')
widgets = [' | ', progressbar.widgets.Timer(),
' | Steps: ', progressbar.widgets.Counter(),
' | ', LossWidget()]
suffix = ' | Dataset: {}'.format(dataset) if dataset else None
pbar = create_progressbar(prefix=prefix, widgets=widgets, suffix=suffix).start()
# Initialize iterator to the appropriate dataset
session.run(init_op)
# Batch loop
while True:
try:
_, current_step, batch_loss, problem_files, step_summary = \
session.run([train_op, global_step, loss, non_finite_files, step_summaries_op],
feed_dict=feed_dict)
except tf.errors.OutOfRangeError:
break
if problem_files.size > 0:
problem_files = [f.decode('utf8') for f in problem_files[..., 0]]
log_error('The following files caused an infinite (or NaN) '
'loss: {}'.format(','.join(problem_files)))
total_loss += batch_loss
step_count += 1
pbar.update(step_count)
step_summary_writer.add_summary(step_summary, current_step)
if is_train and FLAGS.checkpoint_secs > 0 and time.time() - checkpoint_time > FLAGS.checkpoint_secs:
checkpoint_saver.save(session, checkpoint_path, global_step=current_step)
checkpoint_time = time.time()
pbar.finish()
mean_loss = total_loss / step_count if step_count > 0 else 0.0
return mean_loss, step_count
log_info('STARTING Optimization')
train_start_time = datetime.utcnow()
best_dev_loss = float('inf')
dev_losses = []
try:
for epoch in range(FLAGS.epochs):
# Training
log_progress('Training epoch %d...' % epoch)
train_loss, _ = run_set('train', epoch, train_init_op)
log_progress('Finished training epoch %d - loss: %f' % (epoch, train_loss))
checkpoint_saver.save(session, checkpoint_path, global_step=global_step)
if FLAGS.dev_files:
# Validation
dev_loss = 0.0
total_steps = 0
for csv, init_op in zip(dev_csvs, dev_init_ops):
log_progress('Validating epoch %d on %s...' % (epoch, csv))
set_loss, steps = run_set('dev', epoch, init_op, dataset=csv)
dev_loss += set_loss * steps
total_steps += steps
log_progress('Finished validating epoch %d on %s - loss: %f' % (epoch, csv, set_loss))
dev_loss = dev_loss / total_steps
dev_losses.append(dev_loss)
if dev_loss < best_dev_loss:
best_dev_loss = dev_loss
save_path = best_dev_saver.save(session, best_dev_path, global_step=global_step, latest_filename='best_dev_checkpoint')
log_info("Saved new best validating model with loss %f to: %s" % (best_dev_loss, save_path))
# Early stopping
if FLAGS.early_stop and len(dev_losses) >= FLAGS.es_steps:
mean_loss = np.mean(dev_losses[-FLAGS.es_steps:-1])
std_loss = np.std(dev_losses[-FLAGS.es_steps:-1])
dev_losses = dev_losses[-FLAGS.es_steps:]
log_debug('Checking for early stopping (last %d steps) validation loss: '
'%f, with standard deviation: %f and mean: %f' %
(FLAGS.es_steps, dev_losses[-1], std_loss, mean_loss))
if dev_losses[-1] > np.max(dev_losses[:-1]) or \
(abs(dev_losses[-1] - mean_loss) < FLAGS.es_mean_th and std_loss < FLAGS.es_std_th):
log_info('Early stop triggered as (for last %d steps) validation loss:'
' %f with standard deviation: %f and mean: %f' %
(FLAGS.es_steps, dev_losses[-1], std_loss, mean_loss))
break
except KeyboardInterrupt:
pass
log_info('FINISHED optimization in {}'.format(datetime.utcnow() - train_start_time))
log_debug('Session closed.')
def __init__(self,
train_ops,
graph=None,
clip_gradients=5.0,
tensorboard_dir="/tmp/tflearn_logs/",
tensorboard_verbose=0,
checkpoint_path=None,
best_checkpoint_path=None,
max_checkpoints=None,
keep_checkpoint_every_n_hours=10000.0,
random_seed=None,
session=None,
best_val_accuracy=0.0):
self.graph = tf.get_default_graph()
self.summ_writer = None
if graph:
self.graph = graph
with self.graph.as_default():
init_training_mode()
train_ops = to_list(train_ops)
duplicate_identical_ops(train_ops)
if random_seed:
tf.set_random_seed(random_seed)
self.restored = False
self.tensorboard_dir = check_dir_name(tensorboard_dir)
self.training_state = TrainingState()
self.train_ops = to_list(train_ops)
self.validate_trainop_names()
self.global_step = tf.Variable(0.,
name='Global_Step',
trainable=False)
self.incr_global_step = tf.assign(self.global_step,
tf.add(self.global_step, 1))
self.best_val_accuracy = best_val_accuracy
self.best_checkpoint_path = best_checkpoint_path
config = None
tflearn_conf = tf.get_collection(tf.GraphKeys.GRAPH_CONFIG)
if tflearn_conf:
config = tflearn_conf[0]
if not session:
self.session = tf.Session(config=config)
else:
self.session = session
self.restored = True
self.coord = tf.train.Coordinator()
for i, train_op in enumerate(self.train_ops):
# For display simplicity in Tensorboard, if only one optmizer,
# we don't display its name
if len(train_ops) == 1:
train_op.scope_name = ""
train_op.initialize_training_ops(i, self.session,
tensorboard_verbose,
clip_gradients)
# Saver for saving a model
self.saver = tf.train.Saver(
max_to_keep=max_checkpoints,
keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
allow_empty=True)
# Saver for saving a best validation accuracy model
if self.best_checkpoint_path:
self.val_saver = tf.train.Saver(
max_to_keep=1,
keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
allow_empty=True)
# Saver for restoring a model (With exclude variable list)
all_vars = variables.get_all_variables()
excl_vars = tf.get_collection(tf.GraphKeys.EXCL_RESTORE_VARS)
to_restore = [
item for item in all_vars
if check_restore_tensor(item, excl_vars)
]
self.restorer = tf.train.Saver(
var_list=to_restore,
max_to_keep=max_checkpoints,
keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
allow_empty=True)
# A second Saver, that only restore trainable variables
to_restore_trainvars = [
item for item in tf.trainable_variables()
if check_restore_tensor(item, excl_vars)
]
self.restorer_trainvars = tf.train.Saver(
var_list=to_restore_trainvars,
max_to_keep=max_checkpoints,
keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
allow_empty=True)
self.to_restore = to_restore
self.to_restore_trainvars = to_restore_trainvars
self.checkpoint_path = checkpoint_path
if not self.restored:
# TF 0.12 fix
try:
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
self.session.run(
tf.variables_initializer(
tf.get_collection_ref('is_training')))
except Exception as e:
init = tf.initialize_all_variables()
self.session.run(init)
def prepare_processing_graph(self, flags):
"""Builds a TensorFlow graph to apply the input distortions.
Creates a graph that loads a WAVE file, decodes it, scales the volume,
shifts it in time, adds in background noise, calculates a spectrogram, and
then builds an MFCC fingerprint from that.
This must be called with an active TensorFlow session running, and it
creates multiple placeholder inputs, and one output:
- wav_filename_placeholder_: Filename of the WAV to load.
- foreground_volume_placeholder_: How loud the main clip should be.
- foreground_resampling_placeholder_: Controls signal stretching/squeezing
- time_shift_padding_placeholder_: Where to pad the clip.
- time_shift_offset_placeholder_: How much to move the clip in time.
- background_data_placeholder_: PCM sample data for background noise.
- background_volume_placeholder_: Loudness of mixed-in background.
- output_: Output 2D fingerprint of processed audio or raw audio.
Args:
flags: data and model parameters, described at model_train.py
Raises:
ValueError: If the preprocessing mode isn't recognized.
Exception: If the preprocessor wasn't compiled in.
"""
with tf.get_default_graph().name_scope('data'):
desired_samples = flags.desired_samples
self.wav_filename_placeholder_ = tf.placeholder(
tf.string, [], name='wav_filename')
wav_loader = io_ops.read_file(self.wav_filename_placeholder_)
wav_decoder = tf.audio.decode_wav(wav_loader,
desired_channels=1,
desired_samples=desired_samples)
# Allow the audio sample's volume to be adjusted.
self.foreground_volume_placeholder_ = tf.placeholder(
tf.float32, [], name='foreground_volume')
# signal resampling to generate more training data
# it will stretch or squeeze input signal proportinally to:
self.foreground_resampling_placeholder_ = tf.placeholder(
tf.float32, [])
wav_resampled = resample(wav_decoder.audio,
self.foreground_resampling_placeholder_,
desired_samples)
scaled_foreground = tf.multiply(
wav_resampled, self.foreground_volume_placeholder_)
# Shift the sample's start position, and pad any gaps with zeros.
self.time_shift_padding_placeholder_ = tf.placeholder(
tf.int32, [2, 2], name='time_shift_padding')
self.time_shift_offset_placeholder_ = tf.placeholder(
tf.int32, [2], name='time_shift_offset')
sliced_foreground, padded_foreground = shift_in_time(
scaled_foreground, self.time_shift_padding_placeholder_,
self.time_shift_offset_placeholder_, desired_samples)
# Mix in background noise.
self.background_data_placeholder_ = tf.placeholder(
tf.float32, [desired_samples, 1], name='background_data')
self.background_volume_placeholder_ = tf.placeholder(
tf.float32, [], name='background_volume')
background_mul = tf.multiply(self.background_data_placeholder_,
self.background_volume_placeholder_)
background_add = tf.add(background_mul, sliced_foreground)
background_clamp = tf.clip_by_value(background_add, -1.0, 1.0)
# if flags.preprocess == 'raw':
# background_clamp dims: [time, channels]
# remove channel dim
self.output_ = tf.squeeze(background_clamp, axis=1)
def norm(input_tensor: tf.Tensor,
norm_type: Optional[str] = 'BATCH',
groups: Optional[int] = 32,
training: Optional[bool] = False,
name: Optional[str] = None,
epsilon: Optional[float] = 1.001e-5,
momentum: Optional[float] = 0.997,
axis: Optional[int] = -1,
center: Optional[bool] = True,
scale: Optional[bool] = True,
*_):
"""Apply norm ops.
Args:
input_tensor: Input data
norm_type: Type of normalization to be applied - ('BATCH' or 'GROUP')
groups: Number of channel groups over which stats are computed
training: True if network is constructed for training
name: Optional name prefix for this op
epsilon: Small float added to variance to avoid dividing by zero.
momentum: Momentum for the moving average.
axis: An `int`, the axis that should be normalized (typically the features
axis).
center: If True, add offset of `beta` to normalized tensor. If False, `beta`
is ignored.
scale: If True, multiply by `gamma`. If False, `gamma` is
not used. When the next layer is linear (also e.g. `nn.relu`), this can be
disabled since the scaling can be done by the next layer.
Returns:
A `Tensor` representing the output of the operation, same shape as input_tensor
Raises:
ValueError if unsupported norm_type is requested
"""
if norm_type == 'BATCH':
if not name:
name = unique_object_name("bn", zero_based=True)
else:
name = unique_object_name(name, zero_based=True)
name_scope = tf.get_default_graph().get_name_scope()
if name_scope not in (None, ""):
name = name_scope + "/" + name
output_tensor = tf.keras.layers.BatchNormalization(
axis=axis,
fused=True,
center=center,
scale=scale,
trainable=training,
momentum=momentum,
epsilon=epsilon,
name=name)(input_tensor, training=training)
elif norm_type == 'GROUP':
if not name:
name = unique_object_name("gn", zero_based=True)
output_tensor = normalization_ops.group_norm(input_tensor,
groups=groups,
center=center,
scale=scale,
training=training,
trainable=training,
channels_axis=-1,
reduction_axes=[-3, -2],
scope=name)
else:
raise ValueError(
'norm_type must be "BATCH" or "GROUP", got %s instead' % norm_type)
return output_tensor
def prepare_processing_graph(self, flags):
"""Builds a TensorFlow graph to apply the input distortions.
Creates a graph that loads a WAVE file, decodes it, scales the volume,
shifts it in time, adds in background noise, calculates a spectrogram, and
then builds an MFCC fingerprint from that.
This must be called with an active TensorFlow session running, and it
creates multiple placeholder inputs, and one output:
- wav_filename_placeholder_: Filename of the WAV to load.
- foreground_volume_placeholder_: How loud the main clip should be.
- foreground_resampling_placeholder_: Controls signal stretching/squeezing
- time_shift_padding_placeholder_: Where to pad the clip.
- time_shift_offset_placeholder_: How much to move the clip in time.
- background_data_placeholder_: PCM sample data for background noise.
- background_volume_placeholder_: Loudness of mixed-in background.
- output_: Output 2D fingerprint of processed audio or raw audio.
Args:
flags: data and model parameters, described at model_train.py
Raises:
ValueError: If the preprocessing mode isn't recognized.
Exception: If the preprocessor wasn't compiled in.
"""
with tf.get_default_graph().name_scope('data'):
desired_samples = flags.desired_samples
self.wav_filename_placeholder_ = tf.placeholder(
tf.string, [], name='wav_filename')
wav_loader = io_ops.read_file(self.wav_filename_placeholder_)
wav_decoder = tf.audio.decode_wav(
wav_loader, desired_channels=1, desired_samples=desired_samples)
# Allow the audio sample's volume to be adjusted.
self.foreground_volume_placeholder_ = tf.placeholder(
tf.float32, [], name='foreground_volume')
# signal resampling to generate more training data
# it will stretch or squeeze input signal proportinally to:
self.foreground_resampling_placeholder_ = tf.placeholder(tf.float32, [])
if self.foreground_resampling_placeholder_ != 1.0:
image = tf.expand_dims(wav_decoder.audio, 0)
image = tf.expand_dims(image, 2)
shape = tf.shape(wav_decoder.audio)
image_resized = tf.image.resize(
images=image,
size=(tf.cast((tf.cast(shape[0], tf.float32) *
self.foreground_resampling_placeholder_),
tf.int32), 1),
preserve_aspect_ratio=False)
image_resized_cropped = tf.image.resize_with_crop_or_pad(
image_resized,
target_height=desired_samples,
target_width=1,
)
image_resized_cropped = tf.squeeze(image_resized_cropped, axis=[0, 3])
scaled_foreground = tf.multiply(image_resized_cropped,
self.foreground_volume_placeholder_)
else:
scaled_foreground = tf.multiply(wav_decoder.audio,
self.foreground_volume_placeholder_)
# Shift the sample's start position, and pad any gaps with zeros.
self.time_shift_padding_placeholder_ = tf.placeholder(
tf.int32, [2, 2], name='time_shift_padding')
self.time_shift_offset_placeholder_ = tf.placeholder(
tf.int32, [2], name='time_shift_offset')
padded_foreground = tf.pad(
tensor=scaled_foreground,
paddings=self.time_shift_padding_placeholder_,
mode='CONSTANT')
sliced_foreground = tf.slice(padded_foreground,
self.time_shift_offset_placeholder_,
[desired_samples, -1])
# Mix in background noise.
self.background_data_placeholder_ = tf.placeholder(
tf.float32, [desired_samples, 1], name='background_data')
self.background_volume_placeholder_ = tf.placeholder(
tf.float32, [], name='background_volume')
background_mul = tf.multiply(self.background_data_placeholder_,
self.background_volume_placeholder_)
background_add = tf.add(background_mul, sliced_foreground)
background_clamp = tf.clip_by_value(background_add, -1.0, 1.0)
if flags.preprocess == 'raw':
# return raw audio
self.output_ = background_clamp
tf.summary.image(
'input_audio',
tf.expand_dims(tf.expand_dims(background_clamp, -1), -1),
max_outputs=1)
else:
# Run the spectrogram and MFCC ops to get a 2D audio 'fingerprint'
spectrogram = audio_ops.audio_spectrogram(
background_clamp,
window_size=flags.window_size_samples,
stride=flags.window_stride_samples,
magnitude_squared=True)
tf.summary.image(
'spectrogram', tf.expand_dims(spectrogram, -1), max_outputs=1)
# The number of buckets in each FFT row in the spectrogram will depend
# on how many input samples there are in each window. This can be quite
# large, with a 160 sample window producing 127 buckets for example. We
# don't need this level of detail for classification, so we often want
# to shrink them down to produce a smaller result. That's what this
# section implements. One method is to use average pooling to merge
# adjacent buckets, but a more sophisticated approach is to apply the
# MFCC algorithm to shrink the representation.
if flags.preprocess == 'average':
self.output_ = tf.nn.pool(
input=tf.expand_dims(spectrogram, -1),
window_shape=[1, flags.average_window_width],
strides=[1, flags.average_window_width],
pooling_type='AVG',
padding='SAME')
tf.summary.image('shrunk_spectrogram', self.output_, max_outputs=1)
elif flags.preprocess == 'mfcc':
self.output_ = audio_ops.mfcc(
spectrogram,
wav_decoder.sample_rate,
dct_coefficient_count=flags.fingerprint_width)
tf.summary.image(
'mfcc', tf.expand_dims(self.output_, -1), max_outputs=1)
elif flags.preprocess == 'micro':
if not frontend_op:
raise Exception(
'Micro frontend op is currently not available when running'
' TensorFlow directly from Python, you need to build and run'
' through Bazel')
sample_rate = flags.sample_rate
window_size_ms = (flags.window_size_samples * 1000) / sample_rate
window_step_ms = (flags.window_stride_samples * 1000) / sample_rate
int16_input = tf.cast(tf.multiply(background_clamp, 32768), tf.int16)
micro_frontend = frontend_op.audio_microfrontend(
int16_input,
sample_rate=sample_rate,
window_size=window_size_ms,
window_step=window_step_ms,
num_channels=flags.fingerprint_width,
out_scale=1,
out_type=tf.float32)
self.output_ = tf.multiply(micro_frontend, (10.0 / 256.0))
tf.summary.image(
'micro',
tf.expand_dims(tf.expand_dims(self.output_, -1), 0),
max_outputs=1)
else:
raise ValueError('Unknown preprocess mode "%s" (should be "mfcc", '
' "average", or "micro")' % (flags.preprocess))
# Merge all the summaries and write them out to /tmp/retrain_logs (by
# default)
self.merged_summaries_ = tf.summary.merge_all(scope='data')
if flags.summaries_dir:
self.summary_writer_ = tf.summary.FileWriter(
flags.summaries_dir + '/data', tf.get_default_graph())
def depthwise_conv(input_tensor: tf.Tensor,
kernel_size: Union[int, Tuple[int, int]],
filters_out: Optional[int] = None,
stride: Optional[int] = 1,
padding: Optional[str] = 'SAME',
add_bias: Optional[bool] = True,
dtype: Optional[Any] = tf.float16,
name: Optional[str] = None,
*_):
"""Apply depthwise conv and optional bias on input tensor with Tensorflow.
Performs a depthwise convolution
Args:
input_tensor: Input data
kernel_size: Filter size (assumes equal height and width)
filters_out: Number of output filters
stride: Stride of the filter
padding: Type of padding to use
add_bias: Should bias be added
dtype: Data type of parameters
name: Optional name for this op
Returns: Output of convolution operator.
"""
# Assumes input in NHWC format.
filters_in = input_tensor.get_shape()[-1]
if isinstance(kernel_size, int):
depthwise_kernel_shape = [kernel_size, kernel_size, filters_in, 1]
else:
depthwise_kernel_shape = kernel_size + (filters_in, 1)
w_init = contrib.layers.xavier_initializer(dtype=dtype)
name_scope = tf.get_default_graph().get_name_scope()
if name_scope not in ["", None]:
name = name_scope + "/" + name
with tf.get_default_graph().as_default():
with tf.variable_scope(name):
depthwise_kernel = tf.get_variable('depthwise_kernel',
shape=depthwise_kernel_shape,
initializer=w_init,
dtype=dtype)
output_tensor = tf.nn.depthwise_conv2d(input_tensor,
depthwise_kernel,
strides=[1, stride, stride, 1],
padding=padding.upper())
if add_bias:
if filters_out:
b_shape = [filters_out]
else:
b_shape = [filters_in]
b_init = tf.zeros_initializer()
with tf.variable_scope(name):
biases = tf.get_variable('conv/bias',
shape=b_shape,
initializer=b_init,
dtype=dtype)
output_tensor += biases
return output_tensor
def main(): # pylint: disable=R0914, R0915
"""main process"""
args_check(ARGS)
mkdir(OUTPUTS)
# Phase Check original model accuracy
# Step 1: Load and evaluate the target model
graph = tf.get_default_graph()
session = tf.Session()
saver = tf.train.import_meta_graph(ARGS.eval_model)
saver.restore(session, ARGS.ckpt_path)
acc_1_before, acc_5_before = evaluate(session)
session.close()
# Phase retrain the model
# Step 1: Generate training dataset.
tf.reset_default_graph()
graph = tf.get_default_graph()
# Step 2: Load the training model.
saver = tf.train.import_meta_graph(ARGS.train_model)
# Step 3: Create the retraining configuration file.
record_file = os.path.join(OUTPUTS, 'record.txt')
config_defination = ARGS.config_defination
# Step 3: Generate the compressed training model in default graph and create the compression record_file.
retrain_ops = amct.create_compressed_retrain_model(graph,
config_defination,
[PREDICTIONS],
record_file)
# Step 4: Retrain the modified model and save parameters
retrain_ckpt = os.path.join(OUTPUTS, 'resnet_v1_50_retrain')
retrain(saver, retrain_ckpt)
# Phase convert retrain model
# Step 1: Load the evaluation model.
tf.reset_default_graph()
graph = tf.get_default_graph()
saver = tf.train.import_meta_graph(ARGS.eval_model)
# Step 2: Generate the compressed evaluation model accroding to the configuration.
retrain_ops = amct.create_compressed_retrain_model(graph,
config_defination,
[PREDICTIONS],
record_file)
# Step 3: Set the variables which needed to restore.
variables_to_restore = tf.global_variables()
saver_restore = tf.train.Saver(variables_to_restore)
# Step 4: Restore the variables and use the eval_set inference output node
# (retrain_ops[-1]) to write the quantization factor into the record_file.
session = tf.Session()
session.run(tf.global_variables_initializer())
retrain_ckpt = retrain_ckpt + '-' + str(ARGS.train_iter)
saver_restore.restore(session, retrain_ckpt)
evaluate_for_search_n(session, retrain_ops[-1].name[:-2], ARGS.batch_num)
# Step 5: Convert all variables to constants and finally save as 'pb' file.
constant_graph = tf.graph_util.convert_variables_to_constants(
session, graph.as_graph_def(), [PREDICTIONS])
pb_path = os.path.join(OUTPUTS, 'resnet_v1_50.pb')
with tf.io.gfile.GFile(pb_path, 'wb') as pb_file:
pb_file.write(constant_graph.SerializeToString())
session.close()
# Step 6: Convert origin 'pb' model file to 'pb' model, using the factor record_file.
compressed_pb_path = os.path.join(OUTPUTS, 'resnet_v1_50_comp')
amct.save_compressed_retrain_model(pb_path, [PREDICTIONS], record_file,
compressed_pb_path)
# Phase verification
# Step 1: Generate validation dataset.
tf.reset_default_graph()
graph = tf.get_default_graph()
# Step 2: Load the fake quantized model and validation it.
compressed_pb_file = compressed_pb_path + '_quantized.pb'
with tf.io.gfile.GFile(compressed_pb_file, 'rb') as fid:
graph_def = tf.GraphDef()
graph_def.ParseFromString(fid.read())
tf.import_graph_def(graph_def, name='')
session = tf.Session()
acc_1, acc_5 = evaluate(session)
session.close()
print('The origin model top 1 accuracy = {}%.'.format(acc_1_before))
print('The origin model top 5 accuracy = {}%.'.format(acc_5_before))
print('The model after retrain top 1 accuracy = {}%.'.format(acc_1))
print('The model after retrain top 5 accuracy = {}%.'.format(acc_5))
def conv(input_tensor: tf.Tensor,
kernel_size: Union[int, Tuple[int, int]],
filters_out: int,
stride: Optional[int] = 1,
padding: Optional[str] = 'SAME',
add_bias: Optional[bool] = True,
dtype: Optional[Any] = tf.float16,
name: Optional[str] = None,
weight_suffix: Optional[str] = "kernel",
bias_suffix: Optional[str] = "conv/bias",
*_):
"""Apply conv and optional bias on input tensor with Tensorflow.
Args:
input_tensor: Input data
kernel_size: Filter size (assumes equal height and width)
filters_out: Number of output filters
stride: Stride of the filter
padding: Type of padding to use
add_bias: Should bias be added
dtype: Data type of parameters
name: Optional name for this op
Returns: Output of convolution operator.
"""
# Assumes input in NHWC format.
filters_in = input_tensor.get_shape()[-1]
if isinstance(kernel_size, int):
w_shape = [kernel_size, kernel_size, filters_in, filters_out]
else:
w_shape = kernel_size + (filters_in, filters_out)
w_init = contrib.layers.xavier_initializer(dtype=dtype)
if name is None:
name = unique_object_name("conv2d", zero_based=True)
name_scope = tf.get_default_graph().get_name_scope()
if name_scope not in ["", None]:
name = name_scope + "/" + name
with tf.get_default_graph().as_default():
with tf.variable_scope(name):
weights = tf.get_variable(weight_suffix,
shape=w_shape,
initializer=w_init,
dtype=dtype)
output_tensor = tf.nn.conv2d(input_tensor,
weights, [1, stride, stride, 1],
padding=padding.upper(),
name=name)
if add_bias:
b_shape = [filters_out]
b_init = tf.zeros_initializer()
with tf.variable_scope(name):
biases = tf.get_variable(bias_suffix,
shape=b_shape,
initializer=b_init,
dtype=dtype)
output_tensor += biases
return output_tensor
def load_model(args):
"""
load model from file
:param args: command args
:return: model dictionary and session
"""
sess = tf.Session()
# Embedding model
with tf.gfile.GFile(args.model, "rb") as f:
graph_def = tf.GraphDef() # 新建GraphDef文件，用于临时载入模型中的图
graph_def.ParseFromString(
f.read()) # GraphDef加载模型中的图--据说可能失败，因为没有保存variable
tf.import_graph_def(
graph_def, # 在当前默认图中加载GraphDef中的图
input_map=None,
return_elements=None,
name="")
writer = tf.summary.FileWriter("log")
writer.add_graph(sess.graph)
moment_val = 0.9
step_val = 1.0 / args.gradient_step
image_input = tf.get_default_graph().get_tensor_by_name(
'image_input:0') # (?, 3, 112, 112)
keep_prob = tf.get_default_graph().get_tensor_by_name('keep_prob:0')
is_train = tf.get_default_graph().get_tensor_by_name('training_mode:0')
embedding = tf.get_default_graph().get_tensor_by_name(
'embedding:0') # (?, 512)
stage4_unit3_conv2 = tf.get_default_graph().get_tensor_by_name(
'stage4_unit3_conv2:0') # (?, 512, 7, 7)
positive = tf.placeholder(tf.float32,
shape=[512],
name='positive_embedding')
patches = tf.placeholder(tf.int32,
shape=[None, 1, 112, 112],
name='patches')
moments = tf.placeholder(tf.float32,
shape=[None, 3, 112, 112],
name='moments')
loss = tf.math.multiply(
tf.reduce_sum(tf.multiply(embedding, positive), axis=1, name='loss'),
-1) # (?,)
grad_input = tf.gradients(loss, image_input, name="grad_input")[0]
new_moments = tf.math.add(tf.math.multiply(moments, moment_val),
tf.math.multiply(grad_input, (1. - moment_val)))
img_grad = tf.math.multiply(tf.sign(new_moments), step_val)
grad_mask = tf.multiply(img_grad,
tf.cast(patches, dtype=tf.float32),
name='grad_mask')
adv_imgs = tf.add(image_input, grad_mask, name='adv_imgs')
grad_conv = tf.gradients(loss, stage4_unit3_conv2)[0]
model_dic = {
'image_input': image_input,
'keep_prob': keep_prob,
'is_train': is_train,
'embedding': embedding,
'stage4_unit3_conv2': stage4_unit3_conv2,
'positive_embedding': positive,
'loss': loss,
'grad_input': grad_input,
'grad_conv': grad_conv,
'patches': patches,
'adv_imgs': adv_imgs,
'moments': moments,
'new_moments': new_moments
}
return model_dic, sess
def _get_iter_variable(self):
graph = (None if tf.executing_eagerly() else tf.get_default_graph())
return self._get_non_slot_variable("iter", graph=graph)
def __init__(self, atom_types, **kwargs):
self.atom_types = atom_types
self.build_forces = kwargs.get('build_forces', False)
self.precision = kwargs.get('precision', _tf.float32)
self.atomic_contributions = {}
self.atom_indices = {}
self.feature_types = {'error_weights': self.precision}
self.feature_shapes = {
'error_weights': _tf.TensorShape([
None,
])
}
for t in self.atom_types:
self.feature_types['%s_indices' % t] = _tf.int32
self.feature_shapes['%s_indices' % t] = _tf.TensorShape([None, 1])
self.label_types = {'energy': self.precision}
self.label_shapes = {
'energy': _tf.TensorShape([
None,
])
}
if self.build_forces:
self.label_types['forces'] = self.precision
self.label_shapes['forces'] = _tf.TensorShape([None, None, 3])
# The individual atomic contributions and the data iterator is set up
# in this abstact method that has to be implemented for each potential
# type
self.configureAtomicContributions(**kwargs)
# Convenience handle for backwards compatibility
self.ANNs = self.atomic_contributions
self.target = self.labels['energy']
if self.build_forces:
self.target_forces = self.labels['forces']
for t in self.atom_types:
self.atom_indices[t] = self.features['%s_indices' % t]
with _tf.name_scope('Energy_Calculation'):
self.E_predict = _tf.scatter_nd(
_tf.concat([self.atom_indices[t] for t in self.atom_types], 0),
_tf.concat([
_tf.reshape(self.atomic_contributions[t].output, [-1])
for t in self.atom_types
], 0),
_tf.shape(self.target),
name='E_prediction')
with _tf.name_scope('Atom_Counting'):
self.num_atoms = _tf.reduce_sum(
[_tf.bincount(self.atom_indices[t]) for t in self.atom_types],
axis=0,
name='NumberOfAtoms')
with _tf.name_scope('MSE'):
self.error_weights = self.features['error_weights']
self.mse = _tf.reduce_mean(
(self.target - self.E_predict)**2 * self.error_weights)
self.mse_summ = _tf.summary.scalar('MSE',
self.mse,
family='performance')
with _tf.name_scope('RMSE'):
self.error_weights = self.features['error_weights']
self.rmse = _tf.sqrt(
_tf.reduce_mean(
(self.target - self.E_predict)**2 * self.error_weights))
self.rmse_summ = _tf.summary.scalar('RMSE',
self.rmse,
family='performance')
if self.build_forces:
with _tf.name_scope('Force_Calculation'):
self.gradients = {}
for t in self.atom_types:
self.gradients[t] = _tf.gradients(
self.atomic_contributions[t].output,
self.atomic_contributions[t].input,
name='%s_gradients' % t)[0]
self.F_predict = _tf.negative(
_tf.scatter_nd(
_tf.concat(
[self.atom_indices[t] for t in self.atom_types],
0),
_tf.concat([
_tf.einsum(
'ijkl,ij->ikl',
self.atomic_contributions[t].derivatives_input,
self.gradients[t],
name='%s_einsum' % t) for t in self.atom_types
], 0),
_tf.shape(self.target_forces),
name='F_prediction'))
with _tf.name_scope('MSE_Forces'):
# Following Behler. Int. J. Quant. Chem. 2015 115, 1032-1050
# equation (21). !! Not the same !! for varying number of atoms
self.mse_forces = _tf.reduce_mean(
_tf.reduce_mean((self.target_forces - self.F_predict)**2,
axis=[1, 2]) * self.error_weights)
self.mse_forces_summ = _tf.summary.scalar('MSE_Forces',
self.mse_forces,
family='performance')
with _tf.name_scope('RMSE_Forces'):
# Following Behler. Int. J. Quant. Chem. 2015 115, 1032-1050
# equation (21). !! Not the same !! for varying number of atoms
self.rmse_forces = _tf.sqrt(
_tf.reduce_mean(
_tf.reduce_mean(
(self.target_forces - self.F_predict)**2,
axis=[1, 2]) * self.error_weights))
self.rmse_forces_summ = _tf.summary.scalar(
'RMSE_Forces', self.rmse_forces, family='performance')
self.variables = _tf.get_collection(
_tf.GraphKeys.MODEL_VARIABLES,
scope=_tf.get_default_graph().get_name_scope())
self.saver = _tf.train.Saver(self.variables,
max_to_keep=None,
save_relative_paths=True)
def main(argv):
mnist = input_data.read_data_sets("data/mnist", one_hot=True)
train_images, train_labels = mnist.train.images, mnist.train.labels
test_images, test_labels = mnist.test.images, mnist.test.labels
model = RNN(input_size=cfgs.INPUT_SIZE, time_steps=cfgs.TIME_STEPS, num_layers=cfgs.NUM_LAYERS,
num_units=cfgs.NUM_UNITS)
saver = tf.train.Saver(max_to_keep=30)
# get computer graph
graph = tf.get_default_graph()
write = tf.summary.FileWriter(logdir=cfgs.SUMMARY_PATH, graph=graph)
# os.environ["CUDA_VISIBLE_DEVICES"] = '0'
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5 # maximun alloc gpu50% of MEM
config.gpu_options.allow_growth = True
init_op = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer()
)
# train and save model
with tf.Session(config=config) as sess:
sess.run(init_op)
# get model variable of network
model_variable = tf.model_variables()
for var in model_variable:
print(var.op.name, var.shape)
# get and add histogram to summary protocol buffer
# merges all summaries collected in the default graph
summary_op = tf.summary.merge_all()
train_step_per_epoch = mnist.train.num_examples // cfgs.BATCH_SIZE
test_step_pre_epoch = mnist.test.num_examples // cfgs.BATCH_SIZE
for epoch in range(1, cfgs.NUM_EPOCH+1):
train_bar = tqdm(range(1, train_step_per_epoch+1))
for step in train_bar:
x_train, y_train = mnist.train.next_batch(cfgs.BATCH_SIZE)
x_train = x_train.reshape(cfgs.BATCH_SIZE, cfgs.TIME_STEPS, cfgs.INPUT_SIZE)
feed_dict = model.fill_feed_dict(x_train, y_train, keep_prob=cfgs.KEPP_PROB)
summary, global_step, train_loss, train_acc, _ = sess.run([summary_op, model.global_step, model.loss, model.acc, model.train],
feed_dict=feed_dict)
if step % cfgs.SMRY_ITER == 0:
write.add_summary(summary=summary, global_step=global_step)
write.flush()
train_bar.set_description("Epoch {0} : Step {1} => Train Loss: {2:.4f} | Train ACC: {3:.4f}".
format(epoch, step, train_loss, train_acc))
test_loss_list = []
test_acc_list = []
for step in range(test_step_pre_epoch):
x_test, y_test = mnist.test.next_batch(cfgs.BATCH_SIZE)
x_test = x_test.reshape(cfgs.BATCH_SIZE, cfgs.TIME_STEPS, cfgs.INPUT_SIZE)
feed_dict = model.fill_feed_dict(x_test, y_test, keep_prob=1.0)
test_loss, test_acc, _ = sess.run([model.loss, model.acc, model.train], feed_dict=feed_dict)
test_loss_list.append(test_loss)
test_acc_list.append(test_acc)
test_loss = sum(test_loss_list) / len(test_loss_list)
test_acc = sum(test_acc_list) / len(test_acc_list)
print("Epoch {0} : Step {1} => Val Loss: {2:.4f} | Val ACC: {3:.4f} ".format(epoch, step,
test_loss, test_acc))
ckpt_file = os.path.join(cfgs.TRAINED_CKPT, 'model_loss={0:4f}.ckpt'.format(test_loss))
saver.save(sess=sess, save_path=ckpt_file, global_step=global_step)
sess.close()
print('model training has complete')
import tensorflow.compat.v1 as tf from keras.applications import vgg16 from tensorflow.python.keras.backend import set_session import os #### # Backward compatible tf.disable_v2_behavior() #### SESS = tf.compat.v1.Session() GRAPH1 = tf.get_default_graph() # Sets the global Tenorflow session. set_session(SESS) IMAGE_MODEL = vgg16.VGG16(weights="imagenet") from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve(strict=True).parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = '6o7x2#!!(usw+wc!fu!izx0dpp%p651+bk#mi)_w8o!jrd&qlx' # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True
def get_input_nodes(self) -> List[str]:
graph = tf.get_default_graph()
return [
n.name + ':0' for n in graph.as_graph_def().node
if len(n.input) == 0 and n.op == 'Placeholder'
]
# --------------------------------------
# BASIC APP SETUP
# --------------------------------------
app = Flask(__name__, instance_relative_config=True)
# Config
app_settings = os.getenv('APP_SETTINGS', 'main.config.DevelopmentConfig')
app.config.from_object(app_settings)
# Extensions
from flask_cors import CORS
CORS(app)
# Keras stuff
global graph
graph = get_default_graph()
model = load_model('main/Sentiment_CNN_model.h5')
MAX_SEQUENCE_LENGTH = 300
# Twitter
auth = tweepy.OAuthHandler(app.config.get('CONSUMER_KEY'),
app.config.get('CONSUMER_SECRET'))
auth.set_access_token(app.config.get('ACCESS_TOKEN'),
app.config.get('ACCESS_TOKEN_SECRET'))
api = tweepy.API(auth, wait_on_rate_limit=True)
# loading tokenizer
with open('main/tokenizer.pickle', 'rb') as handle:
tokenizer = pickle.load(handle)
def __init__(self):
self.model_path = 'AIR/model'
self.signature_key = 'predict_images'
self.input_size = 456
self.input_key_1 = 'input_img'
self.output_key_1 = 'output_score'
config = tf.ConfigProto(allow_soft_placement=True)
with tf.get_default_graph().as_default():
self.sess = tf.Session(graph=tf.Graph(), config=config)
meta_graph_def = tf.saved_model.loader.load(
self.sess, [tag_constants.SERVING], self.model_path)
self.signature = meta_graph_def.signature_def
input_images_tensor_name = self.signature[
self.signature_key].inputs[self.input_key_1].name
output_score_tensor_name = self.signature[
self.signature_key].outputs[self.output_key_1].name
self.input_images = self.sess.graph.get_tensor_by_name(
input_images_tensor_name)
self.output_score = self.sess.graph.get_tensor_by_name(
output_score_tensor_name)
print('网络载入完成')
self.label_id_name_dict = \
{
"0": "一次性快餐盒",
"1": "污损塑料",
"2": "烟蒂",
"3": "牙签",
"4": "破碎花盆及碟碗",
"5": "竹筷",
"6": "剩饭剩菜",
"7": "大骨头",
"8": "水果果皮",
"9": "水果果肉",
"10": "茶叶渣",
"11": "菜叶菜根",
"12": "蛋壳",
"13": "鱼骨",
"14": "充电宝",
"15": "包",
"16": "化妆品瓶",
"17": "塑料玩具",
"18": "塑料碗盆",
"19": "塑料衣架",
"20": "快递纸袋",
"21": "插头电线",
"22": "旧衣服",
"23": "易拉罐",
"24": "枕头",
"25": "毛绒玩具",
"26": "洗发水瓶",
"27": "玻璃杯",
"28": "皮鞋",
"29": "砧板",
"30": "纸板箱",
"31": "调料瓶",
"32": "酒瓶",
"33": "金属食品罐",
"34": "锅",
"35": "食用油桶",
"36": "饮料瓶",
"37": "干电池",
"38": "软膏",
"39": "过期药物"
}
def export():
r'''
Restores the trained variables into a simpler graph that will be exported for serving.
'''
log_info('Exporting the model...')
inputs, outputs, _ = create_inference_graph(batch_size=FLAGS.export_batch_size, n_steps=FLAGS.n_steps, tflite=FLAGS.export_tflite)
graph_version = int(file_relative_read('GRAPH_VERSION').strip())
assert graph_version > 0
outputs['metadata_version'] = tf.constant([graph_version], name='metadata_version')
outputs['metadata_sample_rate'] = tf.constant([FLAGS.audio_sample_rate], name='metadata_sample_rate')
outputs['metadata_feature_win_len'] = tf.constant([FLAGS.feature_win_len], name='metadata_feature_win_len')
outputs['metadata_feature_win_step'] = tf.constant([FLAGS.feature_win_step], name='metadata_feature_win_step')
outputs['metadata_beam_width'] = tf.constant([FLAGS.export_beam_width], name='metadata_beam_width')
outputs['metadata_alphabet'] = tf.constant([Config.alphabet.Serialize()], name='metadata_alphabet')
if FLAGS.export_language:
outputs['metadata_language'] = tf.constant([FLAGS.export_language.encode('utf-8')], name='metadata_language')
# Prevent further graph changes
tfv1.get_default_graph().finalize()
output_names_tensors = [tensor.op.name for tensor in outputs.values() if isinstance(tensor, tf.Tensor)]
output_names_ops = [op.name for op in outputs.values() if isinstance(op, tf.Operation)]
output_names = output_names_tensors + output_names_ops
with tf.Session() as session:
# Restore variables from checkpoint
load_graph_for_evaluation(session)
output_filename = FLAGS.export_file_name + '.pb'
if FLAGS.remove_export:
if os.path.isdir(FLAGS.export_dir):
log_info('Removing old export')
shutil.rmtree(FLAGS.export_dir)
output_graph_path = os.path.join(FLAGS.export_dir, output_filename)
if not os.path.isdir(FLAGS.export_dir):
os.makedirs(FLAGS.export_dir)
frozen_graph = tfv1.graph_util.convert_variables_to_constants(
sess=session,
input_graph_def=tfv1.get_default_graph().as_graph_def(),
output_node_names=output_names)
frozen_graph = tfv1.graph_util.extract_sub_graph(
graph_def=frozen_graph,
dest_nodes=output_names)
if not FLAGS.export_tflite:
with open(output_graph_path, 'wb') as fout:
fout.write(frozen_graph.SerializeToString())
else:
output_tflite_path = os.path.join(FLAGS.export_dir, output_filename.replace('.pb', '.tflite'))
converter = tf.lite.TFLiteConverter(frozen_graph, input_tensors=inputs.values(), output_tensors=outputs.values())
converter.optimizations = [tf.lite.Optimize.DEFAULT]
# AudioSpectrogram and Mfcc ops are custom but have built-in kernels in TFLite
converter.allow_custom_ops = True
tflite_model = converter.convert()
with open(output_tflite_path, 'wb') as fout:
fout.write(tflite_model)
log_info('Models exported at %s' % (FLAGS.export_dir))
metadata_fname = os.path.join(FLAGS.export_dir, '{}_{}_{}.md'.format(
FLAGS.export_author_id,
FLAGS.export_model_name,
FLAGS.export_model_version))
model_runtime = 'tflite' if FLAGS.export_tflite else 'tensorflow'
with open(metadata_fname, 'w') as f:
f.write('---\n')
f.write('author: {}\n'.format(FLAGS.export_author_id))
f.write('model_name: {}\n'.format(FLAGS.export_model_name))
f.write('model_version: {}\n'.format(FLAGS.export_model_version))
f.write('contact_info: {}\n'.format(FLAGS.export_contact_info))
f.write('license: {}\n'.format(FLAGS.export_license))
f.write('language: {}\n'.format(FLAGS.export_language))
f.write('runtime: {}\n'.format(model_runtime))
f.write('min_ds_version: {}\n'.format(FLAGS.export_min_ds_version))
f.write('max_ds_version: {}\n'.format(FLAGS.export_max_ds_version))
f.write('acoustic_model_url: <replace this with a publicly available URL of the acoustic model>\n')
f.write('scorer_url: <replace this with a publicly available URL of the scorer, if present>\n')
f.write('---\n')
f.write('{}\n'.format(FLAGS.export_description))
log_info('Model metadata file saved to {}. Before submitting the exported model for publishing make sure all information in the metadata file is correct, and complete the URL fields.'.format(metadata_fname))
loss = tf.reduce_mean(tf.square(tf.subtract(
model.y_, model.y))) + tf.add_n([tf.nn.l2_loss(v)
for v in train_vars]) * L2NormConst
train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)
sess.run(tf.initialize_all_variables())
# create a summary to monitor cost tensor
tf.summary.scalar("loss", loss)
# merge all summaries into a single op
merged_summary_op = tf.summary.merge_all()
saver = tf.train.Saver(write_version=saver_pb2.SaverDef.V1)
# op to write logs to Tensorboard
logs_path = r'C:\Users\DM\Desktop\Autopilot-TensorFlow-master\logs'
summary_writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph())
epochs = 30
batch_size = 100
# train over the dataset about 30 times
for epoch in range(epochs):
for i in range(int(driving_data.num_images / batch_size)):
xs, ys = driving_data.LoadTrainBatch(batch_size)
train_step.run(feed_dict={
model.x: xs,
model.y_: ys,
model.keep_prob: 0.8
})
if i % 10 == 0:
xs, ys = driving_data.LoadValBatch(batch_size)
