def accuracy_check(self):
print("Inference for accuracy check.")
total_iter = COCO_NUM_VAL_IMAGES
break_session = 251
fm = category_map
fm = dict(zip(list(fm.values()), list(fm.keys())))
print('total iteration is {0}'.format(str(total_iter)))
global model, graph
result = []
with tf.Session().as_default() as sess:
if self.args.data_location:
self.build_data_sess(need_reshape=True)
else:
raise Exception("no data location provided")
evaluator = CocoDetectionEvaluator()
total_samples = 0
self.coord = tf.train.Coordinator()
tfrecord_paths = [self.args.data_location]
ds = tf.data.TFRecordDataset.list_files(tfrecord_paths)
ds = ds.apply(
parallel_interleave(tf.data.TFRecordDataset,
cycle_length=28,
block_length=5,
buffer_output_elements=10000,
prefetch_input_elements=10000))
ds = ds.prefetch(buffer_size=10000)
ds = ds.apply(
map_and_batch(map_func=parse_and_preprocess,
batch_size=self.args.batch_size,
num_parallel_batches=1,
num_parallel_calls=None))
ds = ds.prefetch(buffer_size=10000)
ds_iterator = tf.data.make_one_shot_iterator(ds)
state = None
warmup_iter = 0
self.ground_truth_dicts = {}
self.detect_dicts = {}
self.total_iter = total_iter
self.image_id_gt_dict = {}
self.weights = np.arange(1000)
if self.args.data_location:
image_batches = []
obj = self
for idx in tqdm(range(total_iter)):
bbox, label, image_id, features = ds_iterator.get_next()
result.append((bbox, label, image_id, features))
for idx in tqdm(range(total_iter)):
run_ice_breaker_session(result, obj, fm, sess, total_iter,
break_session, idx)
def run_inference(self, params):
self.logfile.info("Inference for accuracy check.")
total_iter = COCO_NUM_VAL_IMAGES
fm = category_map
fm = dict(zip(list(fm.values()), list(fm.keys())))
self.logfile.info('total iteration is {0}'.format(str(total_iter)))
result = []
global model, graph
with tf.compat.v1.Session().as_default() as sess:
with sess.graph.as_default() as graph:
evaluator = CocoDetectionEvaluator()
total_samples = 0
self.coord = tf.train.Coordinator()
tfrecord_paths = [self.args.data_location]
ds = tf.data.TFRecordDataset.list_files(tfrecord_paths)
ds = ds.apply(
parallel_interleave(tf.data.TFRecordDataset,
cycle_length=1,
block_length=1,
buffer_output_elements=10000,
prefetch_input_elements=10000))
ds = ds.prefetch(buffer_size=10000)
ds = ds.apply(
map_and_batch(map_func=parse_and_preprocess,
batch_size=self.args.batch_size,
num_parallel_batches=1,
num_parallel_calls=None))
ds = ds.prefetch(buffer_size=10000)
ds_iterator = tf.compat.v1.data.make_one_shot_iterator(ds)
state = None
warmup_iter = 0
self.ground_truth_dicts = {}
self.detect_dicts = {}
self.total_iter = total_iter
self.image_id_gt_dict = {}
obj = self
if self.args.data_location:
for idx in range(total_iter):
bbox, label, image_id, features = ds_iterator.get_next(
)
result.append((bbox, label, image_id, features))
for idx in range(total_iter):
run_ice_breaker_session(result, obj, params, fm, sess,
total_iter, idx)
def minibatch(self, dataset, subset, cache_data=False):
with tf.compat.v1.name_scope('batch_processing'):
glob_pattern = dataset.tf_record_pattern(subset)
file_names = gfile.Glob(glob_pattern)
if not file_names:
raise ValueError(
'Found no files in --data_dir matching: {}'.format(
glob_pattern))
ds = tf.data.TFRecordDataset.list_files(file_names)
# number of parallel open files and tfrecords should be tuned according to
# different batch size
ds = ds.apply(
parallel_interleave(tf.data.TFRecordDataset,
cycle_length=28,
block_length=5,
sloppy=True,
buffer_output_elements=10000,
prefetch_input_elements=10000))
if cache_data:
ds = ds.take(1).cache().repeat()
ds = ds.prefetch(buffer_size=10000)
#ds = ds.prefetch(buffer_size=self.batch_size)
ds = ds.apply(
map_and_batch(
map_func=self.parse_and_preprocess,
batch_size=self.batch_size,
num_parallel_batches=56,
num_parallel_calls=None)) # this number should be tuned
ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE
) # this number can be tuned
ds_iterator = tf.compat.v1.data.make_one_shot_iterator(ds)
images, labels = ds_iterator.get_next()
# reshape
labels = tf.reshape(labels, [self.batch_size])
return images, labels
def minibatch(self, dataset, subset, cache_data=False):
with tf.name_scope('batch_processing'):
glob_pattern = dataset.tf_record_pattern(subset)
file_names = gfile.Glob(glob_pattern)
if not file_names:
raise ValueError(
'Found no files in --data_dir matching: {}'.format(
glob_pattern))
ds = tf.data.TFRecordDataset.list_files(file_names)
ds = ds.apply(
parallel_interleave(tf.data.TFRecordDataset,
cycle_length=self.num_cores,
block_length=5,
sloppy=True,
buffer_output_elements=10000,
prefetch_input_elements=10000))
if cache_data:
ds = ds.take(1).cache().repeat()
ds = ds.prefetch(buffer_size=10000)
# ds = ds.prefetch(buffer_size=self.batch_size)
# num of parallel batches not greater than 56
max_num_parallel_batches = min(56, 2 * self.num_cores)
ds = ds.apply(
map_and_batch(map_func=self.parse_and_preprocess,
batch_size=self.batch_size,
num_parallel_batches=max_num_parallel_batches,
num_parallel_calls=None))
ds = ds.prefetch(buffer_size=tf.contrib.data.AUTOTUNE)
ds_iterator = ds.make_one_shot_iterator()
images, labels, filename = ds_iterator.get_next()
# reshape
labels = tf.reshape(labels, [self.batch_size])
filename = tf.reshape(filename, [self.batch_size])
return images, labels, filename
def get_input(self):
tfrecord_paths = [self.args.data_location]
ds = tf.data.TFRecordDataset.list_files(tfrecord_paths)
ds = ds.apply(
parallel_interleave(tf.data.TFRecordDataset,
cycle_length=28,
block_length=5,
sloppy=True,
buffer_output_elements=10000,
prefetch_input_elements=10000))
ds = ds.prefetch(buffer_size=10000)
ds = ds.apply(
map_and_batch(map_func=parse_and_preprocess,
batch_size=self.args.batch_size,
num_parallel_batches=28,
num_parallel_calls=None))
ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
ds_iterator = tf.compat.v1.data.make_one_shot_iterator(ds)
images, bbox, label, image_id = ds_iterator.get_next()
return images, bbox, label, image_id
def minibatch(self, dataset, subset, cache_data=False):
with tf.compat.v1.name_scope('batch_processing'):
glob_pattern = dataset.tf_record_pattern(subset)
file_names = gfile.Glob(glob_pattern)
if not file_names:
raise ValueError(
'Found no files in --data_dir matching: {}'.format(
glob_pattern))
ds = tf.data.TFRecordDataset.list_files(file_names, shuffle=False)
ds = ds.apply(
parallel_interleave(tf.data.TFRecordDataset,
cycle_length=self.num_cores))
if cache_data:
ds = ds.take(1).cache().repeat()
ds = ds.prefetch(buffer_size=self.batch_size)
# num of parallel batches not greater than 56
max_num_parallel_batches = min(56, 2 * self.num_cores)
ds = ds.apply(
map_and_batch(map_func=self.parse_and_preprocess,
batch_size=self.batch_size,
num_parallel_batches=max_num_parallel_batches,
num_parallel_calls=None))
ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE
) # this number can be tuned
ds_iterator = tf.compat.v1.data.make_one_shot_iterator(ds)
images, labels = ds_iterator.get_next()
# reshape
labels = tf.reshape(labels, [self.batch_size])
return images, labels
def build_model(self):
if self.phase == 'train':
""" Input Image"""
img_class = Image_data(self.img_height, self.img_width,
self.img_ch, self.dataset_path,
self.augment_flag)
img_class.preprocess()
dataset_num = max(len(img_class.train_A_dataset),
len(img_class.train_B_dataset))
print("Dataset number : ", dataset_num)
img_slice_A = tf.data.Dataset.from_tensor_slices(
img_class.train_A_dataset)
img_slice_B = tf.data.Dataset.from_tensor_slices(
img_class.train_B_dataset)
gpu_device = '/gpu:0'
img_slice_A = img_slice_A. \
apply(shuffle_and_repeat(dataset_num)). \
apply(map_and_batch(img_class.image_processing, self.batch_size, num_parallel_batches=AUTOTUNE,
drop_remainder=True)). \
apply(prefetch_to_device(gpu_device, AUTOTUNE))
img_slice_B = img_slice_B. \
apply(shuffle_and_repeat(dataset_num)). \
apply(map_and_batch(img_class.image_processing, self.batch_size, num_parallel_batches=AUTOTUNE,
drop_remainder=True)). \
apply(prefetch_to_device(gpu_device, AUTOTUNE))
self.dataset_A_iter = iter(img_slice_A)
self.dataset_B_iter = iter(img_slice_B)
""" Network """
self.source_generator = Generator(self.ch,
self.n_res,
name='source_generator')
self.target_generator = Generator(self.ch,
self.n_res,
name='target_generator')
self.source_discriminator = Discriminator(
self.ch, self.n_dis, self.sn, name='source_discriminator')
self.target_discriminator = Discriminator(
self.ch, self.n_dis, self.sn, name='target_discriminator')
""" Optimizer """
self.g_optimizer = tf.keras.optimizers.Adam(
learning_rate=self.init_lr,
beta_1=0.5,
beta_2=0.999,
epsilon=1e-08)
self.d_optimizer = tf.keras.optimizers.Adam(
learning_rate=self.init_lr,
beta_1=0.5,
beta_2=0.999,
epsilon=1e-08)
""" Summary """
# mean metric
self.g_adv_loss_metric = tf.keras.metrics.Mean('g_adv_loss',
dtype=tf.float32)
self.g_cyc_loss_metric = tf.keras.metrics.Mean('g_cyc_loss',
dtype=tf.float32)
self.g_identity_loss_metric = tf.keras.metrics.Mean(
'g_identity_loss', dtype=tf.float32)
self.g_loss_metric = tf.keras.metrics.Mean('g_loss',
dtype=tf.float32)
self.d_adv_loss_metric = tf.keras.metrics.Mean('d_adv_loss',
dtype=tf.float32)
self.d_loss_metric = tf.keras.metrics.Mean('d_loss',
dtype=tf.float32)
input_shape = [self.img_height, self.img_width, self.img_ch]
self.source_generator.build_summary(input_shape)
self.source_discriminator.build_summary(input_shape)
self.target_generator.build_summary(input_shape)
self.target_discriminator.build_summary(input_shape)
""" Count parameters """
params = self.source_generator.count_parameter() + self.target_generator.count_parameter() \
+ self.source_discriminator.count_parameter() + self.target_discriminator.count_parameter()
print("Total network parameters : ", format(params, ','))
""" Checkpoint """
self.ckpt = tf.train.Checkpoint(
source_generator=self.source_generator,
target_generator=self.target_generator,
source_discriminator=self.source_discriminator,
target_discriminator=self.target_discriminator,
g_optimizer=self.g_optimizer,
d_optimizer=self.d_optimizer)
self.manager = tf.train.CheckpointManager(self.ckpt,
self.checkpoint_dir,
max_to_keep=2)
self.start_iteration = 0
if self.manager.latest_checkpoint:
self.ckpt.restore(self.manager.latest_checkpoint)
self.start_iteration = int(
self.manager.latest_checkpoint.split('-')[-1])
print('Latest checkpoint restored!!')
print('start iteration : ', self.start_iteration)
else:
print('Not restoring from saved checkpoint')
else:
""" Test """
""" Network """
self.source_generator = Generator(self.ch,
self.n_res,
name='source_generator')
self.target_generator = Generator(self.ch,
self.n_res,
name='target_generator')
self.source_discriminator = Discriminator(
self.ch, self.n_dis, self.sn, name='source_discriminator')
self.target_discriminator = Discriminator(
self.ch, self.n_dis, self.sn, name='target_discriminator')
input_shape = [self.img_height, self.img_width, self.img_ch]
self.source_generator.build_summary(input_shape)
self.source_discriminator.build_summary(input_shape)
self.target_generator.build_summary(input_shape)
self.target_discriminator.build_summary(input_shape)
""" Count parameters """
params = self.source_generator.count_parameter() + self.target_generator.count_parameter() \
+ self.source_discriminator.count_parameter() + self.target_discriminator.count_parameter()
print("Total network parameters : ", format(params, ','))
""" Checkpoint """
self.ckpt = tf.train.Checkpoint(
source_generator=self.source_generator,
target_generator=self.target_generator,
source_discriminator=self.source_discriminator,
target_discriminator=self.target_discriminator)
self.manager = tf.train.CheckpointManager(self.ckpt,
self.checkpoint_dir,
max_to_keep=2)
if self.manager.latest_checkpoint:
self.ckpt.restore(
self.manager.latest_checkpoint).expect_partial()
print('Latest checkpoint restored!!')
else:
print('Not restoring from saved checkpoint')
def build_model(self):
""" Input Image"""
img_data_class = Image_data(self.img_height, self.img_width,
self.img_ch, self.dataset_path,
self.augment_flag)
train_class_id, train_captions, train_images, test_captions, test_images, idx_to_word, word_to_idx = img_data_class.preprocess(
)
self.vocab_size = len(idx_to_word)
self.idx_to_word = idx_to_word
self.word_to_idx = word_to_idx
"""
train_captions: (8855, 10, 18), test_captions: (2933, 10, 18)
train_images: (8855,), test_images: (2933,)
idx_to_word : 5450 5450
"""
if self.phase == 'train':
self.dataset_num = len(train_images)
img_and_caption = tf.data.Dataset.from_tensor_slices(
(train_images, train_captions, train_class_id))
gpu_device = '/gpu:0'
img_and_caption = img_and_caption.apply(
shuffle_and_repeat(self.dataset_num)).apply(
map_and_batch(img_data_class.image_processing,
batch_size=self.batch_size,
num_parallel_batches=16,
drop_remainder=True)).apply(
prefetch_to_device(gpu_device, None))
self.img_caption_iter = iter(img_and_caption)
# real_img_256, caption = iter(img_and_caption)
""" Network """
self.rnn_encoder = RnnEncoder(n_words=self.vocab_size,
embed_dim=self.embed_dim,
drop_rate=0.5,
n_hidden=128,
n_layer=1,
bidirectional=True,
rnn_type='lstm')
self.cnn_encoder = CnnEncoder(embed_dim=self.embed_dim)
self.ca_net = CA_NET(c_dim=self.z_dim)
self.generator = Generator(channels=self.g_dim)
self.discriminator = Discriminator(channels=self.d_dim,
embed_dim=self.embed_dim)
""" Optimizer """
self.g_optimizer = tf.keras.optimizers.Adam(
learning_rate=self.init_lr,
beta_1=0.5,
beta_2=0.999,
epsilon=1e-08)
d_64_optimizer = tf.keras.optimizers.Adam(
learning_rate=self.init_lr,
beta_1=0.5,
beta_2=0.999,
epsilon=1e-08)
d_128_optimizer = tf.keras.optimizers.Adam(
learning_rate=self.init_lr,
beta_1=0.5,
beta_2=0.999,
epsilon=1e-08)
d_256_optimizer = tf.keras.optimizers.Adam(
learning_rate=self.init_lr,
beta_1=0.5,
beta_2=0.999,
epsilon=1e-08)
self.d_optimizer = [
d_64_optimizer, d_128_optimizer, d_256_optimizer
]
self.embed_optimizer = tf.keras.optimizers.Adam(
learning_rate=self.init_lr,
beta_1=0.5,
beta_2=0.999,
epsilon=1e-08)
""" Checkpoint """
self.ckpt = tf.train.Checkpoint(
rnn_encoder=self.rnn_encoder,
cnn_encoder=self.cnn_encoder,
ca_net=self.ca_net,
generator=self.generator,
discriminator=self.discriminator,
g_optimizer=self.g_optimizer,
d_64_optimizer=d_64_optimizer,
d_128_optimizer=d_128_optimizer,
d_256_optimizer=d_256_optimizer,
embed_optimizer=self.embed_optimizer)
self.manager = tf.train.CheckpointManager(self.ckpt,
self.checkpoint_dir,
max_to_keep=2)
self.start_iteration = 0
if self.manager.latest_checkpoint:
self.ckpt.restore(self.manager.latest_checkpoint)
self.start_iteration = int(
self.manager.latest_checkpoint.split('-')[-1])
print('Latest checkpoint restored!!')
print('start iteration : ', self.start_iteration)
else:
print('Not restoring from saved checkpoint')
else:
""" Test """
self.dataset_num = len(test_captions)
gpu_device = '/gpu:0'
img_and_caption = tf.data.Dataset.from_tensor_slices(
(train_images, train_captions))
img_and_caption = img_and_caption.apply(
shuffle_and_repeat(self.dataset_num)).apply(
map_and_batch(img_data_class.image_processing,
batch_size=self.batch_size,
num_parallel_batches=16,
drop_remainder=True)).apply(
prefetch_to_device(gpu_device, None))
self.img_caption_iter = iter(img_and_caption)
""" Network """
self.rnn_encoder = RnnEncoder(n_words=self.vocab_size,
embed_dim=self.embed_dim,
drop_rate=0.5,
n_hidden=128,
n_layer=1,
bidirectional=True,
rnn_type='lstm')
self.cnn_encoder = CnnEncoder(embed_dim=self.embed_dim)
self.ca_net = CA_NET(c_dim=self.z_dim)
self.generator = Generator(channels=self.g_dim)
# self.discriminator = Discriminator(channels=self.d_dim, embed_dim=self.embed_dim)
# """ Optimizer """
# self.g_optimizer = tf.keras.optimizers.Adam(learning_rate=self.init_lr, beta_1=0.5, beta_2=0.999, epsilon=1e-08)
# d_64_optimizer = tf.keras.optimizers.Adam(learning_rate=self.init_lr, beta_1=0.5, beta_2=0.999, epsilon=1e-08)
# d_128_optimizer = tf.keras.optimizers.Adam(learning_rate=self.init_lr, beta_1=0.5, beta_2=0.999, epsilon=1e-08)
# d_256_optimizer = tf.keras.optimizers.Adam(learning_rate=self.init_lr, beta_1=0.5, beta_2=0.999, epsilon=1e-08)
# self.d_optimizer = [d_64_optimizer, d_128_optimizer, d_256_optimizer]
# self.embed_optimizer = tf.keras.optimizers.Adam(learning_rate=self.init_lr, beta_1=0.5, beta_2=0.999, epsilon=1e-08)
""" Checkpoint """
self.ckpt = tf.train.Checkpoint(rnn_encoder=self.rnn_encoder,
cnn_encoder=self.cnn_encoder,
ca_net=self.ca_net,
generator=self.generator)
# discriminator=self.discriminator,
# g_optimizer=self.g_optimizer,
# d_64_optimizer=d_64_optimizer,
# d_128_optimizer=d_128_optimizer,
# d_256_optimizer=d_256_optimizer,
# embed_optimizer=self.embed_optimizer)
self.manager = tf.train.CheckpointManager(self.ckpt,
self.checkpoint_dir,
max_to_keep=2)
self.start_iteration = 0
if self.manager.latest_checkpoint:
self.ckpt.restore(
self.manager.latest_checkpoint).expect_partial()
self.start_iteration = int(
self.manager.latest_checkpoint.split('-')[-1])
print('Latest checkpoint restored!!')
print('start iteration : ', self.start_iteration)
else:
print('Not restoring from saved checkpoint')
def minibatch(self,
dataset,
subset,
use_datasets,
cache_data,
shift_ratio=-1):
if shift_ratio < 0:
shift_ratio = self.shift_ratio
with tf.compat.v1.name_scope('batch_processing'):
# Build final results per split.
images = [[] for _ in range(self.num_splits)]
labels = [[] for _ in range(self.num_splits)]
if use_datasets:
glob_pattern = dataset.tf_record_pattern(subset)
file_names = gfile.Glob(glob_pattern)
if not file_names:
raise ValueError(
'Found no files in --data_dir matching: {}'.format(
glob_pattern))
ds = tf.data.TFRecordDataset.list_files(file_names)
ds = ds.apply(
#interleave_ops.parallel_interleave(
parallel_interleave( #
tf.data.TFRecordDataset, cycle_length=10))
if cache_data:
ds = ds.take(1).cache().repeat()
counter = tf.data.Dataset.range(self.batch_size)
counter = counter.repeat()
ds = tf.data.Dataset.zip((ds, counter))
ds = ds.prefetch(buffer_size=self.batch_size)
ds = ds.shuffle(buffer_size=10000)
ds = ds.repeat()
ds = ds.apply(
#batching.map_and_batch(
map_and_batch( ###
map_func=self.parse_and_preprocess,
batch_size=self.batch_size_per_split,
num_parallel_batches=self.num_splits))
ds = ds.prefetch(buffer_size=self.num_splits)
ds_iterator = tf.compat.v1.data.make_one_shot_iterator(ds)
for d in xrange(self.num_splits):
labels[d], images[d] = ds_iterator.get_next()
else:
record_input = data_flow_ops.RecordInput(
file_pattern=dataset.tf_record_pattern(subset),
seed=301,
parallelism=64,
buffer_size=10000,
batch_size=self.batch_size,
shift_ratio=shift_ratio,
name='record_input')
records = record_input.get_yield_op()
records = tf.split(records, self.batch_size, 0)
records = [tf.reshape(record, []) for record in records]
for idx in xrange(self.batch_size):
value = records[idx]
(label, image) = self.parse_and_preprocess(value, idx)
split_index = idx % self.num_splits
labels[split_index].append(label)
images[split_index].append(image)
for split_index in xrange(self.num_splits):
if not use_datasets:
images[split_index] = tf.parallel_stack(
images[split_index])
labels[split_index] = tf.concat(labels[split_index], 0)
images[split_index] = tf.cast(images[split_index], self.dtype)
depth = 3
images[split_index] = tf.reshape(images[split_index],
shape=[
self.batch_size_per_split,
self.height, self.width,
depth
])
labels[split_index] = tf.reshape(labels[split_index],
[self.batch_size_per_split])
return images, labels
def build_model(self):
if self.phase == 'train':
""" Input Image"""
img_class = Image_data(self.img_height, self.img_width, self.img_ch, self.dataset_path, self.augment_flag)
img_class.preprocess()
dataset_num = len(img_class.dataset)
print("Dataset number : ", dataset_num)
img_slice = tf.data.Dataset.from_tensor_slices(img_class.dataset)
gpu_device = '/gpu:0'
img_slice = img_slice. \
apply(shuffle_and_repeat(dataset_num)). \
apply(map_and_batch(img_class.image_processing, self.batch_size, num_parallel_batches=AUTOTUNE,
drop_remainder=True)). \
apply(prefetch_to_device(gpu_device, AUTOTUNE))
self.dataset_iter = iter(img_slice)
""" Network """
self.classifier = SubNetwork(channels=64, name='classifier')
""" Optimizer """
self.optimizer = tf.keras.optimizers.Adam(learning_rate=self.init_lr, beta_1=0.5, beta_2=0.999,
epsilon=1e-08)
""" Summary """
# mean metric
self.loss_metric = tf.keras.metrics.Mean('loss',
dtype=tf.float32) # In tensorboard, make a loss to smooth graph
# print summary
input_shape = [self.img_height, self.img_width, self.img_ch]
self.classifier.build_summary(input_shape)
""" Count parameters """
params = self.classifier.count_parameter()
print("Total network parameters : ", format(params, ','))
""" Checkpoint """
self.ckpt = tf.train.Checkpoint(classifier=self.classifier, optimizer=self.optimizer)
self.manager = tf.train.CheckpointManager(self.ckpt, self.checkpoint_dir, max_to_keep=2)
self.start_iteration = 0
if self.manager.latest_checkpoint:
self.ckpt.restore(self.manager.latest_checkpoint)
self.start_iteration = int(self.manager.latest_checkpoint.split('-')[-1])
print('Latest checkpoint restored!!')
print('start iteration : ', self.start_iteration)
else:
print('Not restoring from saved checkpoint')
else:
""" Test """
""" Network """
self.classifier = SubNetwork(channels=64, name='classifier')
""" Summary """
input_shape = [self.img_height, self.img_width, self.img_ch]
self.classifier.build_summary(input_shape)
""" Count parameters """
params = self.classifier.count_parameter()
print("Total network parameters : ", format(params, ','))
""" Checkpoint """
self.ckpt = tf.train.Checkpoint(classifier=self.classifier)
self.manager = tf.train.CheckpointManager(self.ckpt, self.checkpoint_dir, max_to_keep=2)
if self.manager.latest_checkpoint:
self.ckpt.restore(self.manager.latest_checkpoint).expect_partial()
print('Latest checkpoint restored!!')
else:
print('Not restoring from saved checkpoint')
def run_benchmark(self):
if self.args.data_location:
print("Inference with real data.")
else:
print("Inference with dummy data.")
global model, graph
with tf.Session().as_default() as sess:
with sess.graph.as_default() as graph:
if self.args.data_location:
self.build_data_sess()
else:
raise Exception("no data location provided")
total_iter = 1000
warmup_iter = 0
ttime = 0.0
print('total iteration is {0}'.format(str(total_iter)))
print('warm up iteration is {0}'.format(str(warmup_iter)))
total_samples = 0
if self.args.data_location:
tfrecord_paths = [self.args.data_location]
ds = tf.data.TFRecordDataset.list_files(tfrecord_paths)
ds = ds.apply(
parallel_interleave(tf.data.TFRecordDataset,
cycle_length=1,
block_length=1,
buffer_output_elements=10000,
prefetch_input_elements=10000))
ds = ds.prefetch(buffer_size=10000)
ds = ds.apply(
map_and_batch(map_func=parse_and_preprocess,
batch_size=self.args.batch_size,
num_parallel_batches=1,
num_parallel_calls=None))
ds = ds.prefetch(buffer_size=10000)
ds_iterator = tf.data.make_one_shot_iterator(ds)
for step in range(total_iter):
box, label, image_source, features = ds_iterator.get_next(
)
if features is None:
break
box = box.eval(session=sess)
label = label.eval(session=sess).flatten()
image_source = image_source.eval(
session=sess).flatten()
image_source = [
image_id if type(image_id) == 'str' else
image_id.decode('utf-8')
for image_id in image_source
]
label = [
x if type(x) == 'str' else x.decode('utf-8')
for x in label
]
start_time = time.time()
images = [
np.asarray(
PIL.Image.open(
os.path.join(self.args.imagesets_dir,
image_id)).convert('RGB'))
for image_id in image_source
]
images = self.preprocess_bounding_box_images(
images, bbox, image_source)
total_samples += images.shape[0]
arcface_features = self.arcface_model.predict(
np.vstack(images), verbose=1)
end_time = time.time()
duration = end_time - start_time
if (step + 1) % 10 == 0:
print('steps = {0}, {1} sec'.format(
str(step), str(duration)))
if step + 1 > warmup_iter:
ttime += duration
# if len(image_batches) == self.args.batch_size:
print('Batchsize: {0}'.format(str(
self.args.batch_size)))
print('Time spent per BATCH: {0:10.4f} ms'.format(
ttime / total_samples * 1000))
print('Total samples/sec: {0:10.4f} samples/s'.format(
total_samples * self.args.batch_size / ttime))
print('Total labeled samples: {0} person'.format(
np.where(np.array(label) == "person")[0].shape[0]))
def input_fn(params):
"""The actual input function."""
batch_size = params["batch_size"]
weights = tf.constant([
2.07500868, 1.03715683, 1.93284648, 1.21630542, 1.13895236,
1.35432071, 1.09689163, 1.6204935, 1.16378624, 1.18903464, 1.,
1.56280185, 1.35888667, 1.28946654, 1.37520308, 1.20126476,
1.1674391, 1.22790734, 1.10949764, 2.07500868, 1.49259834,
2.07500868
])
tf.logging.info("Using {} threads".format(num_cpu_threads))
# `cycle_length` is the number of parallel files that get read.
cycle_length = len(input_files)
# For training, we want a lot of parallel reading and shuffling.
# For eval, we want no shuffling and parallel reading doesn't matter.
if is_training:
tf.logging.info("There are {} files for training".format(
len(input_files)))
upsampling_factor = [
get_upsampling_factor(fn) for fn in input_files
]
tf.logging.info(upsampling_factor)
d = tf.data.Dataset.from_tensor_slices(
(tf.constant(input_files), tf.constant(upsampling_factor)))
d = d.shuffle(buffer_size=len(input_files))
# Repeat data in the file for unlimited number. This solves class imbalance problem.
def get_tfrecord_dataset(filename, upsampling_factor):
tfrecord_dataset = tf.data.TFRecordDataset(
filename, compression_type='GZIP')
if balance:
tfrecord_dataset = tfrecord_dataset.repeat(
tf.cast(upsampling_factor, dtype=tf.int64))
return tfrecord_dataset
# `sloppy` mode means that the interleaving is not exact. This adds
# even more randomness to the training pipeline.
d = d.apply(
parallel_interleave(
lambda filename, upsampling_factor: get_tfrecord_dataset(
filename, upsampling_factor),
sloppy=is_training,
cycle_length=cycle_length))
d = d.shuffle(buffer_size=200000)
d = d.repeat()
else:
d = tf.data.Dataset.from_tensor_slices(tf.constant(input_files))
# `sloppy` mode means that the interleaving is not exact. This adds
# even more randomness to the training pipeline.
d = d.apply(
parallel_interleave(lambda filename: tf.data.TFRecordDataset(
filename, compression_type='GZIP'),
sloppy=is_training,
cycle_length=cycle_length))
# Since we evaluate for a fixed number of steps we don't want to encounter
# out-of-range exceptions.
# d = d.repeat()
d = d.shuffle(buffer_size=10000)
# We must `drop_remainder` on training because the TPU requires fixed
# size dimensions. For eval, we assume we are evaluating on the CPU or GPU
# and we *don't* want to drop the remainder, otherwise we wont cover
# every sample.
d = d.apply(
map_and_batch(lambda record: _decode_record(
record, max_seq_length, max_predictions_per_seq, vocab_size,
is_training, weights),
batch_size=batch_size,
num_parallel_batches=num_cpu_threads,
drop_remainder=True)).prefetch(batch_size)
return d
def build_model(self):
""" Input Image"""
img_data_class = Image_data(self.img_height, self.img_width,
self.img_ch, self.dataset_path,
self.augment_flag)
train_captions, train_images, test_captions, test_images, idx_to_word, word_to_idx = img_data_class.preprocess(
)
"""
train_captions: (8855, 10, 18), test_captions: (2933, 10, 18)
train_images: (8855,), test_images: (2933,)
idx_to_word : 5450 5450
"""
if self.phase == 'train':
self.lr = tf.placeholder(tf.float32, name='learning_rate')
self.dataset_num = len(train_images)
img_and_caption = tf.data.Dataset.from_tensor_slices(
(train_images, train_captions))
gpu_device = '/gpu:0'
img_and_caption = img_and_caption.apply(
shuffle_and_repeat(self.dataset_num)).apply(
map_and_batch(img_data_class.image_processing,
batch_size=self.batch_size,
num_parallel_batches=16,
drop_remainder=True)).apply(
prefetch_to_device(gpu_device, None))
img_and_caption_iterator = img_and_caption.make_one_shot_iterator()
real_img_256, caption = img_and_caption_iterator.get_next()
target_sentence_index = tf.random_uniform(shape=[],
minval=0,
maxval=10,
dtype=tf.int32)
caption = tf.gather(caption, target_sentence_index, axis=1)
word_emb, sent_emb, mask = self.rnn_encoder(
caption,
n_words=len(idx_to_word),
embed_dim=self.embed_dim,
drop_rate=0.5,
n_hidden=128,
n_layers=1,
bidirectional=True,
rnn_type='lstm')
noise = tf.random_normal(shape=[self.batch_size, self.z_dim],
mean=0.0,
stddev=1.0)
fake_imgs, _, mu, logvar = self.generator(noise, sent_emb,
word_emb, mask)
real_img_64, real_img_128 = resize(real_img_256,
target_size=[64, 64]), resize(
real_img_256,
target_size=[128, 128])
fake_img_64, fake_img_128, fake_img_256 = fake_imgs[0], fake_imgs[
1], fake_imgs[2]
uncond_real_logits, cond_real_logits = self.discriminator(
[real_img_64, real_img_128, real_img_256], sent_emb)
_, cond_wrong_logits = self.discriminator([
real_img_64[:(self.batch_size - 1)],
real_img_128[:(self.batch_size - 1)],
real_img_256[:(self.batch_size - 1)]
], sent_emb[1:self.batch_size])
uncond_fake_logits, cond_fake_logits = self.discriminator(
[fake_img_64, fake_img_128, fake_img_256], sent_emb)
fake_img_256_feature = self.caption_cnn(fake_img_256)
fake_img_256_caption = self.caption_rnn(fake_img_256_feature,
caption,
n_words=len(idx_to_word),
embed_dim=self.embed_dim,
n_hidden=256 * 2,
n_layers=1)
self.g_adv_loss, self.d_adv_loss = 0, 0
for i in range(3):
self.g_adv_loss += self.adv_weight * (
generator_loss(self.gan_type, uncond_fake_logits[i]) +
generator_loss(self.gan_type, cond_fake_logits[i]))
uncond_real_loss, uncond_fake_loss = discriminator_loss(
self.gan_type, uncond_real_logits[i],
uncond_fake_logits[i])
cond_real_loss, cond_fake_loss = discriminator_loss(
self.gan_type, cond_real_logits[i], cond_fake_logits[i])
_, cond_wrong_loss = discriminator_loss(
self.gan_type, None, cond_wrong_logits[i])
self.d_adv_loss += self.adv_weight * (
((uncond_real_loss + cond_real_loss) / 2) +
(uncond_fake_loss + cond_fake_loss + cond_wrong_loss) / 3)
self.g_kl_loss = self.kl_weight * kl_loss(mu, logvar)
caption = tf.one_hot(caption, len(idx_to_word))
caption = tf.reshape(caption, [-1, len(idx_to_word)])
self.g_cap_loss = self.cap_weight * caption_loss(
fake_img_256_caption, caption)
self.g_loss = self.g_adv_loss + self.g_kl_loss + self.g_cap_loss
self.d_loss = self.d_adv_loss
self.real_img = real_img_256
self.fake_img = fake_img_256
""" Training """
t_vars = tf.trainable_variables()
G_vars = [var for var in t_vars if 'generator' in var.name]
D_vars = [var for var in t_vars if 'discriminator' in var.name]
self.g_optim = tf.train.AdamOptimizer(self.lr,
beta1=0.5,
beta2=0.999).minimize(
self.g_loss,
var_list=G_vars)
self.d_optim = tf.train.AdamOptimizer(self.lr,
beta1=0.5,
beta2=0.999).minimize(
self.d_loss,
var_list=D_vars)
"""" Summary """
self.summary_g_loss = tf.summary.scalar("g_loss", self.g_loss)
self.summary_d_loss = tf.summary.scalar("d_loss", self.d_loss)
self.summary_g_adv_loss = tf.summary.scalar(
"g_adv_loss", self.g_adv_loss)
self.summary_g_kl_loss = tf.summary.scalar("g_kl_loss",
self.g_kl_loss)
self.summary_g_cap_loss = tf.summary.scalar(
"g_cap_loss", self.g_cap_loss)
self.summary_d_adv_loss = tf.summary.scalar(
"d_adv_loss", self.d_adv_loss)
g_summary_list = [
self.summary_g_loss, self.summary_g_adv_loss,
self.summary_g_kl_loss, self.summary_g_cap_loss
]
d_summary_list = [self.summary_d_loss, self.summary_d_adv_loss]
self.summary_merge_g_loss = tf.summary.merge(g_summary_list)
self.summary_merge_d_loss = tf.summary.merge(d_summary_list)
else:
""" Test """
self.dataset_num = len(test_captions)
gpu_device = '/gpu:0'
img_and_caption = tf.data.Dataset.from_tensor_slices(
(test_images, test_captions))
img_and_caption = img_and_caption.apply(
shuffle_and_repeat(self.dataset_num)).apply(
map_and_batch(img_data_class.image_processing,
batch_size=self.batch_size,
num_parallel_batches=16,
drop_remainder=True)).apply(
prefetch_to_device(gpu_device, None))
img_and_caption_iterator = img_and_caption.make_one_shot_iterator()
real_img_256, caption = img_and_caption_iterator.get_next()
target_sentence_index = tf.random_uniform(shape=[],
minval=0,
maxval=10,
dtype=tf.int32)
caption = tf.gather(caption, target_sentence_index, axis=1)
word_emb, sent_emb, mask = self.rnn_encoder(
caption,
n_words=len(idx_to_word),
embed_dim=self.embed_dim,
drop_rate=0.5,
n_hidden=128,
n_layers=1,
bidirectional=True,
rnn_type='lstm',
is_training=False)
noise = tf.random_normal(shape=[self.batch_size, self.z_dim],
mean=0.0,
stddev=1.0)
fake_imgs, _, _, _ = self.generator(noise,
sent_emb,
word_emb,
mask,
is_training=False)
self.test_real_img = real_img_256
self.test_fake_img = fake_imgs[2]
def accuracy_check(self):
print("Inference for accuracy check.")
total_iter = COCO_NUM_VAL_IMAGES
fm = category_map
fm = dict(zip(list(fm.values()),list(fm.keys())))
print('total iteration is {0}'.format(str(total_iter)))
global model, graph
with tf.Session().as_default() as sess:
if self.args.data_location:
network = self.build_data_sess()
else:
raise Exception("no data location provided")
total_samples = 0
self.coord = tf.train.Coordinator()
tfrecord_paths = [self.args.data_location]
ds = tf.data.TFRecordDataset.list_files(tfrecord_paths)
ds = ds.apply(
parallel_interleave(
tf.data.TFRecordDataset, cycle_length=1, block_length=1,
buffer_output_elements=10000, prefetch_input_elements=10000))
ds = ds.prefetch(buffer_size=10000)
ds = ds.apply(
map_and_batch(
map_func=parse_and_preprocess,
batch_size=self.args.batch_size,
num_parallel_batches=1,
num_parallel_calls=None))
ds = ds.prefetch(buffer_size=10000)
ds_iterator = tf.data.make_one_shot_iterator(ds)
state = None
warmup_iter = 0
self.ground_truth_dicts = {}
self.detect_dicts = {}
self.total_iter = total_iter
self.image_id_gt_dict = {}
if self.args.data_location:
image_batches = []
for step in range(total_iter):
bbox, label, image_id, features = ds_iterator.get_next()
features, bbox, label, image_id = \
tuple(features.items()), bbox, label, image_id
if features is None:
break
bbox, label, image_id = sess.run([bbox, label, image_id])
# ground truth of bounding boxes from pascal voc
ground_truth = {}
ground_truth['boxes'] = np.asarray(bbox[0])
label_gt = [fm[l] if type(l) == 'str' else fm[l.decode('utf-8')] for l in label]
image_id_gt = [i if type(i) == 'str' else i.decode('utf-8') for i in image_id]
ground_truth['classes'] = np.array(label_gt*len(ground_truth['boxes']))
# saving all ground truth dictionaries
self.ground_truth_dicts[step] = ground_truth
self.image_id_gt_dict[step] = image_id_gt[0]
images = np.asarray(PIL.Image.open(os.path.join(self.args.imagesets_dir, image_id_gt[0])).convert('RGB'))
# face detection
boxes, confidences, classIds = self.face_detector(images, image_id_gt, self.args.batch_size)
images = np.asarray(PIL.Image.open(os.path.join(self.args.imagesets_dir, image_id_gt[0])).convert('L'))
boxes = self.filter_conventional_box_images(boxes)
try:
if len(boxes) > 0:
# ground truth bounding box
images_new = self.preprocess_bounding_box_images(images, bbox[0], image_id_gt, grayscale=True)
total_samples += images_new.shape[0]
images_new = self.augment_images(images_new, grayscale=True)
# detection for bounding boxes from pascal voc
detect = {}
label_det = label_gt
image_id_det = image_id_gt
# detected conventional bounding box
images_sync = self.preprocess_conventional_box_images(images, boxes, image_id_det, grayscale=True)
images_sync = self.augment_images(images_sync, grayscale=True)
if True:
images_sync = np.expand_dims(images_sync, 3)
images_sync = images_sync.transpose((0,3,1,2))
images = np.zeros((1,100,1,160,160))
l = 100 if len(images_sync) > 100 else len(images_sync)
images[:,0:l,:,:] = images_sync[0:l]
images_sync = images
detect['boxes'] = np.asarray(boxes)
detect['classes'] = np.asarray(label_det*len(detect['boxes']))
features = self.arcface_model.predict(network, images_sync, 0.5, verbose=1)
measures = self.measure(features)
if measures[0][0]:
detect['scores'] = np.broadcast_to(np.mean(np.asarray(measures[0][1])),len(detect['boxes']))
elif np.mean(measures[0][1]) > 0:
detect['scores'] = np.broadcast_to(np.mean(np.asarray(measures[0][1])),len(detect['boxes']))
else:
detect['scores'] = np.broadcast_to(np.asarray([0]),len(detect['boxes']))
self.detect_dicts[step] = detect
except Exception as e:
raise e
if (step + 1) % 10 == 0:
print('steps = {0} step'.format(str(step)))
if step == 70:
break
