def create_large_summary(real_image_input, gen_sample, suffix=''):
"""Creates a summary op for large summaries, i.e. the ones that don't consume relatively large amounts of disc space. Should not be made too frequently.
Parameters:
real_image_input: input tensor containing a batch of real images
gen_sample: (batch of) generated samples
suffix: (optional) suffix for the summary parameters. Can e.g. be _val or _EMA to indicate the summaries were compute on the validation dataset, or using the EMA model parameters.
Returns:
summary_large: a single op that will update all large summaries
"""
summary_large = []
with tf.name_scope('summaries'):
# Spread out 3D image as 2D grid, slicing in the z-dimension
real_image_grid = tf.transpose(real_image_input[0], (1, 2, 3, 0))
shape = real_image_grid.get_shape().as_list()
print(f'real_image_grid shape: {shape}')
grid_cols = int(2**np.floor(np.log(np.sqrt(shape[0])) / np.log(2)))
# If the image z-dimension isn't divisible by grid_rows, we need to pad
if (shape[0] % grid_cols) != 0:
# Initialize pad_list for numpy padding
pad_list = [[0, 0] for i in range(0, len(shape))]
# Compute number of slices we need to add to get to the next multiple of shape[0]
pad_nslices = grid_cols - (shape[0] % grid_cols)
pad_list[0] = [0, pad_nslices]
real_image_grid = tf.pad(real_image_grid,
tf.constant(pad_list),
"CONSTANT",
constant_values=0)
# Recompute shape, so that the number of grid_rows is adapted to that
shape = real_image_grid.get_shape().as_list()
grid_rows = int(np.ceil(shape[0] / grid_cols))
grid_shape = [grid_rows, grid_cols]
real_image_grid = image_grid(real_image_grid,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
fake_image_grid = tf.transpose(gen_sample[0], (1, 2, 3, 0))
# Use the same padding for the fake_image_grid
if (fake_image_grid.get_shape().as_list()[0] % grid_cols) != 0:
fake_image_grid = tf.pad(fake_image_grid,
tf.constant(pad_list),
"CONSTANT",
constant_values=0)
fake_image_grid = image_grid(fake_image_grid,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
fake_image_grid = tf.clip_by_value(fake_image_grid, -1, 2)
summary_large.append(
tf.summary.image('real_image' + suffix, real_image_grid))
summary_large.append(
tf.summary.image('fake_image' + suffix, fake_image_grid))
summary_large = tf.summary.merge(summary_large)
return summary_large
def find_tickets(state, ticket_settings, ticket_data):
prev_image = state.prev_image
curr_image = state.curr_image
background_image = state.background_image
curr_prev_diff = difference_mask(curr_image, prev_image, ticket_settings)
curr_back_diff = difference_mask(curr_image, background_image,
ticket_settings)
prev_back_diff = difference_mask(prev_image, background_image,
ticket_settings)
added_ticket_mask = cv2.bitwise_and(curr_prev_diff, curr_back_diff)
removed_ticket_mask = cv2.bitwise_and(curr_prev_diff, prev_back_diff)
colored_grid = utils.image_grid(curr_image, prev_image, background_image,
np.zeros_like(prev_image))
threshold_grid = utils.image_grid(curr_back_diff, prev_back_diff,
added_ticket_mask, removed_ticket_mask)
state.set_grids(colored_grid, threshold_grid)
added_tickets = changed_tickets(curr_image, added_ticket_mask,
state.classifier, ticket_settings,
ticket_data)
removed_tickets = changed_tickets(prev_image, removed_ticket_mask,
state.classifier, ticket_settings,
ticket_data)
return added_tickets, removed_tickets
layers.Conv2D(8, 3, padding="same", activation="relu"),
layers.Conv2D(16, 3, padding="same", activation="relu"),
layers.MaxPooling2D((2, 2)),
layers.Flatten(),
layers.Dense(64, activation="relu"),
layers.Dropout(0.1),
layers.Dense(10),
])
return model
model = get_model()
num_epochs = 1
loss_fn = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = keras.optimizers.Adam(lr=0.001)
acc_metric = keras.metrics.SparseCategoricalAccuracy()
writer = tf.summary.create_file_writer("logs/train/")
step = 0
for epoch in range(num_epochs):
for batch_idx, (x, y) in enumerate(ds_train):
figure = image_grid(x, y, class_names)
with writer.as_default():
tf.summary.image(
"Visualize Images",
plot_to_image(figure),
step=step,
)
step += 1
if (episode % 5) == 0:
with file_writer_rewards.as_default():
tf.summary.histogram('action_taken', acc_actions, step=episode)
print(
f"Episode {episode}: Reward {acc_reward} with action {action_meanings[action]} which was {'explored' if do_explore else 'greedy'}"
)
env.render()
# Wrap up
loss = history.history.get("loss", [0])[0]
time_end = np.round(time.time() - start_time, 2)
memory_usage = process.memory_info().rss
tmp = random.choice(experience_batch)
# print(tmp.shape)
episode_image = plot_to_image(
image_grid(tmp, env.unwrapped.get_action_meanings()))
print(f"Loss of episode {episode} is {loss} and took {time_end} seconds")
print(f"TOTAL REWARD: {np.sum(episode_rewards)}")
with file_writer_rewards.as_default():
tf.summary.scalar('episode_rewards',
np.sum(episode_rewards),
step=episode)
tf.summary.scalar('episode_loss', loss, step=episode)
tf.summary.scalar('episode_time_in_secs', time_end, step=episode)
tf.summary.scalar('episode_nr_frames', frame_cnt, step=episode)
tf.summary.scalar('episode_exploration_rate',
exploration_rate,
step=episode)
tf.summary.scalar('episode_mem_usage', memory_usage, step=episode)
tf.summary.scalar('episode_mem_usage_in_GB',
np.round(memory_usage / 1024 / 1024 / 1024),
def main(args, config):
if args.horovod:
verbose = hvd.rank() == 0
global_size = hvd.size()
# global_rank = hvd.rank()
local_rank = hvd.local_rank()
else:
verbose = True
global_size = 1
# global_rank = 0
local_rank = 0
timestamp = time.strftime("%Y-%m-%d_%H:%M:%S", time.gmtime())
logdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'runs',
args.architecture, timestamp)
if verbose:
writer = tf.summary.FileWriter(logdir=logdir)
print("Arguments passed:")
print(args)
print(f"Saving files to {logdir}")
else:
writer = None
final_shape = parse_tuple(args.final_shape)
image_channels = final_shape[0]
final_resolution = final_shape[-1]
num_phases = int(np.log2(final_resolution) - 1)
base_dim = num_filters(1, num_phases, size=args.network_size)
var_list = list()
global_step = 0
for phase in range(1, num_phases + 1):
tf.reset_default_graph()
# ------------------------------------------------------------------------------------------#
# DATASET
size = 2 * 2**phase
if args.dataset == 'imagenet':
dataset = imagenet_dataset(
args.dataset_path,
args.scratch_path,
size,
copy_files=local_rank == 0,
is_correct_phase=phase >= args.starting_phase,
gpu=args.gpu,
num_labels=1 if args.num_labels is None else args.num_labels)
else:
raise ValueError(f"Unknown dataset {args.dataset_path}")
# Get DataLoader
batch_size = max(1, args.base_batch_size // (2**(phase - 1)))
if phase >= args.starting_phase:
assert batch_size * global_size <= args.max_global_batch_size
if verbose:
print(
f"Using local batch size of {batch_size} and global batch size of {batch_size * global_size}"
)
if args.horovod:
dataset.shard(hvd.size(), hvd.rank())
dataset = dataset.batch(batch_size, drop_remainder=True)
dataset = dataset.repeat()
dataset = dataset.prefetch(AUTOTUNE)
dataset = dataset.make_one_shot_iterator()
data = dataset.get_next()
if len(data) == 1:
real_image_input = data
real_label = None
elif len(data) == 2:
real_image_input, real_label = data
else:
raise NotImplementedError()
real_image_input = tf.ensure_shape(
real_image_input, [batch_size, image_channels, size, size])
real_image_input = real_image_input + tf.random.normal(
tf.shape(real_image_input)) * .01
if real_label is not None:
real_label = tf.one_hot(real_label, depth=args.num_labels)
# ------------------------------------------------------------------------------------------#
# OPTIMIZERS
g_lr = args.g_lr
d_lr = args.d_lr
if args.horovod:
if args.g_scaling == 'sqrt':
g_lr = g_lr * np.sqrt(hvd.size())
elif args.g_scaling == 'linear':
g_lr = g_lr * hvd.size()
elif args.g_scaling == 'none':
pass
else:
raise ValueError(args.g_scaling)
if args.d_scaling == 'sqrt':
d_lr = d_lr * np.sqrt(hvd.size())
elif args.d_scaling == 'linear':
d_lr = d_lr * hvd.size()
elif args.d_scaling == 'none':
pass
else:
raise ValueError(args.d_scaling)
# d_lr = tf.Variable(d_lr, name='d_lr', dtype=tf.float32)
# g_lr = tf.Variable(g_lr, name='g_lr', dtype=tf.float32)
# # optimizer_gen = tf.train.AdamOptimizer(learning_rate=g_lr, beta1=args.beta1, beta2=args.beta2)
# # optimizer_disc = tf.train.AdamOptimizer(learning_rate=d_lr, beta1=args.beta1, beta2=args.beta2)
# # optimizer_gen = LAMB(learning_rate=g_lr, beta1=args.beta1, beta2=args.beta2)
# # optimizer_disc = LAMB(learning_rate=d_lr, beta1=args.beta1, beta2=args.beta2)
# # optimizer_gen = LARSOptimizer(learning_rate=g_lr, momentum=0, weight_decay=0)
# # optimizer_disc = LARSOptimizer(learning_rate=d_lr, momentum=0, weight_decay=0)
# # optimizer_gen = tf.train.RMSPropOptimizer(learning_rate=1e-3)
# # optimizer_disc = tf.train.RMSPropOptimizer(learning_rate=1e-3)
# # optimizer_gen = tf.train.GradientDescentOptimizer(learning_rate=1e-3)
# # optimizer_disc = tf.train.GradientDescentOptimizer(learning_rate=1e-3)
# # optimizer_gen = RAdamOptimizer(learning_rate=g_lr, beta1=args.beta1, beta2=args.beta2)
# # optimizer_disc = RAdamOptimizer(learning_rate=d_lr, beta1=args.beta1, beta2=args.beta2)
# lr_step = tf.Variable(0, name='step', dtype=tf.float32)
# update_step = lr_step.assign_add(1.0)
# with tf.control_dependencies([update_step]):
# update_g_lr = g_lr.assign(g_lr * args.g_annealing)
# update_d_lr = d_lr.assign(d_lr * args.d_annealing)
# if args.horovod:
# if args.use_adasum:
# # optimizer_gen = hvd.DistributedOptimizer(optimizer_gen, op=hvd.Adasum)
# optimizer_gen = hvd.DistributedOptimizer(optimizer_gen)
# optimizer_disc = hvd.DistributedOptimizer(optimizer_disc, op=hvd.Adasum)
# else:
# optimizer_gen = hvd.DistributedOptimizer(optimizer_gen)
# optimizer_disc = hvd.DistributedOptimizer(optimizer_disc)
# ------------------------------------------------------------------------------------------#
# NETWORKS
with tf.variable_scope('alpha'):
alpha = tf.Variable(1, name='alpha', dtype=tf.float32)
# Alpha init
init_alpha = alpha.assign(1)
# Specify alpha update op for mixing phase.
num_steps = args.mixing_nimg // (batch_size * global_size)
alpha_update = 1 / num_steps
# noinspection PyTypeChecker
update_alpha = alpha.assign(tf.maximum(alpha - alpha_update, 0))
base_shape = [image_channels, 4, 4]
if args.optim_strategy == 'simultaneous':
gen_loss, disc_loss, gp_loss, gen_sample = forward_simultaneous(
generator,
discriminator,
real_image_input,
args.latent_dim,
alpha,
phase,
num_phases,
base_dim,
base_shape,
args.activation,
args.leakiness,
args.network_size,
args.loss_fn,
args.gp_weight,
conditioning=real_label,
)
gen_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='generator')
disc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='discriminator')
with tf.variable_scope('optimizer_gen'):
# disc_loss = tf.Print(gen_loss, [gen_loss], 'g_loss')
optimizer_gen = create_optimizer(
gen_loss,
gen_vars,
1e-8, (args.mixing_nimg + args.stabilizing_nimg) /
(batch_size * global_size),
8,
hvd=hvd,
optimizer_type='adam')
with tf.variable_scope('optimizer_disc'):
# disc_loss = tf.Print(disc_loss, [disc_loss], 'd_loss')
optimizer_disc = create_optimizer(
disc_loss,
disc_vars,
1e-8, (args.mixing_nimg + args.stabilizing_nimg) /
(batch_size * global_size),
8,
hvd=hvd,
optimizer_type='lamb')
# if args.horovod:
# if args.use_adasum:
# # optimizer_gen = hvd.DistributedOptimizer(optimizer_gen, op=hvd.Adasum)
# optimizer_gen = hvd.DistributedOptimizer(optimizer_gen, sparse_as_dense=True)
# optimizer_disc = hvd.DistributedOptimizer(optimizer_disc, op=hvd.Adasum, sparse_as_dense=True)
# else:
# optimizer_gen = hvd.DistributedOptimizer(optimizer_gen, sparse_as_dense=True)
# optimizer_disc = hvd.DistributedOptimizer(optimizer_disc, sparse_as_dense=True)
# g_gradients = optimizer_gen.compute_gradients(gen_loss, var_list=gen_vars)
# d_gradients = optimizer_disc.compute_gradients(disc_loss, var_list=disc_vars)
# g_norms = tf.stack([tf.norm(grad) for grad, var in g_gradients if grad is not None])
# max_g_norm = tf.reduce_max(g_norms)
# d_norms = tf.stack([tf.norm(grad) for grad, var in d_gradients if grad is not None])
# max_d_norm = tf.reduce_max(d_norms)
# # g_clipped_grads = [(tf.clip_by_norm(grad, clip_norm=128), var) for grad, var in g_gradients]
# # train_gen = optimizer_gen.apply_gradients(g_clipped_grads)
# gs = t
# train_gen = optimizer_gen.apply_gradients(g_gradients)
# train_disc = optimizer_disc.apply_gradients(d_gradients)
# elif args.optim_strategy == 'alternate':
# disc_loss, gp_loss = forward_discriminator(
# generator,
# discriminator,
# real_image_input,
# args.latent_dim,
# alpha,
# phase,
# num_phases,
# base_dim,
# base_shape,
# args.activation,
# args.leakiness,
# args.network_size,
# args.loss_fn,
# args.gp_weight,
# conditioning=real_label
# )
# # disc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='discriminator')
# # d_gradients = optimizer_disc.compute_gradients(disc_loss, var_list=disc_vars)
# # d_norms = tf.stack([tf.norm(grad) for grad, var in d_gradients if grad is not None])
# # max_d_norm = tf.reduce_max(d_norms)
# # train_disc = optimizer_disc.apply_gradients(d_gradients)
# with tf.control_dependencies([train_disc]):
# gen_sample, gen_loss = forward_generator(
# generator,
# discriminator,
# real_image_input,
# args.latent_dim,
# alpha,
# phase,
# num_phases,
# base_dim,
# base_shape,
# args.activation,
# args.leakiness,
# args.network_size,
# args.loss_fn,
# is_reuse=True
# )
# gen_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='generator')
# g_gradients = optimizer_gen.compute_gradients(gen_loss, var_list=gen_vars)
# g_norms = tf.stack([tf.norm(grad) for grad, var in g_gradients if grad is not None])
# max_g_norm = tf.reduce_max(g_norms)
# train_gen = optimizer_gen.apply_gradients(g_gradients)
else:
raise ValueError("Unknown optim strategy ", args.optim_strategy)
if verbose:
print(f"Generator parameters: {count_parameters('generator')}")
print(
f"Discriminator parameters:: {count_parameters('discriminator')}"
)
# train_gen = optimizer_gen.minimize(gen_loss, var_list=gen_vars)
# train_disc = optimizer_disc.minimize(disc_loss, var_list=disc_vars)
ema = tf.train.ExponentialMovingAverage(decay=args.ema_beta)
ema_op = ema.apply(gen_vars)
# Transfer EMA values to original variables
ema_update_weights = tf.group(
[tf.assign(var, ema.average(var)) for var in gen_vars])
with tf.name_scope('summaries'):
# Summaries
tf.summary.scalar('d_loss', disc_loss)
tf.summary.scalar('g_loss', gen_loss)
tf.summary.scalar('gp', tf.reduce_mean(gp_loss))
# for g in g_gradients:
# tf.summary.histogram(f'grad_{g[1].name}', g[0])
# for g in d_gradients:
# tf.summary.histogram(f'grad_{g[1].name}', g[0])
# tf.summary.scalar('convergence', tf.reduce_mean(disc_real) - tf.reduce_mean(tf.reduce_mean(disc_fake_d)))
# tf.summary.scalar('max_g_grad_norm', max_g_norm)
# tf.summary.scalar('max_d_grad_norm', max_d_norm)
real_image_grid = tf.transpose(real_image_input,
(0, 2, 3, 1)) # D H W C -> B H W C
shape = real_image_grid.get_shape().as_list()
grid_cols = int(2**np.floor(np.log(np.sqrt(shape[0])) / np.log(2)))
grid_rows = shape[0] // grid_cols
grid_shape = [grid_rows, grid_cols]
real_image_grid = image_grid(real_image_grid,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
fake_image_grid = tf.transpose(gen_sample, (0, 2, 3, 1))
fake_image_grid = image_grid(fake_image_grid,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
fake_image_grid = tf.clip_by_value(fake_image_grid, -1, 1)
tf.summary.image('real_image', real_image_grid)
tf.summary.image('fake_image', fake_image_grid)
tf.summary.scalar('fake_image_min', tf.math.reduce_min(gen_sample))
tf.summary.scalar('fake_image_max', tf.math.reduce_max(gen_sample))
tf.summary.scalar('real_image_min',
tf.math.reduce_min(real_image_input[0]))
tf.summary.scalar('real_image_max',
tf.math.reduce_max(real_image_input[0]))
tf.summary.scalar('alpha', alpha)
tf.summary.scalar('g_lr', g_lr)
tf.summary.scalar('d_lr', d_lr)
merged_summaries = tf.summary.merge_all()
# Other ops
init_op = tf.global_variables_initializer()
assign_starting_alpha = alpha.assign(args.starting_alpha)
assign_zero = alpha.assign(0)
broadcast = hvd.broadcast_global_variables(0)
with tf.Session(config=config) as sess:
sess.run(init_op)
trainable_variable_names = [
v.name for v in tf.trainable_variables()
]
if var_list is not None and phase > args.starting_phase:
print("Restoring variables from:",
os.path.join(logdir, f'model_{phase - 1}'))
var_names = [v.name for v in var_list]
load_vars = [
sess.graph.get_tensor_by_name(n) for n in var_names
if n in trainable_variable_names
]
saver = tf.train.Saver(load_vars)
saver.restore(sess, os.path.join(logdir, f'model_{phase - 1}'))
elif var_list is not None and args.continue_path and phase == args.starting_phase:
print("Restoring variables from:", args.continue_path)
var_names = [v.name for v in var_list]
load_vars = [
sess.graph.get_tensor_by_name(n) for n in var_names
if n in trainable_variable_names
]
saver = tf.train.Saver(load_vars)
saver.restore(sess, os.path.join(args.continue_path))
else:
if verbose:
print("Not restoring variables.")
print("Variable List Length:", len(var_list))
var_list = gen_vars + disc_vars
if phase < args.starting_phase:
continue
if phase == args.starting_phase:
sess.run(assign_starting_alpha)
else:
sess.run(init_alpha)
if verbose:
print(f"Begin mixing epochs in phase {phase}")
if args.horovod:
sess.run(broadcast)
local_step = 0
# take_first_snapshot = True
while True:
start = time.time()
if local_step % 128 == 0 and local_step > 1:
if args.horovod:
sess.run(broadcast)
saver = tf.train.Saver(var_list)
if verbose:
saver.save(
sess,
os.path.join(logdir,
f'model_{phase}_ckpt_{global_step}'))
# _, _, summary, d_loss, g_loss = sess.run(
# [train_gen, train_disc, merged_summaries,
# disc_loss, gen_loss])
_, _, summary, d_loss, g_loss = sess.run([
optimizer_gen, optimizer_disc, merged_summaries, disc_loss,
gen_loss
])
global_step += batch_size * global_size
local_step += 1
end = time.time()
img_s = global_size * batch_size / (end - start)
if verbose:
writer.add_summary(summary, global_step)
writer.add_summary(
tf.Summary(value=[
tf.Summary.Value(tag='img_s', simple_value=img_s)
]), global_step)
memory_percentage = psutil.Process(
os.getpid()).memory_percent()
writer.add_summary(
tf.Summary(value=[
tf.Summary.Value(tag='memory_percentage',
simple_value=memory_percentage)
]), global_step)
print(f"Step {global_step:09} \t"
f"img/s {img_s:.2f} \t "
f"d_loss {d_loss:.4f} \t "
f"g_loss {g_loss:.4f} \t "
f"memory {memory_percentage:.4f} % \t"
f"alpha {alpha.eval():.2f}")
# if take_first_snapshot:
# import tracemalloc
# tracemalloc.start()
# snapshot_first = tracemalloc.take_snapshot()
# take_first_snapshot = False
# snapshot = tracemalloc.take_snapshot()
# top_stats = snapshot.compare_to(snapshot_first, 'lineno')
# print("[ Top 10 differences ]")
# for stat in top_stats[:10]:
# print(stat)
# snapshot_prev = snapshot
if global_step >= ((phase - args.starting_phase) *
(args.mixing_nimg + args.stabilizing_nimg) +
args.mixing_nimg):
break
sess.run(update_alpha)
sess.run(ema_op)
# sess.run(update_d_lr)
# sess.run(update_g_lr)
assert alpha.eval() >= 0
if verbose:
writer.flush()
if verbose:
print(f"Begin stabilizing epochs in phase {phase}")
sess.run(assign_zero)
while True:
start = time.time()
assert alpha.eval() == 0
if local_step % 128 == 0 and local_step > 0:
if args.horovod:
sess.run(broadcast)
saver = tf.train.Saver(var_list)
if verbose:
saver.save(
sess,
os.path.join(logdir,
f'model_{phase}_ckpt_{global_step}'))
# _, _, summary, d_loss, g_loss = sess.run(
# [train_gen, train_disc, merged_summaries,
# disc_loss, gen_loss])
_, _, summary, d_loss, g_loss = sess.run([
optimizer_gen, optimizer_disc, merged_summaries, disc_loss,
gen_loss
])
global_step += batch_size * global_size
local_step += 1
end = time.time()
img_s = global_size * batch_size / (end - start)
if verbose:
writer.add_summary(
tf.Summary(value=[
tf.Summary.Value(tag='img_s', simple_value=img_s)
]), global_step)
writer.add_summary(summary, global_step)
memory_percentage = psutil.Process(
os.getpid()).memory_percent()
writer.add_summary(
tf.Summary(value=[
tf.Summary.Value(tag='memory_percentage',
simple_value=memory_percentage)
]), global_step)
print(f"Step {global_step:09} \t"
f"img/s {img_s:.2f} \t "
f"d_loss {d_loss:.4f} \t "
f"g_loss {g_loss:.4f} \t "
f"memory {memory_percentage:.4f} % \t"
f"alpha {alpha.eval():.2f}")
sess.run(ema_op)
if verbose:
writer.flush()
if global_step >= (phase - args.starting_phase + 1) * (
args.stabilizing_nimg + args.mixing_nimg):
# if verbose:
# run_metadata = tf.RunMetadata()
# opts = tf.profiler.ProfileOptionBuilder.float_operation()
# g = tf.get_default_graph()
# flops = tf.profiler.profile(g, run_meta=run_metadata, cmd='op', options=opts)
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag='graph_flops',
# simple_value=flops.total_float_ops)]),
# global_step)
#
# # Print memory info.
# try:
# print(nvgpu.gpu_info())
# except subprocess.CalledProcessError:
# pid = os.getpid()
# py = psutil.Process(pid)
# print(f"CPU Percent: {py.cpu_percent()}")
# print(f"Memory info: {py.memory_info()}")
break
# # Calculate metrics.
# calc_swds: bool = size >= 16
# calc_ssims: bool = min(npy_data.shape[1:]) >= 16
#
# if args.calc_metrics:
# fids_local = []
# swds_local = []
# psnrs_local = []
# mses_local = []
# nrmses_local = []
# ssims_local = []
#
# counter = 0
# while True:
# if args.horovod:
# start_loc = counter + hvd.rank() * batch_size
# else:
# start_loc = 0
# real_batch = np.stack([npy_data[i] for i in range(start_loc, start_loc + batch_size)])
# real_batch = real_batch.astype(np.int16) - 1024
# fake_batch = sess.run(gen_sample).astype(np.float32)
#
# # Turn fake batch into HUs and clip to training range.
# fake_batch = (np.clip(fake_batch, -1, 2) * 1024).astype(np.int16)
#
# if verbose:
# print('real min, max', real_batch.min(), real_batch.max())
# print('fake min, max', fake_batch.min(), fake_batch.max())
#
# fids_local.append(calculate_fid_given_batch_volumes(real_batch, fake_batch, sess))
#
# if calc_swds:
# swds = get_swd_for_volumes(real_batch, fake_batch)
# swds_local.append(swds)
#
# psnr = get_psnr(real_batch, fake_batch)
# if calc_ssims:
# ssim = get_ssim(real_batch, fake_batch)
# ssims_local.append(ssim)
# mse = get_mean_squared_error(real_batch, fake_batch)
# nrmse = get_normalized_root_mse(real_batch, fake_batch)
#
# psnrs_local.append(psnr)
# mses_local.append(mse)
# nrmses_local.append(nrmse)
#
# if args.horovod:
# counter = counter + global_size * batch_size
# else:
# counter += batch_size
#
# if counter >= args.num_metric_samples:
# break
#
# fid_local = np.mean(fids_local)
# psnr_local = np.mean(psnrs_local)
# ssim_local = np.mean(ssims_local)
# mse_local = np.mean(mses_local)
# nrmse_local = np.mean(nrmses_local)
#
# if args.horovod:
# fid = MPI.COMM_WORLD.allreduce(fid_local, op=MPI.SUM) / hvd.size()
# psnr = MPI.COMM_WORLD.allreduce(psnr_local, op=MPI.SUM) / hvd.size()
# mse = MPI.COMM_WORLD.allreduce(mse_local, op=MPI.SUM) / hvd.size()
# nrmse = MPI.COMM_WORLD.allreduce(nrmse_local, op=MPI.SUM) / hvd.size()
# if calc_ssims:
# ssim = MPI.COMM_WORLD.allreduce(ssim_local, op=MPI.SUM) / hvd.size()
# else:
# fid = fid_local
# psnr = psnr_local
# ssim = ssim_local
# mse = mse_local
# nrmse = nrmse_local
#
# if calc_swds:
# swds_local = np.array(swds_local)
# # Average over batches
# swds_local = swds_local.mean(axis=0)
# if args.horovod:
# swds = MPI.COMM_WORLD.allreduce(swds_local, op=MPI.SUM) / hvd.size()
# else:
# swds = swds_local
#
# if verbose:
# print(f"FID: {fid:.4f}")
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag='fid',
# simple_value=fid)]),
# global_step)
#
# print(f"PSNR: {psnr:.4f}")
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag='psnr',
# simple_value=psnr)]),
# global_step)
#
# print(f"MSE: {mse:.4f}")
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag='mse',
# simple_value=mse)]),
# global_step)
#
# print(f"Normalized Root MSE: {nrmse:.4f}")
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag='nrmse',
# simple_value=nrmse)]),
# global_step)
#
# if calc_swds:
# print(f"SWDS: {swds}")
# for i in range(len(swds))[:-1]:
# lod = 16 * 2 ** i
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag=f'swd_{lod}',
# simple_value=swds[
# i])]),
# global_step)
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag=f'swd_mean',
# simple_value=swds[
# -1])]), global_step)
# if calc_ssims:
# print(f"SSIM: {ssim}")
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag=f'ssim',
# simple_value=ssim)]), global_step)
if verbose:
print("\n\n\n End of phase.")
# Save Session.
sess.run(ema_update_weights)
saver = tf.train.Saver(var_list)
saver.save(sess, os.path.join(logdir, f'model_{phase}'))
if args.ending_phase:
if phase == args.ending_phase:
print("Reached final phase, breaking.")
break
with open("data/api_key.txt",'r') as f:
api_key = f.readline()
logger = comet_ml.Experiment(
api_key=api_key,
project_name="sim_real",
auto_metric_logging=True,
auto_param_logging=True,
)
if args.mixed_precision:
print("Applied: Mixed Precision")
tf.keras.mixed_precision.set_global_policy("mixed_float16")
train_ds, test_ds = get_dataset(args)
grid = image_grid(next(iter(train_ds))[0])[0]
logger.log_image(grid.numpy())
model = get_model(args)
criterion = get_criterion(args)
optimizer = get_optimizer(args)
lr_scheduler = get_lr_scheduler(args)
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', mode='max', patience=args.patience, restore_best_weights=True)
experiment_name = get_experiment_name(args)
logger.set_name(experiment_name)
logger.log_parameters(vars(args))
with logger.train():
filename =f'{args.model_name}.hdf5'
checkpoint = tf.keras.callbacks.ModelCheckpoint(filename, monitor='val_accuracy', mode='max', save_best_only=True, verbose=True)
model.compile(loss=criterion, optimizer=optimizer, metrics=['accuracy'])
if args.dry_run:
def main(args, config):
if args.horovod:
verbose = hvd.rank() == 0
local_rank = hvd.local_rank()
else:
verbose = True
local_rank = 0
global_batch_size = args.batch_size * hvd.size(
) if args.horovod else args.batch_size
timestamp = time.strftime("%Y-%m-%d_%H:%M:%S", time.gmtime())
logdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'runs',
timestamp)
global_step = 0
tf.reset_default_graph()
# ------------------------------------------------------------------------------------------#
# DATASET
data_path = os.path.join(args.dataset_root,
f'{args.image_size}x{args.image_size}/')
# retrieve dataset
npy_data = NumpyPathDataset(data_path,
args.scratch_path,
copy_files=local_rank == 0,
is_correct_phase=True)
dataset = tf.data.Dataset.from_tensor_slices(npy_data.scratch_files)
if args.horovod:
dataset.shard(hvd.size(), hvd.rank())
if args.data_format == "NCDHW":
current_shape = [
args.batch_size, args.image_channels, args.image_size // 4,
args.image_size, args.image_size
]
else:
current_shape = [
args.batch_size, args.image_size // 4, args.image_size,
args.image_size, args.image_channels
]
real_image_input = tf.placeholder(shape=current_shape, dtype=tf.float32)
# ------------------ NOISE ----------------
rand_batch1 = np.random.rand(*real_image_input.shape) * 0.5
noise_black_patches1 = rand_batch1.copy()
# x_input = image_input + tf.random.normal(shape=image_input.shape) * args.noise_strength
# x_input = image_input + tf.random.gamma(shape=x_input.shape, alpha=0.05)
# x_input = x_input + tf.random.uniform(shape=x_input.shape) * args.noise_strength
# x_input = x_input + tf.random.poisson(lam=0.5, shape=x_input.shape)
#add box_sampler noise which mimics conebeam noise
for i in range(real_image_input.shape[0]):
for _ in range(100):
arr_slices = uniform_box_sampler(noise_black_patches1,
min_width=(1, 1, 1, 3, 3),
max_width=(1, 1, 3, 6, 6))[0]
noise_black_patches1[arr_slices] = 0
x_input = real_image_input + noise_black_patches1
y = real_image_input
# ------------------ NETWORK ----------------
prediction = forward(x_input, args)
# ------------------ OPTIM -----------------
if args.loss_fn is "mean_squared_error":
loss = tf.losses.mean_squared_error(labels=y, predictions=prediction)
else:
assert args.loss_fn != "mean_squared_error", "Choose one of the available args.loss_fn"
lr_scaler = hvd.size() if args.horovod else 1
optimizer = tf.train.AdamOptimizer(args.learning_rate * lr_scaler)
if args.horovod:
optimizer = hvd.DistributedOptimizer(optimizer)
train_step = optimizer.minimize(loss)
# ------------- SUMMARIES -------------
if args.data_format == "NCDHW":
train_input = tf.transpose(x_input[0], (1, 2, 3, 0))
prediction_input = tf.transpose(prediction[0], (1, 2, 3, 0))
real_input = tf.transpose(y[0], (1, 2, 3, 0))
else:
train_input = tf.transpose(x_input[0], (0, 1, 2, 3))
prediction_input = tf.transpose(prediction[0], (0, 1, 2, 3))
real_input = tf.transpose(y[0], (0, 1, 2, 3))
prediction_input = tf.clip_by_value(prediction_input,
clip_value_min=args.clip_value_min,
clip_value_max=args.clip_value_max)
#transform images into grid
shape = train_input.get_shape().as_list()
image_shape = shape[1:3]
print(shape)
print(image_shape)
grid_cols = int(2**np.floor(np.log(np.sqrt(shape[0])) / np.log(2)))
grid_rows = shape[0] // grid_cols
grid_shape = [grid_rows, grid_cols]
train_input = image_grid(train_input,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
shape = prediction_input.get_shape().as_list()
grid_cols = int(2**np.floor(np.log(np.sqrt(shape[0])) / np.log(2)))
grid_rows = shape[0] // grid_cols
grid_shape = [grid_rows, grid_cols]
prediction_input = image_grid(prediction_input,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
shape = real_input.get_shape().as_list()
grid_cols = int(2**np.floor(np.log(np.sqrt(shape[0])) / np.log(2)))
grid_rows = shape[0] // grid_cols
grid_shape = [grid_rows, grid_cols]
real_input = image_grid(real_input,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
with tf.variable_scope("train_summaries"):
train_loss = tf.summary.scalar('train_loss', loss)
train_imageNoise = tf.summary.image('train_imageNoise', train_input)
train_imageRemake = tf.summary.image('train_imageRemake',
prediction_input)
train_imageReal = tf.summary.image('train_imageReal', real_input)
image_summary_train = tf.summary.merge(
[train_loss, train_imageReal, train_imageRemake, train_imageNoise])
with tf.variable_scope("test_summaries"):
test_loss = tf.summary.scalar('test_loss', loss)
test_imageNoise = tf.summary.image('test_imageNoise', train_input)
test_imageRemake = tf.summary.image('test_imageRemake',
prediction_input)
test_imageReal = tf.summary.image('test_imageReal', real_input)
image_summary_test = tf.summary.merge(
[test_loss, test_imageNoise, test_imageRemake, test_imageReal])
# -------------- SESSION -------------
with tf.Session(config=config) as sess:
sess.run(tf.initialize_all_variables())
if verbose:
writer = tf.summary.FileWriter(logdir=logdir,
graph=sess.graph,
session=sess)
#calculate percentage testset and trainingset
train_size = int(len(npy_data) * args.train_size)
test_size = int(len(npy_data) * (1 - args.train_size) + 1)
num_train_steps = train_size // global_batch_size
num_test_steps = test_size // global_batch_size
for epoch in range(args.epochs):
epoch_loss_train = 0
epoch_loss_test = 0
# TRAINING
for i in range(num_train_steps):
#prepare trainingbatch
batch_loc = np.random.randint(num_test_steps,
len(npy_data) - args.batch_size)
batch_paths = npy_data[batch_loc:batch_loc + args.batch_size]
batch = np.stack(np.load(path) for path in batch_paths)
batch = batch[:, np.newaxis, ...].astype(np.float32) / 1024 - 1
if args.data_format == "NDHWC":
batch = np.transpose(batch, (0, 2, 3, 4, 1))
_, summary, c = sess.run(
[train_step, image_summary_train, loss],
feed_dict={real_image_input: batch})
if i % args.logging_interval == 0 and verbose:
global_step = (epoch * num_train_steps *
global_batch_size) + i * global_batch_size
writer.add_summary(summary, global_step)
writer.flush()
epoch_loss_train += c
# TESTING
for i in range(num_test_steps):
#prepare testbatch
batch_loc = np.random.randint(0,
num_test_steps - args.batch_size)
batch_paths = npy_data[batch_loc:batch_loc + args.batch_size]
batch = np.stack(np.load(path) for path in batch_paths)
batch = batch[:, np.newaxis, ...].astype(np.float32) / 1024 - 1
if args.data_format == "NDHWC":
batch = np.transpose(batch, (0, 2, 3, 4, 1))
c = sess.run(loss, feed_dict={real_image_input: batch})
if i % args.logging_interval == 0 and verbose:
epoch_loss_test += c
if verbose:
# writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag='loss_test', simple_value=epoch_loss_test / num_test_steps)]), global_step)
test_image_summary = sess.run(
image_summary_test, feed_dict={real_image_input: batch})
writer.add_summary(test_image_summary, global_step)
writer.flush()
if verbose:
print(f'Epoch [{epoch}/{args.epochs}]\t'
f'Train Loss: {epoch_loss_train / num_train_steps}\t'
f'Test Loss: {epoch_loss_test / num_test_steps}\t')
def doc_scan_pipeline(input=PATH, output="./img/scanned_doc.jpg"):
img = cv2.imread(input)
# 0. Convert given image from BGR to RGB format
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
h, w, _ = img.shape
img = cv2.resize(img, (width, height))
# 1. Convert to grayscale
img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# 2. Add Gaussian blur
img_blur = cv2.GaussianBlur(img_gray, (5, 5), 1)
# 3. Add Canny edge detection
img_threshold = cv2.Canny(img_blur, 100, 200, L2gradient=True)
# 3.1 Apply dilation
kernel = np.ones((3, 3))
img_threshold = cv2.dilate(img_threshold, kernel, iterations=2)
# 4. Find all the contours
img_contours = img.copy()
img_big_contour = img.copy()
contours, hierarchy = cv2.findContours(img_threshold, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(image=img_contours,
contours=contours,
contourIdx=-1,
color=(0, 255, 0),
thickness=5)
# 5. Find the biggest contour
biggest, maxArea = biggest_contour(contours)
biggest = reorder(biggest)
cv2.drawContours(image=img_big_contour,
contours=biggest,
contourIdx=-1,
color=(0, 255, 0),
thickness=10)
# 5.1 Draw a rectangle, i.e., 4 lines connecting the 4 dots corresponding to the largest contour
img_big_contour = draw_rectangle(img_big_contour, biggest, thickness=2)
pts1 = np.float32(biggest)
pts2 = np.float32([[0, 0], [width, 0], [0, height], [width, height]])
# 6. Image Warp
# 6.1 Calculate a 3x3 perspective transform matrix
matrix = cv2.getPerspectiveTransform(pts1, pts2)
# 6.2 Apply the perspective matrix to the image
img_warp_coloured = cv2.warpPerspective(img, matrix, (width, height))
# 7. Adaptive thresholding
img_warp_gray = cv2.cvtColor(img_warp_coloured, cv2.COLOR_BGR2GRAY)
img_adaptive_th = cv2.adaptiveThreshold(img_warp_gray, 255, 1,
cv2.THRESH_BINARY, 5, 2)
# 7.1 Apply median blurring to remove tiny speckles of noise
img_adaptive_th = cv2.medianBlur(img_adaptive_th, 3)
# Save the document to disk
cv2.imwrite(output, img_adaptive_th)
# Add labels to each image
img = draw_text(img, "Original")
img_gray = draw_text(img_gray, "Grayscale")
img_blur = draw_text(img_blur, "Gaussian Blur", pos=(int(width / 4), 50))
img_threshold = draw_text(img_threshold,
"Canny Edge",
pos=(int(width / 4), 50))
img_contours = draw_text(img_contours, "Contours")
img_big_contour = draw_text(img_big_contour,
"Largest Contour",
pos=(int(width / 7), 50))
img_warp_coloured = draw_text(img_warp_coloured,
"Warp",
pos=(int(width / 3), 50))
img_adaptive_th = draw_text(img_adaptive_th,
"Adaptive Thresholding",
pos=(int(width / 7), 50),
font_scale=2,
font_thickness=6)
blank_img = np.zeros((height, width, 3), dtype=np.uint8)
image_list = [
img, img_gray, img_blur, img_threshold, img_contours, img_big_contour,
img_warp_coloured, img_adaptive_th
]
# Combine the images into a grid
# image_grid returns PIL image, np.asarray() can be used to convert it back to cv2 compatible format
grid = np.asarray(image_grid(image_list, width, height))
next_q = set_of_batch_rewards + (discount_rate *
tf.reduce_max(next_q_values, axis=1))
history = approximator_model.fit(
[set_of_batch_initial_states, set_of_batch_actions],
next_q,
verbose=1,
callbacks=[tensorflow_callback])
# Wrap up
loss = history.history.get("loss", [0])[0]
time_end = np.round(time.time() - start_time, 2)
memory_usage = process.memory_info().rss
tmp = random.choice(experience_batch)
# print(tmp.shape)
episode_image = plot_to_image(image_grid(tmp, action_meanings))
print(f"Current memory consumption is {memory_usage}")
print(f"Loss of episode {episode} is {loss} and took {time_end} seconds")
print(f"TOTAL REWARD: {np.sum(episode_rewards)}")
with file_writer_rewards.as_default():
tf.summary.scalar('episode_rewards',
np.sum(episode_rewards),
step=episode)
tf.summary.scalar('episode_loss', loss, step=episode)
tf.summary.scalar('episode_time_in_secs', time_end, step=episode)
tf.summary.scalar('episode_nr_frames', frame_cnt, step=episode)
tf.summary.scalar('episode_exploration_rate',
exploration_rate,
step=episode)
tf.summary.scalar('episode_mem_usage', memory_usage, step=episode)
def main(args, config):
phase = args.phase
if args.horovod:
verbose = hvd.rank() == 0
global_size = hvd.size()
global_rank = hvd.rank()
local_rank = hvd.local_rank()
else:
verbose = True
global_size = 1
global_rank = 0
local_rank = 0
if verbose:
timestamp = time.strftime("%Y-%m-%d_%H:%M", time.gmtime())
logdir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'generated_samples', timestamp)
os.makedirs(logdir)
else:
logdir = None
if args.horovod:
logdir = MPI.COMM_WORLD.bcast(logdir, root=0)
if verbose:
print("Arguments passed:")
print(args)
print(f"Saving files to {logdir}")
tf.reset_default_graph()
# Get Dataset.
final_shape = parse_tuple(args.final_shape)
final_resolution = final_shape[-1]
num_phases = int(np.log2(final_resolution) - 1)
size = 2 * 2**phase
data_path = os.path.join(args.dataset_path, f'{size}x{size}/')
npy_data = NumpyPathDataset(data_path,
None,
copy_files=False,
is_correct_phase=False)
dataset = tf.data.Dataset.from_tensor_slices(npy_data.scratch_files)
batch_size = 1
if args.horovod:
dataset.shard(hvd.size(), hvd.rank())
def load(x):
x = np.load(x.numpy().decode('utf-8'))[np.newaxis, ...]
return x
# Lay out the graph.
dataset = dataset.shuffle(len(npy_data))
dataset = dataset.map(
lambda x: tf.py_function(func=load, inp=[x], Tout=tf.uint16),
num_parallel_calls=AUTOTUNE)
dataset = dataset.map(lambda x: tf.cast(x, tf.float32) / 1024 - 1,
num_parallel_calls=AUTOTUNE)
dataset = dataset.batch(batch_size, drop_remainder=True)
dataset = dataset.repeat()
dataset = dataset.prefetch(AUTOTUNE)
dataset = dataset.make_one_shot_iterator()
real_image_input = dataset.get_next()
real_image_input = tf.ensure_shape(real_image_input,
[batch_size] + list(npy_data.shape))
# real_image_input = real_image_input + tf.random.normal(tf.shape(real_image_input)) * .01
with tf.variable_scope('alpha'):
alpha = tf.Variable(0, name='alpha', dtype=tf.float32)
zdim_base = max(1, final_shape[1] // (2**(num_phases - 1)))
base_shape = (1, zdim_base, 4, 4)
noise_input_d = tf.random.normal(
shape=[tf.shape(real_image_input)[0], args.latent_dim])
gen_sample_d = generator(noise_input_d,
alpha,
phase,
num_phases,
args.base_dim,
base_shape,
activation=args.activation,
param=args.leakiness)
if verbose:
print(f"Generator parameters: {count_parameters('generator')}")
gen_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='generator')
disc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='discriminator')
real_image_grid = tf.transpose(real_image_input[0], (1, 2, 3, 0))
shape = real_image_grid.get_shape().as_list()
grid_cols = int(2**np.floor(np.log(np.sqrt(shape[0])) / np.log(2)))
grid_rows = shape[0] // grid_cols
grid_shape = [grid_rows, grid_cols]
real_image_grid = image_grid(real_image_grid,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
fake_image_grid = tf.transpose(gen_sample_d[0], (1, 2, 3, 0))
fake_image_grid = image_grid(fake_image_grid,
grid_shape,
image_shape=shape[1:3],
num_channels=shape[-1])
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
trainable_variable_names = [v.name for v in tf.trainable_variables()]
var_list = gen_vars + disc_vars
var_names = [v.name for v in var_list]
load_vars = [
sess.graph.get_tensor_by_name(n) for n in var_names
if n in trainable_variable_names
]
saver = tf.train.Saver(load_vars)
saver.restore(sess, os.path.join(args.model_path))
if args.horovod:
sess.run(hvd.broadcast_global_variables(0))
num_samples = args.num_samples // global_size
calc_swds: bool = size >= 16
calc_ssims: bool = min(npy_data.shape[1:]) >= 16
fids_local = []
swds_local = []
psnrs_local = []
mses_local = []
nrmses_local = []
ssims_local = []
for i in tqdm(range(num_samples)):
ix = (global_rank + i * global_size)
real_batch, fake_batch, grid_real, grid_fake = sess.run([
real_image_input, gen_sample_d, real_image_grid,
fake_image_grid
])
# fake_batch = sess.run(real_image_input)
grid_real = np.squeeze(grid_real)
grid_fake = np.squeeze(grid_fake)
imageio.imwrite(os.path.join(logdir, f'grid_real_{ix}.png'),
grid_real)
imageio.imwrite(os.path.join(logdir, f'grid_fake_{ix}.png'),
grid_fake)
fake_batch = (np.clip(fake_batch, -1, 2) * 1024).astype(np.int16)
real_batch = (np.clip(real_batch, -1, 2) * 1024).astype(np.int16)
# fake_batch = real_batch
assert real_batch.min() < -512
assert fake_batch.min() < -512
fids_local.append(
calculate_fid_given_batch_volumes(real_batch, fake_batch,
sess))
if calc_swds:
swds = get_swd_for_volumes(real_batch, fake_batch)
swds_local.append(swds)
psnr = get_psnr(real_batch, fake_batch)
if calc_ssims:
ssim = get_ssim(real_batch, fake_batch)
ssims_local.append(ssim)
mse = get_mean_squared_error(real_batch, fake_batch)
nrmse = get_normalized_root_mse(real_batch, fake_batch)
psnrs_local.append(psnr)
mses_local.append(mse)
nrmses_local.append(nrmse)
save_path = os.path.join(logdir, f'{ix}.npy')
np.save(save_path, fake_batch)
fid_local = np.stack(fids_local).mean(0)
psnr_local = np.mean(psnrs_local)
ssim_local = np.mean(ssims_local)
mse_local = np.mean(mses_local)
nrmse_local = np.mean(nrmses_local)
if args.horovod:
fid = MPI.COMM_WORLD.allreduce(fid_local, op=MPI.SUM) / hvd.size()
psnr = MPI.COMM_WORLD.allreduce(psnr_local,
op=MPI.SUM) / hvd.size()
mse = MPI.COMM_WORLD.allreduce(mse_local, op=MPI.SUM) / hvd.size()
nrmse = MPI.COMM_WORLD.allreduce(nrmse_local,
op=MPI.SUM) / hvd.size()
if calc_ssims:
ssim = MPI.COMM_WORLD.allreduce(ssim_local,
op=MPI.SUM) / hvd.size()
else:
fid = fid_local
psnr = psnr_local
ssim = ssim_local
mse = mse_local
nrmse = nrmse_local
if calc_swds:
swds_local = np.array(swds_local)
# Average over batches
swds_local = swds_local.mean(axis=0)
if args.horovod:
swds = MPI.COMM_WORLD.allreduce(swds_local,
op=MPI.SUM) / hvd.size()
else:
swds = swds_local
summary_str = ""
if verbose:
summary_str += f"FIDS: {fid.tolist()} \n\n"
summary_str += f"FID: {fid.mean():.4f} \n"
summary_str += f"PSNR: {psnr:.4f} \n"
summary_str += f"MSE: {mse:.4f} \n"
summary_str += f"Normalized Root MSE: {nrmse:.4f} \n"
if calc_swds:
summary_str += f"SWDS: {swds} \n"
if calc_ssims:
summary_str += f"SSIM: {ssim} \n"
if verbose:
with open(os.path.join(logdir, 'summary.txt'), 'w') as f:
f.write(summary_str)
def map_tickets_to_sections(self, added_tickets, removed_tickets,
assignees, sections, database):
ASSIGNEE_MULTIPLE_COLOR = (0, 160, 225)
ASSIGNEE_DUPLICATE_LINE_COLOR = (0, 175, 255)
ASSIGNEE_NOT_ON_TICKET = (0, 0, 255)
ASSIGNEE_COLOR = (230, 90, 10)
TICKET_MOVED_LINE_COLOR = (50, 255, 50)
TICKET_MOVED_COLOR = (30, 255, 30)
TICKET_MOVED_ILLEGALLY_COLOR = (0, 0, 255)
TICKET_REMOVED_COLOR = (0, 200, 225)
TICKET_REMOVED_FROM_FINAL_COLOR = (70, 90, 70)
TICKET_DUPLICATE_COLOR = (0, 175, 255)
TICKET_DUPLICATE_LINE_COLOR = (0, 175, 255)
TICKET_NUMBER_INVALID_COLOR = (200, 0, 200)
SECTION_LIMIT_REACHED_COLOR = (45, 195, 255)
SECTION_LIMIT_SURPASSED_COLOR = (100, 100, 255)
image = self.curr_image.copy()
workflow = re.compile(self.data["Sections"]["regex"])
moved_ticket_numbers = set(added_tickets.keys()) & set(
removed_tickets.keys())
for ticket_num, ticket in removed_tickets.items():
x, y, w, h = ticket.bounding_rect
previous_section, certainty = ticket.belongs_to(sections)
sections[previous_section].remove_ticket(ticket)
if sections[
previous_section].function == Function.LIMIT or sections[
previous_section].function == Function.NONE:
continue
if sections[
previous_section].function == Function.FINAL and ticket_num not in added_tickets:
if ticket_num in self.data["Tickets"].keys():
self.data["Tickets"][ticket_num]["history"] = ""
database.update_ticket(ticket, self.data["Tickets"],
"database")
cv2.rectangle(image, (x, y), (x + w, y + h),
TICKET_REMOVED_FROM_FINAL_COLOR, 3)
logging.info(
f"TICKET - Ticket #{ticket.num} has been SUCCESSFULLY REMOVED from section \"{sections[previous_section].name}\" (ID:#{previous_section}, {certainty}%)"
)
else:
cv2.rectangle(image, (x, y), (x + w, y + h),
TICKET_REMOVED_COLOR, 3)
if ticket_num not in added_tickets:
logging.warning(
f"TICKET - Ticket #{ticket_num} REMOVED from section \"{sections[previous_section].name}\" (ID:#{previous_section}, {certainty}%)"
)
ticket.prev_section = sections[previous_section].name
for ticket_num, ticket in added_tickets.items():
x, y, w, h = ticket.bounding_rect
section_num, certainty = ticket.belongs_to(sections)
if sections[section_num].function == Function.LIMIT or sections[
section_num].function == Function.NONE:
continue
duplicates = ticket.num_belongs_to(sections)
for num in duplicates:
x2, y2, w2, h2 = sections[num].tickets[
ticket_num].bounding_rect
cv2.rectangle(image, (x2, y2), (x2 + w2, y2 + h2),
TICKET_DUPLICATE_COLOR, 3)
cv2.line(image, (int(x + w / 2), int(y + h / 2)),
(int(x2 + w2 / 2), int(y2 + h2 / 2)),
TICKET_DUPLICATE_LINE_COLOR, 3)
ticket.add_msg(
f"WARNING: TICKET - Ticket #{ticket_num} already exists in section \"{sections[num].name}\" (ID:#{num})"
)
ticket.add_msg(
f"INFO: TICKET - Ticket #{ticket_num} NEW reference REMOVED from section \"{sections[section_num].name}\" (ID:#{section_num})"
)
sections[section_num].remove_ticket(ticket)
section_num, certainty = ticket.belongs_to(sections)
if ticket_num in self.data["Tickets"].keys():
if not (ticket_num in removed_tickets
and removed_tickets[ticket_num].prev_section
== section_num):
ticket_history = self.data["Tickets"][ticket_num][
"history"] + str(sections[section_num].name) + "-"
ticket.desc = self.data["Tickets"][ticket_num][
"description"]
else:
ticket.add_error(
Msg_Level.TICKET_NUMBER_INVALID,
f"ERROR: TICKET - Ticket #{ticket_num} does not exist in the database!"
)
ticket_history = ""
if workflow.fullmatch(ticket_history) is not None:
self.data["Tickets"][ticket_num]["history"] = ticket_history
ticket.section_history = ticket_history
if ticket_num in removed_tickets:
x2, y2, w2, h2 = removed_tickets[ticket_num].bounding_rect
cv2.line(image, (int(x + w / 2), int(y + h / 2)),
(int(x2 + w2 / 2), int(y2 + h2 / 2)),
TICKET_MOVED_LINE_COLOR, 3)
previous_section = removed_tickets[ticket_num].prev_section
if previous_section != sections[section_num].name:
ticket.add_msg(
f"INFO: TICKET - Ticket #{ticket_num} successfully MOVED from section \"{previous_section}\" to section \"{sections[section_num].name}\" (ID:#{section_num}, {certainty}%)"
)
else:
ticket.add_msg(
f"INFO: TICKET - Ticket #{ticket.num} has been ADDED to section \"{sections[section_num].name}\" (ID:#{section_num}, {certainty}%)"
)
else:
if ticket_num in removed_tickets:
x2, y2, w2, h2 = removed_tickets[ticket_num].bounding_rect
cv2.line(image, (int(x + w / 2), int(y + h / 2)),
(int(x2 + w2 / 2), int(y2 + h2 / 2)),
TICKET_MOVED_ILLEGALLY_COLOR, 3)
previous_section = removed_tickets[ticket_num].prev_section
ticket.add_error(
Msg_Level.TICKET_REGEX,
f"WARNING: TICKET - Ticket #{ticket_num} illegally MOVED from section \"{previous_section}\" to section \"{sections[section_num].name}\" (ID:#{section_num}, {certainty}%)"
)
else:
ticket.add_error(
Msg_Level.TICKET_REGEX,
f"WARNING: TICKET - Ticket #{ticket_num} illegally ADDED to section \"{sections[section_num].name}\" (ID:#{section_num}, {certainty}%)"
)
if len(ticket.errors) == 0:
cv2.rectangle(image, (x, y), (x + w, y + h),
TICKET_MOVED_COLOR, 3)
sections[section_num].add_ticket(ticket)
for assignee in assignees:
assignee.assign_to_ticket(sections)
for section_num, section in sections.items():
database.update_section(section)
for ticket_num, ticket in section.tickets.items():
ticket_assignees = Assignee.belonging_to_ticket_and_section(
assignees, ticket_num, section_num)
ticket.set_assignee(ticket_assignees)
ticket.print_msgs()
database.update_ticket(ticket, self.data["Tickets"],
section.name)
x, y, w, h = ticket.bounding_rect
offset = 0
for error in ticket.errors:
if error == Msg_Level.ASSIGNEE_MULTIPLE:
cv2.rectangle(image, (x - offset, y - offset),
(x + w + offset, y + h + offset),
ASSIGNEE_MULTIPLE_COLOR, 3)
elif error == Msg_Level.TICKET_REGEX:
cv2.rectangle(image, (x - offset, y - offset),
(x + w + offset, y + h + offset),
TICKET_MOVED_ILLEGALLY_COLOR, 3)
elif error == Msg_Level.TICKET_NUMBER_INVALID:
cv2.rectangle(image, (x - offset, y - offset),
(x + w + offset, y + h + offset),
TICKET_NUMBER_INVALID_COLOR, 3)
offset += 15
if section.over_section_limit():
x, y, w, h = section.bounding_rect
cv2.rectangle(image, (x, y), (x + w, y + h),
SECTION_LIMIT_SURPASSED_COLOR, 4)
logging.warning(
f"SECTION - The ticket limit of {section.limit} has been surpassed for section \"{section.name}\" (ID:#{section_num})"
)
elif len(section.tickets) == section.limit:
x, y, w, h = section.bounding_rect
cv2.rectangle(image, (x, y), (x + w, y + h),
SECTION_LIMIT_REACHED_COLOR, 4)
logging.info(
f"SECTION - The ticket limit of {section.limit} has been reached for section \"{section.name}\" (ID:#{section_num})"
)
while len(assignees) > 0:
a1 = assignees.pop(0)
tl1, br1 = a1.bounding_rect
if a1.ticket_num == None:
cv2.rectangle(image, (tl1[0], tl1[1]), (br1[0], br1[1]),
ASSIGNEE_NOT_ON_TICKET, 3)
else:
cv2.putText(image, f"{a1.name}",
(int(tl1[0]), int(tl1[1] - 10)),
cv2.FONT_HERSHEY_DUPLEX, 1, ASSIGNEE_COLOR, 1)
cv2.rectangle(image, (tl1[0], tl1[1]), (br1[0], br1[1]),
ASSIGNEE_COLOR, 3)
for a2 in assignees:
if a1.name == a2.name:
tl2, br2 = a2.bounding_rect
cv2.putText(image, f"{a2.name}",
(int(tl2[0]), int(tl2[1] - 10)),
cv2.FONT_HERSHEY_DUPLEX, 1, ASSIGNEE_COLOR,
1)
cv2.rectangle(image, (tl2[0], tl2[1]),
(br2[0], br2[1]), ASSIGNEE_COLOR, 3)
x1 = int((tl1[0] + br1[0]) // 2)
x2 = int((tl2[0] + br2[0]) // 2)
y1 = int((tl1[1] + br1[1]) // 2)
y2 = int((tl2[1] + br2[1]) // 2)
cv2.line(image, (x1, y1), (x2, y2),
ASSIGNEE_DUPLICATE_LINE_COLOR, 3)
assignees.remove(a2)
self.board_extracted = image
self.colored_grid = utils.image_grid(self.curr_image, self.prev_image,
self.board_extracted,
self.board_sections)
