def run(self):
self.parent.clear_plots()
if self.data is None:
length, tMin, tMax, fMin, fMax, lMin, lMax, peakF, peakL, peakT = ("",) * 10
else:
length = len(self.data)
tMin = format_time(self.extent.tMin, True)
tMax = format_time(self.extent.tMax, True)
fMin = format_precision(self.settings, freq=self.extent.fMin, fancyUnits=True)
fMax = format_precision(self.settings, freq=self.extent.fMax, fancyUnits=True)
lMin = format_precision(self.settings, level=self.extent.lMin, fancyUnits=True)
lMax = format_precision(self.settings, level=self.extent.lMax, fancyUnits=True)
peak = self.extent.get_peak_flt()
peakF = format_precision(self.settings, freq=peak[0], fancyUnits=True)
peakL = format_precision(self.settings, level=peak[1], fancyUnits=True)
peakT = format_time(peak[2], True)
text = [
["Sweeps", "", length],
["Extents", "", ""],
["", "Start", tMin],
["", "End", tMax],
["", "Min frequency", fMin],
["", "Max frequency", fMax],
["", "Min level", lMin],
["", "Max level", lMax],
["Peak", "", ""],
["", "Level", peakL],
["", "Frequency", peakF],
["", "Time", peakT],
]
table = Table(self.axes, loc="center")
table.set_gid("table")
rows = len(text)
cols = len(text[0])
for row in xrange(rows):
for col in xrange(cols):
table.add_cell(row, col, text=text[row][col], width=1.0 / cols, height=1.0 / rows)
if self.settings.grid:
colour = "LightGray"
else:
colour = "w"
set_table_colour(table, colour)
for i in range(3):
table.auto_set_column_width(i)
self.axes.add_table(table)
self.parent.redraw_plot()
self.parent.threadPlot = None
def __update_grid(self, level=None):
self.gridLog.ClearGrid()
fontCell = self.gridLog.GetDefaultCellFont()
fontSize = fontCell.GetPointSize()
fontStyle = fontCell.GetStyle()
fontWeight = fontCell.GetWeight()
font = wx.Font(fontSize, wx.FONTFAMILY_MODERN, fontStyle,
fontWeight)
i = 0
for event in self.log.get(level):
self.gridLog.SetCellValue(i, 0, format_time(event[0], True))
self.gridLog.SetCellValue(i, 1, self.log.TEXT_LEVEL[event[1]])
eventText = '\n'.join(textwrap.wrap(event[2], width=70))
self.gridLog.SetCellValue(i, 2, eventText)
self.gridLog.SetCellFont(i, 0, font)
self.gridLog.SetCellFont(i, 1, font)
self.gridLog.SetCellFont(i, 2, font)
self.gridLog.SetCellAlignment(i, 0, wx.ALIGN_LEFT, wx.ALIGN_CENTRE)
self.gridLog.SetCellAlignment(i, 1, wx.ALIGN_LEFT, wx.ALIGN_CENTRE)
i += 1
self.gridLog.AppendRows()
self.gridLog.SetCellValue(i, 0, '#' * 18)
self.gridLog.SetCellValue(i, 1, '#' * 5)
self.gridLog.SetCellValue(i, 2, '#' * 80)
self.gridLog.AutoSize()
self.gridLog.DeleteRows(i)
size = self.gridLog.GetBestSize()
size.width += wx.SystemSettings.GetMetric(wx.SYS_VSCROLL_X) + 10
size.height = 400
self.SetClientSize(size)
self.sizer.Layout()
def annotate_plot(self):
self.clear_markers()
fMax, lMax, tMax = self.extent.get_peak_flt()
y = epoch_to_mpl(tMax)
start, stop = self.axes.get_xlim()
textX = ((stop - start) / 50.0) + fMax
when = format_time(fMax)
if(matplotlib.__version__ < '1.3'):
self.axes.annotate('{0:.6f}MHz\n{1:.2f}dB\n{2}'.format(fMax,
lMax,
when),
xy=(fMax, y), xytext=(textX, y),
ha='left', va='bottom', size='small',
color='w', gid='peak')
self.axes.plot(fMax, y, marker='x', markersize=10, color='w',
mew=3, gid='peak')
self.axes.plot(fMax, y, marker='x', markersize=10, color='r',
gid='peak')
else:
effect = patheffects.withStroke(linewidth=3, foreground="w",
alpha=0.75)
self.axes.annotate('{0:.6f}MHz\n{1:.2f}dB\n{2}'.format(fMax,
lMax,
when),
xy=(fMax, y), xytext=(textX, y),
ha='left', va='bottom', size='small',
path_effects=[effect], gid='peak')
self.axes.plot(fMax, y, marker='x', markersize=10, color='r',
path_effects=[effect], gid='peak')
def __annotate_plot(self):
f, l, t = self.extent.get_peak_flt()
when = format_time(t)
tPos = utc_to_mpl(t)
text = '{}\n{}\n{when}'.format(*format_precision(self.settings, f, l,
fancyUnits=True),
when=when)
if matplotlib.__version__ < '1.3':
self.axes.text(f, tPos, l,
text,
ha='left', va='bottom', size='x-small', gid='peak')
self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
mew=3, color='w', gid='peak')
self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
color='r', gid='peak')
else:
effect = patheffects.withStroke(linewidth=2, foreground="w",
alpha=0.75)
self.axes.text(f, tPos, l,
text,
ha='left', va='bottom', size='x-small', gid='peak',
path_effects=[effect])
self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
color='r', gid='peak', path_effects=[effect])
def __plot_peak(self, peakF, peakL, peakT):
when = format_time(peakT)
tPos = utc_to_mpl(peakT)
text = '{}\n{}\n{when}'.format(*format_precision(self.settings,
peakF, peakL,
fancyUnits=True),
when=when)
if matplotlib.__version__ < '1.3':
self.axes.text(peakF, tPos, peakL,
text,
ha='left', va='bottom', size='x-small', gid='peakText')
self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
mew=3, color='w', gid='peak')
self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
color='r', gid='peakShadow')
else:
effect = patheffects.withStroke(linewidth=2, foreground="w",
alpha=0.75)
self.axes.text(peakF, tPos, peakL,
text,
ha='left', va='bottom', size='x-small', gid='peakText',
path_effects=[effect])
self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
color='r', gid='peak', path_effects=[effect])
def __plot_peak(self, peakF, peakL, peakT):
when = format_time(peakT)
tPos = utc_to_mpl(peakT)
text = '{}\n{}\n{when}'.format(*format_precision(self.settings,
peakF, peakL,
fancyUnits=True),
when=when)
if matplotlib.__version__ < '1.3':
self.axes.text(peakF, tPos, peakL,
text,
ha='left', va='bottom', size='x-small', gid='peakText')
self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
mew=3, color='w', gid='peak')
self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
color='r', gid='peakShadow')
else:
effect = patheffects.withStroke(linewidth=2, foreground="w",
alpha=0.75)
self.axes.text(peakF, tPos, peakL,
text,
ha='left', va='bottom', size='x-small', gid='peakText',
path_effects=[effect])
self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
color='r', gid='peak', path_effects=[effect])
def __annotate_plot(self):
self.__clear_markers()
fMax, lMax, tMax = self.extent.get_peak_flt()
y = epoch_to_mpl(tMax)
start, stop = self.axes.get_xlim()
textX = ((stop - start) / 50.0) + fMax
when = format_time(tMax)
text = "{0:.6f} MHz\n{1:.2f} $\mathsf{{dB/\sqrt{{Hz}}}}$\n{2}".format(fMax, lMax, when)
if matplotlib.__version__ < "1.3":
self.axes.annotate(
text, xy=(fMax, y), xytext=(textX, y), ha="left", va="bottom", size="x-small", color="w", gid="peak"
)
self.axes.plot(fMax, y, marker="x", markersize=10, color="w", mew=3, gid="peak")
self.axes.plot(fMax, y, marker="x", markersize=10, color="r", gid="peak")
else:
effect = patheffects.withStroke(linewidth=2, foreground="w", alpha=0.75)
self.axes.annotate(
text,
xy=(fMax, y),
xytext=(textX, y),
ha="left",
va="bottom",
size="x-small",
path_effects=[effect],
gid="peak",
)
self.axes.plot(fMax, y, marker="x", markersize=10, color="r", path_effects=[effect], gid="peak")
def __plot_peak(self, peakF, peakL, peakT):
self.__clear_markers()
y = utc_to_mpl(peakT)
start, stop = self.axes.get_xlim()
textX = ((stop - start) / 50.0) + peakF
when = format_time(peakT)
text = '{}\n{}\n{when}'.format(*format_precision(self.settings,
peakF, peakL,
fancyUnits=True),
when=when)
if matplotlib.__version__ < '1.3':
self.axes.annotate(text,
xy=(peakF, y), xytext=(textX, y),
ha='left', va='bottom', size='x-small',
color='w', gid='peakText')
self.axes.plot(peakF, y, marker='x', markersize=10, color='w',
mew=3, gid='peakShadow')
self.axes.plot(peakF, y, marker='x', markersize=10, color='r',
gid='peak')
else:
effect = patheffects.withStroke(linewidth=2, foreground="w",
alpha=0.75)
self.axes.annotate(text,
xy=(peakF, y), xytext=(textX, y),
ha='left', va='bottom', size='x-small',
path_effects=[effect], gid='peakText')
self.axes.plot(peakF, y, marker='x', markersize=10, color='r',
path_effects=[effect], gid='peak')
def __plot_peak(self, peakF, peakL, peakT):
self.__clear_markers()
y = utc_to_mpl(peakT)
start, stop = self.axes.get_xlim()
textX = ((stop - start) / 50.0) + peakF
when = format_time(peakT)
text = '{}\n{}\n{when}'.format(*format_precision(self.settings,
peakF, peakL,
fancyUnits=True),
when=when)
if matplotlib.__version__ < '1.3':
self.axes.annotate(text,
xy=(peakF, y), xytext=(textX, y),
ha='left', va='bottom', size='x-small',
color='w', gid='peakText')
self.axes.plot(peakF, y, marker='x', markersize=10, color='w',
mew=3, gid='peakShadow')
self.axes.plot(peakF, y, marker='x', markersize=10, color='r',
gid='peak')
else:
effect = patheffects.withStroke(linewidth=2, foreground="w",
alpha=0.75)
self.axes.annotate(text,
xy=(peakF, y), xytext=(textX, y),
ha='left', va='bottom', size='x-small',
path_effects=[effect], gid='peakText')
self.axes.plot(peakF, y, marker='x', markersize=10, color='r',
path_effects=[effect], gid='peak')
def __update_grid(self, level=None):
self.gridLog.ClearGrid()
fontCell = self.gridLog.GetDefaultCellFont()
fontSize = fontCell.GetPointSize()
fontStyle = fontCell.GetStyle()
fontWeight = fontCell.GetWeight()
font = wx.Font(fontSize, wx.FONTFAMILY_MODERN, fontStyle, fontWeight)
i = 0
for event in self.log.get(level):
self.gridLog.SetCellValue(i, 0, format_time(event[0], True))
self.gridLog.SetCellValue(i, 1, self.log.TEXT_LEVEL[event[1]])
eventText = '\n'.join(textwrap.wrap(event[2], width=70))
self.gridLog.SetCellValue(i, 2, eventText)
self.gridLog.SetCellFont(i, 0, font)
self.gridLog.SetCellFont(i, 1, font)
self.gridLog.SetCellFont(i, 2, font)
self.gridLog.SetCellAlignment(i, 0, wx.ALIGN_LEFT, wx.ALIGN_CENTRE)
self.gridLog.SetCellAlignment(i, 1, wx.ALIGN_LEFT, wx.ALIGN_CENTRE)
i += 1
self.gridLog.AppendRows()
self.gridLog.SetCellValue(i, 0, '#' * 18)
self.gridLog.SetCellValue(i, 1, '#' * 5)
self.gridLog.SetCellValue(i, 2, '#' * 80)
self.gridLog.AutoSize()
self.gridLog.DeleteRows(i)
size = self.gridLog.GetBestSize()
size.width += wx.SystemSettings.GetMetric(wx.SYS_VSCROLL_X) + 10
size.height = 400
self.SetClientSize(size)
self.sizer.Layout()
def calc_inception_scores(run_id,
log='inception.txt',
num_images=50000,
minibatch_size=16,
eval_reals=True,
reverse_order=False):
result_subdir = misc.locate_result_subdir(run_id)
network_pkls = misc.list_network_pkls(result_subdir)
misc.set_output_log_file(os.path.join(result_subdir, log))
print 'Importing inception score module...'
import inception_score
def calc_inception_score(images):
if images.shape[1] == 1:
images = images.repeat(3, axis=1)
images = list(images.transpose(0, 2, 3, 1))
return inception_score.get_inception_score(images)
# Load dataset.
training_set, drange_orig = load_dataset_for_previous_run(result_subdir,
shuffle=False)
reals, labels = training_set.get_random_minibatch(num_images, labels=True)
# Evaluate reals.
if eval_reals:
print 'Evaluating inception score for reals...'
time_begin = time.time()
mean, std = calc_inception_score(reals)
print 'Done in %s' % misc.format_time(time.time() - time_begin)
print '%-32s mean %-8.4f std %-8.4f' % ('reals', mean, std)
# Evaluate each network snapshot.
network_pkls = list(enumerate(network_pkls))
if reverse_order:
network_pkls = network_pkls[::-1]
for network_idx, network_pkl in network_pkls:
print '%-32s' % os.path.basename(network_pkl),
net = imgapi_load_net(run_id=result_subdir,
snapshot=network_pkl,
num_example_latents=num_images,
random_seed=network_idx,
load_dataset=False)
fakes = imgapi_generate_batch(net,
net.example_latents,
np.random.permutation(labels),
minibatch_size=minibatch_size,
convert_to_uint8=True)
mean, std = calc_inception_score(fakes)
print 'mean %-8.4f std %-8.4f' % (mean, std)
print
print 'Done.'
def annotate_plot(self):
f, l, t = self.extent.get_peak_flt()
when = format_time(t)
tPos = epoch_to_mpl(t)
if(matplotlib.__version__ < '1.3'):
self.axes.text(f, tPos, l,
'{0:.6f}MHz\n{1:.2f}dB\n{2}'.format(f, l, when),
ha='left', va='bottom', size='small', gid='peak')
self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
mew=3, color='w', gid='peak')
self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
color='r', gid='peak')
else:
effect = patheffects.withStroke(linewidth=3, foreground="w",
alpha=0.75)
self.axes.text(f, tPos, l,
'{0:.6f}MHz\n{1:.2f}dB\n{2}'.format(f, l, when),
ha='left', va='bottom', size='small', gid='peak',
path_effects=[effect])
self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
color='r', gid='peak', path_effects=[effect])
def __create_last(self):
loc = self.server.currentLoc
if loc[0] is None:
return ''
last = ('\t\t<Placemark>\n'
'\t\t\t<name>Last Location</name>\n'
'\t\t\t<description>{}</description>\n'
'\t\t\t<styleUrl>#last</styleUrl>\n'
'\t\t\t<altitudeMode>clampToGround</altitudeMode>\n'
'\t\t\t<Point>\n').format(format_time(loc[3]))
if loc[2] is None:
last += '\t\t\t\t<coordinates>{},{}</coordinates>\n'.\
format(loc[1], loc[0])
else:
last += '\t\t\t\t<coordinates>{},{},{}</coordinates>\n'.\
format(loc[1], loc[0], loc[2])
last += ('\t\t\t</Point>\n' '\t\t</Placemark>\n')
return last
def main(args,
model,
train_loader,
test_loader,
decreasing_lr=[],
print=print):
if args.cuda:
model = model.cuda()
if isinstance(args.decreasing_lr, str):
args.decreasing_lr = list(map(int, args.decreasing_lr.split(',')))
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
info = {'best_acc': 0, 'old_file': None, 't_begin': time.time()}
try:
# ready to go
for epoch in range(args.epochs):
info['epoch'] = epoch
train(args, model, optimizer, train_loader, info, print)
elapse_time = time.time() - info['t_begin']
speed_epoch = elapse_time / (epoch + 1)
speed_batch = speed_epoch / len(train_loader)
eta = speed_epoch * args.epochs - elapse_time
print("Elapsed {}, {:.2f} s/epoch, {:.2f} s/batch, ets {:.2f}s".
format(misc.format_time(elapse_time), speed_epoch,
speed_batch, eta))
if epoch % args.test_interval == 0:
eval(args, model, test_loader, info, print)
except Exception as e:
import traceback
traceback.print_exc()
except KeyboardInterrupt:
exit()
finally:
print("Total Elapse: {:.2f}, Best Result: {:.3f}%".format(
time.time() - info['t_begin'], info['best_acc']))
def __create_last(self):
loc = self.server.currentLoc
if loc[0] is None:
return ''
last = ('\t\t<Placemark>\n'
'\t\t\t<name>Last Location</name>\n'
'\t\t\t<description>{}</description>\n'
'\t\t\t<styleUrl>#last</styleUrl>\n'
'\t\t\t<altitudeMode>clampToGround</altitudeMode>\n'
'\t\t\t<Point>\n').format(format_time(loc[3]))
if loc[2] is None:
last += '\t\t\t\t<coordinates>{},{}</coordinates>\n'.\
format(loc[1], loc[0])
else:
last += '\t\t\t\t<coordinates>{},{},{}</coordinates>\n'.\
format(loc[1], loc[0], loc[2])
last += ('\t\t\t</Point>\n'
'\t\t</Placemark>\n')
return last
def train_progressive_gan(
G_smoothing=0.999, # Exponential running average of generator weights.
D_repeats=1, # How many times the discriminator is trained per G iteration.
minibatch_repeats=4, # Number of minibatches to run before adjusting training parameters.
reset_opt_for_new_lod=True, # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
total_kimg=15000, # Total length of the training, measured in thousands of real images.
mirror_augment=False, # Enable mirror augment?
drange_net=[
-1, 1
], # Dynamic range used when feeding image data to the networks.
image_snapshot_ticks=1, # How often to export image snapshots?
network_snapshot_ticks=10, # How often to export network snapshots?
save_tf_graph=False, # Include full TensorFlow computation graph in the tfevents file?
save_weight_histograms=False, # Include weight histograms in the tfevents file?
resume_run_id=None, # Run ID or network pkl to resume training from, None = start from scratch.
resume_snapshot=None, # Snapshot index to resume training from, None = autodetect.
resume_kimg=0.0, # Assumed training progress at the beginning. Affects reporting and training schedule.
resume_time=0.0
): # Assumed wallclock time at the beginning. Affects reporting.
maintenance_start_time = time.time()
training_set = dataset.load_dataset(data_dir=config.data_dir,
verbose=True,
**config.dataset)
#resume_run_id = '/dresden/users/mk1391/evl/pggan_logs/logs_celeba128cc/fsg16_results_0/000-pgan-celeba-preset-v2-2gpus-fp32/network-snapshot-010211.pkl'
resume_with_new_nets = False
# Construct networks.
with tf.device('/gpu:0'):
if resume_run_id is None or resume_with_new_nets:
print('Constructing networks...')
G = tfutil.Network('G',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.G)
D = tfutil.Network('D',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.D)
Gs = G.clone('Gs')
if resume_run_id is not None:
network_pkl = misc.locate_network_pkl(resume_run_id,
resume_snapshot)
print('Loading networks from "%s"...' % network_pkl)
rG, rD, rGs = misc.load_pkl(network_pkl)
if resume_with_new_nets:
G.copy_vars_from(rG)
D.copy_vars_from(rD)
Gs.copy_vars_from(rGs)
else:
G = rG
D = rD
Gs = rGs
Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
G.print_layers()
D.print_layers()
### pyramid draw fsg (comment out for actual training to happen)
#draw_gen_fsg(Gs, 10, os.path.join(config.result_dir, 'pggan_fsg_draw.png'))
#print('>>> done printing fsgs.')
#return
print('Building TensorFlow graph...')
with tf.name_scope('Inputs'):
lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
minibatch_split = minibatch_in // config.num_gpus
reals, labels = training_set.get_minibatch_tf()
reals_split = tf.split(reals, config.num_gpus)
labels_split = tf.split(labels, config.num_gpus)
G_opt = tfutil.Optimizer(name='TrainG',
learning_rate=lrate_in,
**config.G_opt)
D_opt = tfutil.Optimizer(name='TrainD',
learning_rate=lrate_in,
**config.D_opt)
for gpu in range(config.num_gpus):
with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
lod_assign_ops = [
tf.assign(G_gpu.find_var('lod'), lod_in),
tf.assign(D_gpu.find_var('lod'), lod_in)
]
reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment,
training_set.dynamic_range, drange_net)
labels_gpu = labels_split[gpu]
with tf.name_scope('G_loss'), tf.control_dependencies(
lod_assign_ops):
G_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
opt=G_opt,
training_set=training_set,
minibatch_size=minibatch_split,
**config.G_loss)
with tf.name_scope('D_loss'), tf.control_dependencies(
lod_assign_ops):
D_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
opt=D_opt,
training_set=training_set,
minibatch_size=minibatch_split,
reals=reals_gpu,
labels=labels_gpu,
**config.D_loss)
G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
G_train_op = G_opt.apply_updates()
D_train_op = D_opt.apply_updates()
print('Setting up snapshot image grid...')
grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(
G, training_set, **config.grid)
### shift reals
print('>>> reals shape: ', grid_reals.shape)
fc_x = 0.5
fc_y = 0.5
im_size = grid_reals.shape[-1]
kernel_loc = 2.*np.pi*fc_x * np.arange(im_size).reshape((1, 1, im_size)) + \
2.*np.pi*fc_y * np.arange(im_size).reshape((1, im_size, 1))
kernel_cos = np.cos(kernel_loc)
kernel_sin = np.sin(kernel_loc)
reals_t = (grid_reals / 255.) * 2. - 1
reals_t *= kernel_cos
grid_reals_sh = np.rint(
(reals_t + 1.) * 255. / 2.).clip(0, 255).astype(np.uint8)
### end shift reals
sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
grid_fakes = Gs.run(grid_latents,
grid_labels,
minibatch_size=sched.minibatch // config.num_gpus)
### fft drawing
#sys.path.insert(1, '/home/mahyar/CV_Res/ganist')
#from fig_draw import apply_fft_win
#data_size = 1000
#latents = np.random.randn(data_size, *Gs.input_shapes[0][1:])
#labels = np.zeros([latents.shape[0]] + Gs.input_shapes[1][1:])
#g_samples = Gs.run(latents, labels, minibatch_size=sched.minibatch//config.num_gpus)
#g_samples = g_samples.transpose(0, 2, 3, 1)
#print('>>> g_samples shape: {}'.format(g_samples.shape))
#apply_fft_win(g_samples, 'fft_pggan_hann.png')
### end fft drawing
print('Setting up result dir...')
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
misc.save_image_grid(grid_reals,
os.path.join(result_subdir, 'reals.png'),
drange=training_set.dynamic_range,
grid_size=grid_size)
### drawing shifted real images
misc.save_image_grid(grid_reals_sh,
os.path.join(result_subdir, 'reals_sh.png'),
drange=training_set.dynamic_range,
grid_size=grid_size)
misc.save_image_grid(grid_fakes,
os.path.join(result_subdir, 'fakes%06d.png' % 0),
drange=drange_net,
grid_size=grid_size)
### drawing shifted fake images
misc.save_image_grid(grid_fakes * kernel_cos,
os.path.join(result_subdir, 'fakes%06d_sh.png' % 0),
drange=drange_net,
grid_size=grid_size)
summary_log = tf.summary.FileWriter(result_subdir)
if save_tf_graph:
summary_log.add_graph(tf.get_default_graph())
if save_weight_histograms:
G.setup_weight_histograms()
D.setup_weight_histograms()
#### True cosine fft eval
#fft_data_size = 1000
#im_size = training_set.shape[1]
#freq_centers = [(64/128., 64/128.)]
#true_samples = sample_true(training_set, fft_data_size, dtype=training_set.dtype, batch_size=32).transpose(0, 2, 3, 1) / 255. * 2. - 1.
#true_fft, true_fft_hann, true_hist = cosine_eval(true_samples, 'true', freq_centers, log_dir=result_subdir)
#fractal_eval(true_samples, f'koch_snowflake_true', result_subdir)
print('Training...')
cur_nimg = int(resume_kimg * 1000)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
train_start_time = tick_start_time - resume_time
prev_lod = -1.0
while cur_nimg < total_kimg * 1000:
# Choose training parameters and configure training ops.
sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
training_set.configure(sched.minibatch, sched.lod)
if reset_opt_for_new_lod:
if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
sched.lod) != np.ceil(prev_lod):
G_opt.reset_optimizer_state()
D_opt.reset_optimizer_state()
prev_lod = sched.lod
# Run training ops.
for repeat in range(minibatch_repeats):
for _ in range(D_repeats):
tfutil.run(
[D_train_op, Gs_update_op], {
lod_in: sched.lod,
lrate_in: sched.D_lrate,
minibatch_in: sched.minibatch
})
cur_nimg += sched.minibatch
tfutil.run(
[G_train_op], {
lod_in: sched.lod,
lrate_in: sched.G_lrate,
minibatch_in: sched.minibatch
})
# Perform maintenance tasks once per tick.
done = (cur_nimg >= total_kimg * 1000)
if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
total_time = cur_time - train_start_time
maintenance_time = tick_start_time - maintenance_start_time
maintenance_start_time = cur_time
# Report progress.
print(
'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f'
% (tfutil.autosummary('Progress/tick', cur_tick),
tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
tfutil.autosummary('Progress/lod', sched.lod),
tfutil.autosummary('Progress/minibatch', sched.minibatch),
misc.format_time(
tfutil.autosummary('Timing/total_sec', total_time)),
tfutil.autosummary('Timing/sec_per_tick', tick_time),
tfutil.autosummary('Timing/sec_per_kimg',
tick_time / tick_kimg),
tfutil.autosummary('Timing/maintenance_sec',
maintenance_time)))
tfutil.autosummary('Timing/total_hours',
total_time / (60.0 * 60.0))
tfutil.autosummary('Timing/total_days',
total_time / (24.0 * 60.0 * 60.0))
tfutil.save_summaries(summary_log, cur_nimg)
# Save snapshots.
if cur_tick % image_snapshot_ticks == 0 or done:
grid_fakes = Gs.run(grid_latents,
grid_labels,
minibatch_size=sched.minibatch //
config.num_gpus)
misc.save_image_grid(grid_fakes,
os.path.join(
result_subdir,
'fakes%06d.png' % (cur_nimg // 1000)),
drange=drange_net,
grid_size=grid_size)
### drawing shifted fake images
misc.save_image_grid(
grid_fakes * kernel_cos,
os.path.join(result_subdir,
'fakes%06d_sh.png' % (cur_nimg // 1000)),
drange=drange_net,
grid_size=grid_size)
### drawing fsg
#draw_gen_fsg(Gs, 10, os.path.join(config.result_dir, 'fakes%06d_fsg_draw.png' % (cur_nimg // 1000)))
### Gen fft eval
#gen_samples = sample_gen(Gs, fft_data_size).transpose(0, 2, 3, 1)
#print(f'>>> fake_samples: max={np.amax(grid_fakes)} min={np.amin(grid_fakes)}')
#print(f'>>> gen_samples: max={np.amax(gen_samples)} min={np.amin(gen_samples)}')
#misc.save_image_grid(gen_samples[:25], os.path.join(result_subdir, 'fakes%06d_gsample.png' % (cur_nimg // 1000)), drange=drange_net, grid_size=grid_size)
#cosine_eval(gen_samples, f'gen_{cur_nimg//1000:06d}', freq_centers, log_dir=result_subdir, true_fft=true_fft, true_fft_hann=true_fft_hann, true_hist=true_hist)
#fractal_eval(gen_samples, f'koch_snowflake_fakes{cur_nimg//1000:06d}', result_subdir)
if cur_tick % network_snapshot_ticks == 0 or done:
misc.save_pkl(
(G, D, Gs),
os.path.join(
result_subdir,
'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))
# Record start time of the next tick.
tick_start_time = time.time()
# Write final results.
misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
summary_log.close()
open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
def __init__(self, parent, scanInfo):
wx.Dialog.__init__(self, parent, title="Scan Properties")
self.scanInfo = scanInfo
box = wx.BoxSizer(wx.VERTICAL)
grid = wx.GridBagSizer(0, 0)
boxScan = wx.StaticBoxSizer(wx.StaticBox(self, wx.ID_ANY, "Scan"),
wx.HORIZONTAL)
gridScan = wx.GridBagSizer(0, 0)
textDesc = wx.StaticText(self, label="Description")
gridScan.Add(textDesc, (0, 0), (1, 1), wx.ALL, 5)
self.textCtrlDesc = wx.TextCtrl(self, value=scanInfo.desc,
style=wx.TE_MULTILINE)
gridScan.Add(self.textCtrlDesc, (0, 1), (2, 2), wx.ALL | wx.EXPAND, 5)
textStart = wx.StaticText(self, label="Start")
gridScan.Add(textStart, (2, 0), (1, 1), wx.ALL, 5)
textCtrlStart = wx.TextCtrl(self, value="Unknown",
style=wx.TE_READONLY)
if scanInfo.start is not None:
textCtrlStart.SetValue(str(scanInfo.start))
gridScan.Add(textCtrlStart, (2, 1), (1, 1), wx.ALL, 5)
textMHz1 = wx.StaticText(self, wx.ID_ANY, label="MHz")
gridScan.Add(textMHz1, (2, 2), (1, 1), wx.ALL, 5)
textStop = wx.StaticText(self, label="Stop")
gridScan.Add(textStop, (3, 0), (1, 1), wx.ALL, 5)
textCtrlStop = wx.TextCtrl(self, value="Unknown",
style=wx.TE_READONLY)
if scanInfo.stop is not None:
textCtrlStop.SetValue(str(scanInfo.stop))
gridScan.Add(textCtrlStop, (3, 1), (1, 1), wx.ALL, 5)
textMHz2 = wx.StaticText(self, label="MHz")
gridScan.Add(textMHz2, (3, 2), (1, 1), wx.ALL, 5)
textDwell = wx.StaticText(self, label="Dwell")
gridScan.Add(textDwell, (4, 0), (1, 1), wx.ALL, 5)
textCtrlDwell = wx.TextCtrl(self, value="Unknown",
style=wx.TE_READONLY)
if scanInfo.dwell is not None:
textCtrlDwell.SetValue(str(scanInfo.dwell))
gridScan.Add(textCtrlDwell, (4, 1), (1, 1), wx.ALL, 5)
textSeconds = wx.StaticText(self, label="seconds")
gridScan.Add(textSeconds, (4, 2), (1, 1), wx.ALL, 5)
textNfft = wx.StaticText(self, label="FFT Size")
gridScan.Add(textNfft, (5, 0), (1, 1), wx.ALL, 5)
textCtrlNfft = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
if scanInfo.nfft is not None:
textCtrlNfft.SetValue(str(scanInfo.nfft))
gridScan.Add(textCtrlNfft, (5, 1), (1, 1), wx.ALL, 5)
textRbw = wx.StaticText(self, label="RBW")
gridScan.Add(textRbw, (6, 0), (1, 1), wx.ALL, 5)
rbw = ((SAMPLE_RATE / scanInfo.nfft) / 1000.0) * 2.0
textCtrlStop = wx.TextCtrl(self, value="{0:.3f}".format(rbw),
style=wx.TE_READONLY)
gridScan.Add(textCtrlStop, (6, 1), (1, 1), wx.ALL, 5)
textKHz = wx.StaticText(self, label="kHz")
gridScan.Add(textKHz, (6, 2), (1, 1), wx.ALL, 5)
textTime = wx.StaticText(self, label="First scan")
gridScan.Add(textTime, (7, 0), (1, 1), wx.ALL, 5)
textCtrlTime = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
if scanInfo.timeFirst is not None:
textCtrlTime.SetValue(format_time(scanInfo.timeFirst, True))
gridScan.Add(textCtrlTime, (7, 1), (1, 1), wx.ALL, 5)
textTime = wx.StaticText(self, label="Last scan")
gridScan.Add(textTime, (8, 0), (1, 1), wx.ALL, 5)
textCtrlTime = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
if scanInfo.timeLast is not None:
textCtrlTime.SetValue(format_time(scanInfo.timeLast, True))
gridScan.Add(textCtrlTime, (8, 1), (1, 1), wx.ALL, 5)
textLat = wx.StaticText(self, label="Latitude")
gridScan.Add(textLat, (9, 0), (1, 1), wx.ALL, 5)
self.textCtrlLat = wx.TextCtrl(self, value="Unknown")
self.textCtrlLat.SetValidator(ValidatorCoord(True))
if scanInfo.lat is not None:
self.textCtrlLat.SetValue(str(scanInfo.lat))
gridScan.Add(self.textCtrlLat, (9, 1), (1, 1), wx.ALL, 5)
textLon = wx.StaticText(self, label="Longitude")
gridScan.Add(textLon, (10, 0), (1, 1), wx.ALL, 5)
self.textCtrlLon = wx.TextCtrl(self, value="Unknown")
self.textCtrlLon.SetValidator(ValidatorCoord(False))
if scanInfo.lon is not None:
self.textCtrlLon.SetValue(str(scanInfo.lon))
gridScan.Add(self.textCtrlLon, (10, 1), (1, 1), wx.ALL, 5)
boxScan.Add(gridScan, 0, 0, 5)
grid.Add(boxScan, (0, 0), (1, 1), wx.ALL | wx.EXPAND, 5)
boxDevice = wx.StaticBoxSizer(wx.StaticBox(self, label="Device"),
wx.VERTICAL)
gridDevice = wx.GridBagSizer(0, 0)
textName = wx.StaticText(self, label="Name")
gridDevice.Add(textName, (0, 0), (1, 1), wx.ALL, 5)
textCtrlName = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
if scanInfo.name is not None:
textCtrlName.SetValue(scanInfo.name)
gridDevice.Add(textCtrlName, (0, 1), (1, 2), wx.ALL | wx.EXPAND, 5)
textTuner = wx.StaticText(self, label="Tuner")
gridDevice.Add(textTuner, (1, 0), (1, 1), wx.ALL, 5)
textCtrlTuner = wx.TextCtrl(self, value="Unknown",
style=wx.TE_READONLY)
if scanInfo.tuner != -1:
textCtrlTuner.SetValue(TUNER[scanInfo.tuner])
gridDevice.Add(textCtrlTuner, (1, 1), (1, 2), wx.ALL | wx.EXPAND, 5)
testGain = wx.StaticText(self, label="Gain")
gridDevice.Add(testGain, (2, 0), (1, 1), wx.ALL, 5)
textCtrlGain = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
if scanInfo.gain is not None:
textCtrlGain.SetValue(str(scanInfo.gain))
gridDevice.Add(textCtrlGain, (2, 1), (1, 1), wx.ALL, 5)
textDb = wx.StaticText(self, label="dB")
gridDevice.Add(textDb, (2, 2), (1, 1), wx.ALL, 5)
textLo = wx.StaticText(self, label="LO")
gridDevice.Add(textLo, (3, 0), (1, 1), wx.ALL, 5)
textCtrlLo = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
if scanInfo.lo is not None:
textCtrlLo.SetValue(str(scanInfo.lo))
gridDevice.Add(textCtrlLo, (3, 1), (1, 1), wx.ALL, 5)
textMHz3 = wx.StaticText(self, label="MHz")
gridDevice.Add(textMHz3, (3, 2), (1, 1), wx.ALL, 5)
textCal = wx.StaticText(self, label="Calibration")
gridDevice.Add(textCal, (4, 0), (1, 1), wx.ALL, 5)
textCtrlCal = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
if scanInfo.calibration is not None:
textCtrlCal.SetValue(str(scanInfo.calibration))
gridDevice.Add(textCtrlCal, (4, 1), (1, 1), wx.ALL, 5)
testPpm = wx.StaticText(self, label="ppm")
gridDevice.Add(testPpm, (4, 2), (1, 1), wx.ALL, 5)
boxDevice.Add(gridDevice, 1, wx.EXPAND, 5)
grid.Add(boxDevice, (1, 0), (1, 1), wx.ALL | wx.EXPAND, 5)
box.Add(grid, 1, wx.ALL | wx.EXPAND, 5)
sizerButtons = wx.StdDialogButtonSizer()
buttonOk = wx.Button(self, wx.ID_OK)
buttonCancel = wx.Button(self, wx.ID_CANCEL)
sizerButtons.AddButton(buttonOk)
sizerButtons.AddButton(buttonCancel)
sizerButtons.Realize()
self.Bind(wx.EVT_BUTTON, self.__on_ok, buttonOk)
box.Add(sizerButtons, 0, wx.ALIGN_RIGHT | wx.ALL, 5)
self.SetSizerAndFit(box)
def __init__(self, parent, spectrum, settings):
self.spectrum = spectrum
self.settings = settings
self.sweeps = None
self.isExporting = False
wx.Dialog.__init__(self, parent=parent, title='Export Plot Sequence')
self.queue = Queue.Queue()
self.timer = wx.Timer(self)
self.Bind(wx.EVT_TIMER, self.__on_timer, self.timer)
self.timer.Start(self.POLL)
self.figure = matplotlib.figure.Figure(facecolor='white')
self.canvas = FigureCanvas(self, -1, self.figure)
self.plot = Plotter(self.queue, self.figure, settings)
textPlot = wx.StaticText(self, label='Plot')
self.checkAxes = wx.CheckBox(self, label='Axes')
self.checkAxes.SetValue(True)
self.Bind(wx.EVT_CHECKBOX, self.__on_axes, self.checkAxes)
self.checkGrid = wx.CheckBox(self, label='Grid')
self.checkGrid.SetValue(True)
self.Bind(wx.EVT_CHECKBOX, self.__on_grid, self.checkGrid)
self.checkBar = wx.CheckBox(self, label='Bar')
self.checkBar.SetValue(True)
self.Bind(wx.EVT_CHECKBOX, self.__on_bar, self.checkBar)
sizerCheck = wx.BoxSizer(wx.HORIZONTAL)
sizerCheck.Add(self.checkAxes, flag=wx.ALL, border=5)
sizerCheck.Add(self.checkGrid, flag=wx.ALL, border=5)
sizerCheck.Add(self.checkBar, flag=wx.ALL, border=5)
textRange = wx.StaticText(self, label='Range')
self.sweepTimeStamps = sorted([timeStamp for timeStamp in spectrum.keys()])
sweepChoices = [format_time(timeStamp, True) for timeStamp in self.sweepTimeStamps]
textStart = wx.StaticText(self, label="Start")
self.choiceStart = wx.Choice(self, choices=sweepChoices)
self.choiceStart.SetSelection(0)
self.Bind(wx.EVT_CHOICE, self.__on_choice, self.choiceStart)
textEnd = wx.StaticText(self, label="End")
self.choiceEnd = wx.Choice(self, choices=sweepChoices)
self.choiceEnd.SetSelection(len(self.sweepTimeStamps) - 1)
self.Bind(wx.EVT_CHOICE, self.__on_choice, self.choiceEnd)
textSweeps = wx.StaticText(self, label='Sweeps')
self.textSweeps = wx.StaticText(self, label="")
textOutput = wx.StaticText(self, label='Output')
self.textSize = wx.StaticText(self)
buttonSize = wx.Button(self, label='Change...')
buttonSize.SetToolTipString('Change exported image size')
self.Bind(wx.EVT_BUTTON, self.__on_imagesize, buttonSize)
self.__show_image_size()
buttonBrowse = wx.Button(self, label='Browse...')
self.Bind(wx.EVT_BUTTON, self.__on_browse, buttonBrowse)
self.editDir = wx.TextCtrl(self)
self.editDir.SetValue(settings.dirExport)
font = textPlot.GetFont()
fontSize = font.GetPointSize()
font.SetPointSize(fontSize + 4)
textPlot.SetFont(font)
textRange.SetFont(font)
textOutput.SetFont(font)
sizerButtons = wx.StdDialogButtonSizer()
buttonOk = wx.Button(self, wx.ID_OK)
buttonCancel = wx.Button(self, wx.ID_CANCEL)
sizerButtons.AddButton(buttonOk)
sizerButtons.AddButton(buttonCancel)
sizerButtons.Realize()
self.Bind(wx.EVT_BUTTON, self.__on_ok, buttonOk)
sizerGrid = wx.GridBagSizer(5, 5)
sizerGrid.Add(self.canvas, pos=(0, 0), span=(10, 6),
flag=wx.EXPAND | wx.ALL, border=5)
sizerGrid.Add(textPlot, pos=(0, 7),
flag=wx.TOP | wx.BOTTOM, border=5)
sizerGrid.Add(sizerCheck, pos=(1, 7), span=(1, 2),
flag=wx.ALL, border=5)
sizerGrid.Add(textRange, pos=(2, 7),
flag=wx.TOP | wx.BOTTOM, border=5)
sizerGrid.Add(textStart, pos=(3, 7),
flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
sizerGrid.Add(self.choiceStart, pos=(3, 8),
flag=wx.ALL, border=5)
sizerGrid.Add(textEnd, pos=(4, 7),
flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
sizerGrid.Add(self.choiceEnd, pos=(4, 8),
flag=wx.ALL, border=5)
sizerGrid.Add(textSweeps, pos=(5, 7),
flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
sizerGrid.Add(self.textSweeps, pos=(5, 8),
flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
sizerGrid.Add(textOutput, pos=(6, 7),
flag=wx.TOP | wx.BOTTOM, border=5)
sizerGrid.Add(self.textSize, pos=(7, 7),
flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
sizerGrid.Add(buttonSize, pos=(7, 8),
flag=wx.ALL, border=5)
sizerGrid.Add(self.editDir, pos=(8, 7), span=(1, 2),
flag=wx.ALL | wx.EXPAND, border=5)
sizerGrid.Add(buttonBrowse, pos=(9, 7),
flag=wx.ALL, border=5)
sizerGrid.Add(sizerButtons, pos=(10, 7), span=(1, 2),
flag=wx.ALIGN_RIGHT | wx.ALL, border=5)
self.SetSizerAndFit(sizerGrid)
self.__draw_plot()
def train_progressive_gan(
G_smoothing = 0.999, # Exponential running average of generator weights.
D_repeats = 1, # How many times the discriminator is trained per G iteration.
minibatch_repeats = 4, # Number of minibatches to run before adjusting training parameters.
reset_opt_for_new_lod = True, # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
total_kimg = 15000, # Total length of the training, measured in thousands of real images.
mirror_augment = False, # Enable mirror augment?
drange_net = [-1,1], # Dynamic range used when feeding image data to the networks.
image_snapshot_ticks = 1, # How often to export image snapshots?
network_snapshot_ticks = 10, # How often to export network snapshots?
save_tf_graph = False, # Include full TensorFlow computation graph in the tfevents file?
save_weight_histograms = False, # Include weight histograms in the tfevents file?
resume_run_id = None, # Run ID or network pkl to resume training from, None = start from scratch.
resume_snapshot = None, # Snapshot index to resume training from, None = autodetect.
resume_kimg = 0.0, # Assumed training progress at the beginning. Affects reporting and training schedule.
resume_time = 0.0): # Assumed wallclock time at the beginning. Affects reporting.
maintenance_start_time = time.time()
training_set = dataset.load_dataset(data_dir=config.data_dir, verbose=True, **config.dataset)
# Construct networks.
with tf.device('/gpu:0'):
if resume_run_id is not None:
network_pkl = misc.locate_network_pkl(resume_run_id, resume_snapshot)
print('Loading networks from "%s"...' % network_pkl)
G, D, Gs = misc.load_pkl(network_pkl)
else:
print('Constructing networks...')
G = tfutil.Network('G', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **config.G)
D = tfutil.Network('D', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **config.D)
Gs = G.clone('Gs')
Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
G.print_layers(); D.print_layers()
print('Building TensorFlow graph...')
with tf.name_scope('Inputs'):
lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
minibatch_split = minibatch_in // config.num_gpus
reals, labels = training_set.get_minibatch_tf()
reals_split = tf.split(reals, config.num_gpus)
labels_split = tf.split(labels, config.num_gpus)
G_opt = tfutil.Optimizer(name='TrainG', learning_rate=lrate_in, **config.G_opt)
D_opt = tfutil.Optimizer(name='TrainD', learning_rate=lrate_in, **config.D_opt)
for gpu in range(config.num_gpus):
with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
lod_assign_ops = [tf.assign(G_gpu.find_var('lod'), lod_in), tf.assign(D_gpu.find_var('lod'), lod_in)]
reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment, training_set.dynamic_range, drange_net)
labels_gpu = labels_split[gpu]
with tf.name_scope('G_loss'), tf.control_dependencies(lod_assign_ops):
G_loss = tfutil.call_func_by_name(G=G_gpu, D=D_gpu, opt=G_opt, training_set=training_set, minibatch_size=minibatch_split, **config.G_loss)
with tf.name_scope('D_loss'), tf.control_dependencies(lod_assign_ops):
D_loss = tfutil.call_func_by_name(G=G_gpu, D=D_gpu, opt=D_opt, training_set=training_set, minibatch_size=minibatch_split, reals=reals_gpu, labels=labels_gpu, **config.D_loss)
G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
G_train_op = G_opt.apply_updates()
D_train_op = D_opt.apply_updates()
print('Setting up snapshot image grid...')
grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(G, training_set, **config.grid)
sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
grid_fakes = Gs.run(grid_latents, grid_labels, minibatch_size=sched.minibatch//config.num_gpus)
print('Setting up result dir...')
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
misc.save_image_grid(grid_reals, os.path.join(result_subdir, 'reals.png'), drange=training_set.dynamic_range, grid_size=grid_size)
misc.save_image_grid(grid_fakes, os.path.join(result_subdir, 'fakes%06d.png' % 0), drange=drange_net, grid_size=grid_size)
summary_log = tf.summary.FileWriter(result_subdir)
if save_tf_graph:
summary_log.add_graph(tf.get_default_graph())
if save_weight_histograms:
G.setup_weight_histograms(); D.setup_weight_histograms()
print('Training...')
cur_nimg = int(resume_kimg * 1000)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
train_start_time = tick_start_time - resume_time
prev_lod = -1.0
while cur_nimg < total_kimg * 1000:
# Choose training parameters and configure training ops.
sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
training_set.configure(sched.minibatch, sched.lod)
if reset_opt_for_new_lod:
if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(sched.lod) != np.ceil(prev_lod):
G_opt.reset_optimizer_state(); D_opt.reset_optimizer_state()
prev_lod = sched.lod
# Run training ops.
for repeat in range(minibatch_repeats):
for _ in range(D_repeats):
tfutil.run([D_train_op, Gs_update_op], {lod_in: sched.lod, lrate_in: sched.D_lrate, minibatch_in: sched.minibatch})
cur_nimg += sched.minibatch
tfutil.run([G_train_op], {lod_in: sched.lod, lrate_in: sched.G_lrate, minibatch_in: sched.minibatch})
# Perform maintenance tasks once per tick.
done = (cur_nimg >= total_kimg * 1000)
if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
total_time = cur_time - train_start_time
maintenance_time = tick_start_time - maintenance_start_time
maintenance_start_time = cur_time
# Report progress.
print('tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f' % (
tfutil.autosummary('Progress/tick', cur_tick),
tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
tfutil.autosummary('Progress/lod', sched.lod),
tfutil.autosummary('Progress/minibatch', sched.minibatch),
misc.format_time(tfutil.autosummary('Timing/total_sec', total_time)),
tfutil.autosummary('Timing/sec_per_tick', tick_time),
tfutil.autosummary('Timing/sec_per_kimg', tick_time / tick_kimg),
tfutil.autosummary('Timing/maintenance_sec', maintenance_time)))
tfutil.autosummary('Timing/total_hours', total_time / (60.0 * 60.0))
tfutil.autosummary('Timing/total_days', total_time / (24.0 * 60.0 * 60.0))
tfutil.save_summaries(summary_log, cur_nimg)
# Save snapshots.
if cur_tick % image_snapshot_ticks == 0 or done:
grid_fakes = Gs.run(grid_latents, grid_labels, minibatch_size=sched.minibatch//config.num_gpus)
misc.save_image_grid(grid_fakes, os.path.join(result_subdir, 'fakes%06d.png' % (cur_nimg // 1000)), drange=drange_net, grid_size=grid_size)
if cur_tick % network_snapshot_ticks == 0 or done:
misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))
# Record start time of the next tick.
tick_start_time = time.time()
# Write final results.
misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
summary_log.close()
open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
def run(self):
self.parent.clear_plots()
if self.data is None:
self.parent.threadPlot = None
return
tMin = format_time(self.extent.tMin, True)
tMax = format_time(self.extent.tMax, True)
fMin = format_precision(self.settings, freq=self.extent.fMin,
fancyUnits=True)
fMax = format_precision(self.settings, freq=self.extent.fMax,
fancyUnits=True)
lMin = format_precision(self.settings, level=self.extent.lMin,
fancyUnits=True)
lMax = format_precision(self.settings, level=self.extent.lMax,
fancyUnits=True)
peak = self.extent.get_peak_flt()
peakF = format_precision(self.settings, freq=peak[0],
fancyUnits=True)
peakL = format_precision(self.settings, level=peak[1],
fancyUnits=True)
peakT = format_time(peak[2], True)
text = [['Sweeps', '', len(self.data)],
['Extents', '', ''],
['', 'Start', tMin],
['', 'End', tMax],
['', 'Min frequency', fMin],
['', 'Max frequency', fMax],
['', 'Min level', lMin],
['', 'Max level', lMax],
['Peak', '', ''],
['', 'Level', peakL],
['', 'Frequency', peakF],
['', 'Time', peakT],
]
table = Table(self.axes, loc='center', gid='table')
rows = len(text)
cols = len(text[0])
for row in xrange(rows):
for col in xrange(cols):
table.add_cell(row, col,
text=text[row][col],
width=1.0 / cols, height=1.0 / rows)
if self.settings.grid:
colour = 'LightGray'
else:
colour = 'w'
set_table_colour(table, colour)
for i in range(3):
table.auto_set_column_width(i)
self.axes.add_table(table)
noData = find_artists(self.axes, 'noData')
noData[0].set_alpha(0)
self.parent.redraw_plot()
self.parent.threadPlot = None
def train_progressive_gan(
G_smoothing=0.999, # Exponential running average of generator weights.
D_repeats=1, # How many times the discriminator is trained per G iteration.
minibatch_repeats=4, # Number of minibatches to run before adjusting training parameters.
reset_opt_for_new_lod=True, # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
total_kimg=15000, # Total length of the training, measured in thousands of real images.
mirror_augment=False, # Enable mirror augment?
drange_net=[
-1, 1
], # Dynamic range used when feeding image data to the networks.
image_snapshot_ticks=1, # How often to export image snapshots?
network_snapshot_ticks=10, # How often to export network snapshots?
compute_fid_score=False, # Compute FID during training once sched.lod=0.0
minimum_fid_kimg=0, # Compute FID after
fid_snapshot_ticks=1, # How often to compute FID
fid_patience=2, # When to end training based on FID
save_tf_graph=False, # Include full TensorFlow computation graph in the tfevents file?
save_weight_histograms=False, # Include weight histograms in the tfevents file?
resume_run_id=None, # Run ID or network pkl to resume training from, None = start from scratch.
resume_snapshot=None, # Snapshot index to resume training from, None = autodetect.
resume_kimg=0.0, # Assumed training progress at the beginning. Affects reporting and training schedule.
resume_time=0.0, # Assumed wallclock time at the beginning. Affects reporting.
result_subdir="./"):
maintenance_start_time = time.time()
training_set = dataset.load_dataset(data_dir=config.data_dir,
verbose=True,
**config.dataset)
# Construct networks.
with tf.device('/gpu:0'):
if resume_run_id != "None":
network_pkl = misc.locate_network_pkl(resume_run_id,
resume_snapshot)
resume_pkl_name = os.path.splitext(
os.path.basename(network_pkl))[0]
try:
resume_kimg = int(resume_pkl_name.split('-')[-1])
print('** Setting resume kimg to', resume_kimg, flush=True)
except:
print('** Keeping resume kimg as:', resume_kimg, flush=True)
print('Loading networks from "%s"...' % network_pkl, flush=True)
G, D, Gs = misc.load_pkl(network_pkl)
else:
print('Constructing networks...', flush=True)
G = tfutil.Network('G',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.G)
D = tfutil.Network('D',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.D)
Gs = G.clone('Gs')
Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
G.print_layers()
D.print_layers()
print('Building TensorFlow graph...', flush=True)
with tf.name_scope('Inputs'):
lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
minibatch_split = minibatch_in // config.num_gpus
reals, labels = training_set.get_minibatch_tf()
reals_split = tf.split(reals, config.num_gpus)
labels_split = tf.split(labels, config.num_gpus)
G_opt = tfutil.Optimizer(name='TrainG',
learning_rate=lrate_in,
**config.G_opt)
D_opt = tfutil.Optimizer(name='TrainD',
learning_rate=lrate_in,
**config.D_opt)
for gpu in range(config.num_gpus):
with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment,
training_set.dynamic_range, drange_net)
labels_gpu = labels_split[gpu]
with tf.name_scope('G_loss'):
G_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
opt=G_opt,
training_set=training_set,
minibatch_size=minibatch_split,
**config.G_loss)
with tf.name_scope('D_loss'):
D_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
opt=D_opt,
training_set=training_set,
minibatch_size=minibatch_split,
reals=reals_gpu,
labels=labels_gpu,
**config.D_loss)
G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
G_train_op = G_opt.apply_updates()
D_train_op = D_opt.apply_updates()
print('Setting up snapshot image grid...', flush=True)
grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(
G, training_set, **config.grid)
sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
grid_fakes = Gs.run(grid_latents,
grid_labels,
minibatch_size=sched.minibatch // config.num_gpus)
print('Setting up result dir...', flush=True)
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
misc.save_image_grid(grid_reals,
os.path.join(result_subdir, 'reals.png'),
drange=training_set.dynamic_range,
grid_size=grid_size)
misc.save_image_grid(grid_fakes,
os.path.join(result_subdir, 'fakes%06d.png' % 0),
drange=drange_net,
grid_size=grid_size)
summary_log = tf.summary.FileWriter(result_subdir)
if save_tf_graph:
summary_log.add_graph(tf.get_default_graph())
if save_weight_histograms:
G.setup_weight_histograms()
D.setup_weight_histograms()
print('Training...', flush=True)
# FID patience parameters:
fid_list = []
fid_steps = 0
fid_stop = False
fid_patience_step = 0
cur_nimg = int(resume_kimg * 1000)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
train_start_time = tick_start_time - resume_time
prev_lod = -1.0
while cur_nimg < total_kimg * 1000:
# Choose training parameters and configure training ops.
sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
training_set.configure(sched.minibatch, sched.lod)
if reset_opt_for_new_lod:
if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
sched.lod) != np.ceil(prev_lod):
G_opt.reset_optimizer_state()
D_opt.reset_optimizer_state()
prev_lod = sched.lod
# Run training ops.
for repeat in range(minibatch_repeats):
for _ in range(D_repeats):
tfutil.run(
[D_train_op, Gs_update_op], {
lod_in: sched.lod,
lrate_in: sched.D_lrate,
minibatch_in: sched.minibatch
})
cur_nimg += sched.minibatch
tfutil.run(
[G_train_op], {
lod_in: sched.lod,
lrate_in: sched.G_lrate,
minibatch_in: sched.minibatch
})
# Perform maintenance tasks once per tick.
done = (cur_nimg >= total_kimg * 1000)
if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
total_time = cur_time - train_start_time
maintenance_time = tick_start_time - maintenance_start_time
maintenance_start_time = cur_time
if (compute_fid_score
== True) and (cur_tick % fid_snapshot_ticks
== 0) and (sched.lod == 0.0) and (
cur_nimg >= minimum_fid_kimg * 1000):
fid = compute_fid(Gs=Gs,
minibatch_size=sched.minibatch,
dataset_obj=training_set,
iter_number=cur_nimg / 1000,
lod=0.0,
num_images=10000,
printing=False)
fid_list.append(fid)
# Report progress without FID.
print(
'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f'
% (tfutil.autosummary('Progress/tick', cur_tick),
tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
tfutil.autosummary('Progress/lod', sched.lod),
tfutil.autosummary('Progress/minibatch', sched.minibatch),
misc.format_time(
tfutil.autosummary('Timing/total_sec', total_time)),
tfutil.autosummary('Timing/sec_per_tick', tick_time),
tfutil.autosummary('Timing/sec_per_kimg',
tick_time / tick_kimg),
tfutil.autosummary('Timing/maintenance_sec',
maintenance_time)),
flush=True)
tfutil.autosummary('Timing/total_hours',
total_time / (60.0 * 60.0))
tfutil.autosummary('Timing/total_days',
total_time / (24.0 * 60.0 * 60.0))
tfutil.save_summaries(summary_log, cur_nimg)
# Save image snapshots.
if cur_tick % image_snapshot_ticks == 0 or done:
grid_fakes = Gs.run(grid_latents,
grid_labels,
minibatch_size=sched.minibatch //
config.num_gpus)
misc.save_image_grid(grid_fakes,
os.path.join(
result_subdir,
'fakes%06d.png' % (cur_nimg // 1000)),
drange=drange_net,
grid_size=grid_size)
# Save network snapshots
if cur_tick % network_snapshot_ticks == 0 or done or (
compute_fid_score
== True) and (cur_tick % fid_snapshot_ticks == 0) and (
cur_nimg >= minimum_fid_kimg * 1000):
misc.save_pkl(
(G, D, Gs),
os.path.join(
result_subdir,
'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))
# End training based on FID patience
if (compute_fid_score
== True) and (cur_tick % fid_snapshot_ticks
== 0) and (sched.lod == 0.0) and (
cur_nimg >= minimum_fid_kimg * 1000):
fid_patience_step += 1
if len(fid_list) == 1:
fid_patience_step = 0
misc.save_pkl((G, D, Gs),
os.path.join(result_subdir,
'network-final-full-conv.pkl'))
print(
"Save network-final-full-conv for FID: %.3f at kimg %-8.1f."
% (fid_list[-1], cur_nimg // 1000),
flush=True)
else:
if fid_list[-1] < np.min(fid_list[:-1]):
fid_patience_step = 0
misc.save_pkl(
(G, D, Gs),
os.path.join(result_subdir,
'network-final-full-conv.pkl'))
print(
"Save network-final-full-conv for FID: %.3f at kimg %-8.1f."
% (fid_list[-1], cur_nimg // 1000),
flush=True)
else:
print("No improvement for FID: %.3f at kimg %-8.1f." %
(fid_list[-1], cur_nimg // 1000),
flush=True)
if fid_patience_step == fid_patience:
fid_stop = True
print("Training stopped due to FID early-stopping.",
flush=True)
cur_nimg = total_kimg * 1000
# Record start time of the next tick.
tick_start_time = time.time()
# Write final results.
# Save final only if FID-Stopping has not happend:
if fid_stop == False:
fid = compute_fid(Gs=Gs,
minibatch_size=sched.minibatch,
dataset_obj=training_set,
iter_number=cur_nimg / 1000,
lod=0.0,
num_images=10000,
printing=False)
print("Final FID: %.3f at kimg %-8.1f." % (fid, cur_nimg // 1000),
flush=True)
### save final FID to .csv file in result_parent_dir
csv_file = os.path.join(
os.path.dirname(os.path.dirname(result_subdir)),
"results_full_conv.csv")
list_to_append = [
result_subdir.split("/")[-2] + "/" + result_subdir.split("/")[-1],
fid
]
with open(csv_file, 'a') as f_object:
writer_object = writer(f_object)
writer_object.writerow(list_to_append)
f_object.close()
misc.save_pkl((G, D, Gs),
os.path.join(result_subdir,
'network-final-full-conv.pkl'))
print("Save network-final-full-conv.", flush=True)
summary_log.close()
open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
def train_classifier(
smoothing=0.999, # Exponential running average of encoder weights.
minibatch_repeats=4, # Number of minibatches to run before adjusting training parameters.
reset_opt_for_new_lod=True, # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
total_kimg=25000, # Total length of the training, measured in thousands of real images.
lr_mirror_augment=True, # Enable mirror augment?
ud_mirror_augment=False, # Enable up-down mirror augment?
drange_net=[
-1, 1
], # Dynamic range used when feeding image data to the networks.
image_snapshot_ticks=10, # How often to export image snapshots?
save_tf_graph=False, # Include full TensorFlow computation graph in the tfevents file?
save_weight_histograms=False
): # Include weight histograms in the tfevents file?
maintenance_start_time = time.time()
training_set = dataset.load_dataset(data_dir=config.data_dir,
verbose=True,
**config.training_set)
validation_set = dataset.load_dataset(data_dir=config.data_dir,
verbose=True,
**config.validation_set)
network_snapshot_ticks = total_kimg // 100 # How often to export network snapshots?
# Construct networks.
with tf.device('/gpu:0'):
try:
network_pkl = misc.locate_network_pkl()
resume_kimg, resume_time = misc.resume_kimg_time(network_pkl)
print('Loading networks from "%s"...' % network_pkl)
EG, D_rec, EGs = misc.load_pkl(network_pkl)
except:
print('Constructing networks...')
resume_kimg = 0.0
resume_time = 0.0
EG = tfutil.Network('EG',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.EG)
D_rec = tfutil.Network('D_rec',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
**config.D_rec)
EGs = EG.clone('EGs')
EGs_update_op = EGs.setup_as_moving_average_of(EG, beta=smoothing)
EG.print_layers()
D_rec.print_layers()
print('Building TensorFlow graph...')
with tf.name_scope('Inputs'):
lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
minibatch_split = minibatch_in // config.num_gpus
reals, labels = training_set.get_minibatch_tf()
reals_split = tf.split(reals, config.num_gpus)
labels_split = tf.split(labels, config.num_gpus)
EG_opt = tfutil.Optimizer(name='TrainEG',
learning_rate=lrate_in,
**config.EG_opt)
D_rec_opt = tfutil.Optimizer(name='TrainD_rec',
learning_rate=lrate_in,
**config.D_rec_opt)
for gpu in range(config.num_gpus):
with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
EG_gpu = EG if gpu == 0 else EG.clone(EG.name + '_shadow_%d' % gpu)
D_rec_gpu = D_rec if gpu == 0 else D_rec.clone(D_rec.name +
'_shadow_%d' % gpu)
reals_fade_gpu, reals_orig_gpu = process_reals(
reals_split[gpu], lod_in, lr_mirror_augment, ud_mirror_augment,
training_set.dynamic_range, drange_net)
labels_gpu = labels_split[gpu]
with tf.name_scope('EG_loss'):
EG_loss = tfutil.call_func_by_name(EG=EG_gpu,
D_rec=D_rec_gpu,
reals_orig=reals_orig_gpu,
labels=labels_gpu,
**config.EG_loss)
with tf.name_scope('D_rec_loss'):
D_rec_loss = tfutil.call_func_by_name(
EG=EG_gpu,
D_rec=D_rec_gpu,
D_rec_opt=D_rec_opt,
minibatch_size=minibatch_split,
reals_orig=reals_orig_gpu,
**config.D_rec_loss)
EG_opt.register_gradients(tf.reduce_mean(EG_loss),
EG_gpu.trainables)
D_rec_opt.register_gradients(tf.reduce_mean(D_rec_loss),
D_rec_gpu.trainables)
EG_train_op = EG_opt.apply_updates()
D_rec_train_op = D_rec_opt.apply_updates()
print('Setting up snapshot image grid...')
grid_size, train_reals, train_labels = setup_snapshot_image_grid(
training_set, drange_net, [450, 10], **config.grid)
grid_size, val_reals, val_labels = setup_snapshot_image_grid(
validation_set, drange_net, [450, 10], **config.grid)
sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
train_recs, train_fingerprints, train_logits = EGs.run(
train_reals, minibatch_size=sched.minibatch // config.num_gpus)
train_preds = np.argmax(train_logits, axis=1)
train_gt = np.argmax(train_labels, axis=1)
train_acc = np.float32(np.sum(train_gt == train_preds)) / np.float32(
len(train_gt))
print('Training Accuracy = %f' % train_acc)
val_recs, val_fingerprints, val_logits = EGs.run(
val_reals, minibatch_size=sched.minibatch // config.num_gpus)
val_preds = np.argmax(val_logits, axis=1)
val_gt = np.argmax(val_labels, axis=1)
val_acc = np.float32(np.sum(val_gt == val_preds)) / np.float32(len(val_gt))
print('Validation Accuracy = %f' % val_acc)
print('Setting up result dir...')
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
misc.save_image_grid(train_reals[::30, :, :, :],
os.path.join(result_subdir, 'train_reals.png'),
drange=drange_net,
grid_size=[15, 10])
misc.save_image_grid(train_recs[::30, :, :, :],
os.path.join(result_subdir, 'train_recs-init.png'),
drange=drange_net,
grid_size=[15, 10])
misc.save_image_grid(train_fingerprints[::30, :, :, :],
os.path.join(result_subdir,
'train_fingerrints-init.png'),
drange=drange_net,
grid_size=[15, 10])
misc.save_image_grid(val_reals[::30, :, :, :],
os.path.join(result_subdir, 'val_reals.png'),
drange=drange_net,
grid_size=[15, 10])
misc.save_image_grid(val_recs[::30, :, :, :],
os.path.join(result_subdir, 'val_recs-init.png'),
drange=drange_net,
grid_size=[15, 10])
misc.save_image_grid(val_fingerprints[::30, :, :, :],
os.path.join(result_subdir,
'val_fingerrints-init.png'),
drange=drange_net,
grid_size=[15, 10])
est_fingerprints = np.transpose(
EGs.vars['Conv_fingerprints/weight'].eval(), axes=[3, 2, 0, 1])
misc.save_image_grid(
est_fingerprints,
os.path.join(result_subdir, 'est_fingerrints-init.png'),
drange=[np.amin(est_fingerprints),
np.amax(est_fingerprints)],
grid_size=[est_fingerprints.shape[0], 1])
summary_log = tf.summary.FileWriter(result_subdir)
if save_tf_graph:
summary_log.add_graph(tf.get_default_graph())
if save_weight_histograms:
EG.setup_weight_histograms()
D_rec.setup_weight_histograms()
print('Training...')
cur_nimg = int(resume_kimg * 1000)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
train_start_time = tick_start_time - resume_time
prev_lod = -1.0
while cur_nimg < total_kimg * 1000:
# Choose training parameters and configure training ops.
sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
training_set.configure(sched.minibatch, sched.lod)
if reset_opt_for_new_lod:
if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
sched.lod) != np.ceil(prev_lod):
EG_opt.reset_optimizer_state()
D_rec_opt.reset_optimizer_state()
prev_lod = sched.lod
# Run training ops.
for repeat in range(minibatch_repeats):
tfutil.run(
[D_rec_train_op], {
lod_in: sched.lod,
lrate_in: sched.lrate,
minibatch_in: sched.minibatch
})
tfutil.run(
[EG_train_op], {
lod_in: sched.lod,
lrate_in: sched.lrate,
minibatch_in: sched.minibatch
})
tfutil.run([EGs_update_op], {})
cur_nimg += sched.minibatch
# Perform maintenance tasks once per tick.
done = (cur_nimg >= total_kimg * 1000)
if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
total_time = cur_time - train_start_time
maintenance_time = tick_start_time - maintenance_start_time
maintenance_start_time = cur_time
# Report progress.
print(
'tick %-5d kimg %-8.1f lod %-5.2f resolution %-4d minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f'
% (tfutil.autosummary('Progress/tick', cur_tick),
tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
tfutil.autosummary('Progress/lod', sched.lod),
tfutil.autosummary('Progress/resolution', sched.resolution),
tfutil.autosummary('Progress/minibatch', sched.minibatch),
misc.format_time(
tfutil.autosummary('Timing/total_sec', total_time)),
tfutil.autosummary('Timing/sec_per_tick', tick_time),
tfutil.autosummary('Timing/sec_per_kimg',
tick_time / tick_kimg),
tfutil.autosummary('Timing/maintenance_sec',
maintenance_time)))
tfutil.autosummary('Timing/total_hours',
total_time / (60.0 * 60.0))
tfutil.autosummary('Timing/total_days',
total_time / (24.0 * 60.0 * 60.0))
tfutil.save_summaries(summary_log, cur_nimg)
# Print accuracy.
if cur_tick % image_snapshot_ticks == 0 or done:
train_recs, train_fingerprints, train_logits = EGs.run(
train_reals,
minibatch_size=sched.minibatch // config.num_gpus)
train_preds = np.argmax(train_logits, axis=1)
train_gt = np.argmax(train_labels, axis=1)
train_acc = np.float32(np.sum(
train_gt == train_preds)) / np.float32(len(train_gt))
print('Training Accuracy = %f' % train_acc)
val_recs, val_fingerprints, val_logits = EGs.run(
val_reals,
minibatch_size=sched.minibatch // config.num_gpus)
val_preds = np.argmax(val_logits, axis=1)
val_gt = np.argmax(val_labels, axis=1)
val_acc = np.float32(np.sum(val_gt == val_preds)) / np.float32(
len(val_gt))
print('Validation Accuracy = %f' % val_acc)
misc.save_image_grid(train_recs[::30, :, :, :],
os.path.join(result_subdir,
'train_recs-final.png'),
drange=drange_net,
grid_size=[15, 10])
misc.save_image_grid(train_fingerprints[::30, :, :, :],
os.path.join(
result_subdir,
'train_fingerrints-final.png'),
drange=drange_net,
grid_size=[15, 10])
misc.save_image_grid(val_recs[::30, :, :, :],
os.path.join(result_subdir,
'val_recs-final.png'),
drange=drange_net,
grid_size=[15, 10])
misc.save_image_grid(val_fingerprints[::30, :, :, :],
os.path.join(result_subdir,
'val_fingerrints-final.png'),
drange=drange_net,
grid_size=[15, 10])
est_fingerprints = np.transpose(
EGs.vars['Conv_fingerprints/weight'].eval(),
axes=[3, 2, 0, 1])
misc.save_image_grid(est_fingerprints,
os.path.join(result_subdir,
'est_fingerrints-final.png'),
drange=[
np.amin(est_fingerprints),
np.amax(est_fingerprints)
],
grid_size=[est_fingerprints.shape[0], 1])
if cur_tick % network_snapshot_ticks == 0 or done:
misc.save_pkl(
(EG, D_rec, EGs),
os.path.join(
result_subdir,
'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))
# Record start time of the next tick.
tick_start_time = time.time()
# Write final results.
misc.save_pkl((EG, D_rec, EGs),
os.path.join(result_subdir, 'network-final.pkl'))
summary_log.close()
open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
def train_progressive_gan(
G_smoothing=0.999, # Exponential running average of generator weights.
D_repeats=1, # How many times the discriminator is trained per G iteration.
minibatch_repeats=4, # Number of minibatches to run before adjusting training parameters.
reset_opt_for_new_lod=True, # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
total_kimg=15000, # Total length of the training, measured in thousands of real images.
mirror_augment=False, # Enable mirror augment?
drange_net=[
-1, 1
], # Dynamic range used when feeding image data to the networks.
image_snapshot_ticks=1, # How often to export image snapshots?
network_snapshot_ticks=10, # How often to export network snapshots?
save_tf_graph=False, # Include full TensorFlow computation graph in the tfevents file?
save_weight_histograms=False, # Include weight histograms in the tfevents file?
resume_run_id=None, # Run ID or network pkl to resume training from, None = start from scratch.
resume_snapshot=None, # Snapshot index to resume training from, None = autodetect.
resume_kimg=0.0, # Assumed training progress at the beginning. Affects reporting and training schedule.
resume_time=0.0
): # Assumed wallclock time at the beginning. Affects reporting.
maintenance_start_time = time.time()
training_set = dataset.load_dataset(data_dir=config.data_dir,
verbose=True,
**config.dataset)
unlabeled_training_set = dataset.load_dataset(
data_dir=config.unlabeled_data_dir,
verbose=True,
**config.unlabeled_dataset)
# Construct networks.
with tf.device('/gpu:0'):
if resume_run_id is not None:
network_pkl = misc.locate_network_pkl(resume_run_id,
resume_snapshot)
print('Loading networks from "%s"...' % network_pkl)
G, D, Gs = misc.load_pkl(network_pkl)
else:
print('Constructing networks...')
print("Training-Set Label Size: ", training_set.label_size)
print("Unlabeled-Training-Set Label Size: ",
unlabeled_training_set.label_size)
G = tfutil.Network('G',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.G)
D = tfutil.Network('D',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.D)
Gs = G.clone('Gs')
Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
G.print_layers()
D.print_layers()
print('Building TensorFlow graph...')
with tf.name_scope('Inputs'):
lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
minibatch_split = minibatch_in // config.num_gpus
reals, labels = training_set.get_minibatch_tf()
unlabeled_reals, _ = unlabeled_training_set.get_minibatch_tf()
reals_split = tf.split(reals, config.num_gpus)
unlabeled_reals_split = tf.split(unlabeled_reals, config.num_gpus)
labels_split = tf.split(labels, config.num_gpus)
G_opt = tfutil.Optimizer(name='TrainG',
learning_rate=lrate_in,
**config.G_opt)
G_opt_pggan = tfutil.Optimizer(name='TrainG_pggan',
learning_rate=lrate_in,
**config.G_opt)
D_opt_pggan = tfutil.Optimizer(name='TrainD_pggan',
learning_rate=lrate_in,
**config.D_opt)
D_opt = tfutil.Optimizer(name='TrainD',
learning_rate=lrate_in,
**config.D_opt)
print("CUDA_VISIBLE_DEVICES: ", os.environ['CUDA_VISIBLE_DEVICES'])
for gpu in range(config.num_gpus):
with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
lod_assign_ops = [
tf.assign(G_gpu.find_var('lod'), lod_in),
tf.assign(D_gpu.find_var('lod'), lod_in)
]
reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment,
training_set.dynamic_range, drange_net)
unlabeled_reals_gpu = process_reals(
unlabeled_reals_split[gpu], lod_in, mirror_augment,
unlabeled_training_set.dynamic_range, drange_net)
labels_gpu = labels_split[gpu]
with tf.name_scope('G_loss'), tf.control_dependencies(
lod_assign_ops):
G_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
opt=G_opt,
training_set=training_set,
minibatch_size=minibatch_split,
unlabeled_reals=unlabeled_reals_gpu,
**config.G_loss)
with tf.name_scope('G_loss_pggan'), tf.control_dependencies(
lod_assign_ops):
G_loss_pggan = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
opt=G_opt,
training_set=training_set,
minibatch_size=minibatch_split,
**config.G_loss_pggan)
with tf.name_scope('D_loss'), tf.control_dependencies(
lod_assign_ops):
D_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
opt=D_opt,
training_set=training_set,
minibatch_size=minibatch_split,
reals=reals_gpu,
labels=labels_gpu,
unlabeled_reals=unlabeled_reals_gpu,
**config.D_loss)
with tf.name_scope('D_loss_pggan'), tf.control_dependencies(
lod_assign_ops):
D_loss_pggan = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
opt=D_opt,
training_set=training_set,
minibatch_size=minibatch_split,
unlabeled_reals=unlabeled_reals_gpu,
**config.D_loss_pggan)
G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
G_opt_pggan.register_gradients(tf.reduce_mean(G_loss_pggan),
G_gpu.trainables)
D_opt_pggan.register_gradients(tf.reduce_mean(D_loss_pggan),
D_gpu.trainables)
D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
print('GPU %d loaded!' % gpu)
G_train_op = G_opt.apply_updates()
G_train_op_pggan = G_opt_pggan.apply_updates()
D_train_op_pggan = D_opt_pggan.apply_updates()
D_train_op = D_opt.apply_updates()
print('Setting up snapshot image grid...')
grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(
G, training_set, **config.grid)
sched = TrainingSchedule(total_kimg * TrainingSpeedInt, training_set,
**config.sched)
grid_fakes = Gs.run(grid_latents,
grid_labels,
minibatch_size=sched.minibatch // config.num_gpus)
print('Setting up result dir...')
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
misc.save_image_grid(grid_reals,
os.path.join(result_subdir, 'reals.png'),
drange=training_set.dynamic_range,
grid_size=grid_size)
misc.save_image_grid(grid_fakes,
os.path.join(result_subdir, 'fakes%06d.png' % 0),
drange=drange_net,
grid_size=grid_size)
summary_log = tf.compat.v1.summary.FileWriter(result_subdir)
if save_tf_graph:
summary_log.add_graph(tf.get_default_graph())
if save_weight_histograms:
G.setup_weight_histograms()
D.setup_weight_histograms()
print("Start Time: ",
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print('Training...')
cur_nimg = int(resume_kimg * TrainingSpeedInt)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
train_start_time = tick_start_time - resume_time
prev_lod = -1.0
while cur_nimg < total_kimg * TrainingSpeedInt:
# Choose training parameters and configure training ops.
sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
sched2 = TrainingSchedule(cur_nimg, unlabeled_training_set,
**config.sched)
training_set.configure(sched.minibatch, sched.lod)
unlabeled_training_set.configure(sched2.minibatch, sched2.lod)
if reset_opt_for_new_lod:
if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
sched.lod) != np.ceil(prev_lod):
G_opt.reset_optimizer_state()
D_opt.reset_optimizer_state()
G_opt_pggan.reset_optimizer_state()
D_opt_pggan.reset_optimizer_state()
prev_lod = sched.lod
# Run training ops.
for repeat in range(minibatch_repeats):
for _ in range(D_repeats):
# Run the Pggan loss if lod != 0 else run SSL loss with feature matching
if sched.lod == 0:
tfutil.run(
[D_train_op, Gs_update_op], {
lod_in: sched.lod,
lrate_in: sched.D_lrate,
minibatch_in: sched.minibatch
})
else:
tfutil.run(
[D_train_op_pggan, Gs_update_op], {
lod_in: sched.lod,
lrate_in: sched.D_lrate,
minibatch_in: sched.minibatch
})
cur_nimg += sched.minibatch
#tmp = min(tick_start_nimg + sched.tick_kimg * TrainingSpeedInt, total_kimg * TrainingSpeedInt)
#print("Tick progress: {}/{}".format(cur_nimg, tmp), end="\r", flush=True)
# Run the Pggan loss if lod != 0 else run SSL loss with feature matching
if sched.lod == 0:
tfutil.run(
[G_train_op], {
lod_in: sched.lod,
lrate_in: sched.G_lrate,
minibatch_in: sched.minibatch
})
else:
tfutil.run(
[G_train_op_pggan], {
lod_in: sched.lod,
lrate_in: sched.G_lrate,
minibatch_in: sched.minibatch
})
# Perform maintenance tasks once per tick.
done = (cur_nimg >= total_kimg * TrainingSpeedInt)
if cur_nimg >= tick_start_nimg + sched.tick_kimg * TrainingSpeedInt or done:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / TrainingSpeedFloat
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
total_time = cur_time - train_start_time
maintenance_time = tick_start_time - maintenance_start_time
maintenance_start_time = cur_time
# Report progress.
print(
'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f date %s'
% (tfutil.autosummary('Progress/tick', cur_tick),
tfutil.autosummary('Progress/kimg',
cur_nimg / TrainingSpeedFloat),
tfutil.autosummary('Progress/lod', sched.lod),
tfutil.autosummary('Progress/minibatch', sched.minibatch),
misc.format_time(
tfutil.autosummary('Timing/total_sec', total_time)),
tfutil.autosummary('Timing/sec_per_tick', tick_time),
tfutil.autosummary('Timing/sec_per_kimg',
tick_time / tick_kimg),
tfutil.autosummary(
'Timing/maintenance_sec', maintenance_time),
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
tfutil.autosummary('Timing/total_hours',
total_time / (60.0 * 60.0))
tfutil.autosummary('Timing/total_days',
total_time / (24.0 * 60.0 * 60.0))
#######################
# VALIDATION ACCURACY #
#######################
# example ndim = 512 for an image that is 512x512 pixels
# All images for SSL-PGGAN must be square
ndim = 256
correct = 0
guesses = 0
dir_tuple = (config.validation_dog, config.validation_cat)
# If guessed the wrong class seeing if there is a bias
FP_RATE = [[0], [0]]
# For each class
for indx, directory in enumerate(dir_tuple):
# Go through every image that needs to be tested
for filename in os.listdir(directory):
guesses += 1
#tensor = np.zeros((1, 3, 512, 512))
print(filename)
img = np.asarray(PIL.Image.open(directory +
filename)).reshape(
3, ndim, ndim)
img = np.expand_dims(
img, axis=0) # makes the image (1,3,512,512)
K_logits_out, fake_logit_out, features_out = test_discriminator(
D, img)
#print("K Logits Out:",K_logits_out.eval())
sample_probs = tf.nn.softmax(K_logits_out)
#print("Softmax Output:", sample_probs.eval())
label = np.argmax(sample_probs.eval()[0], axis=0)
if label == indx:
correct += 1
else:
FP_RATE[indx][0] += 1
print("-----------------------------------")
print("GUESSED LABEL: ", label)
print("CORRECT LABEL: ", indx)
validation = (correct / guesses)
print("Total Correct: ", correct, "\n", "Total Guesses: ",
guesses, "\n", "Percent correct: ", validation)
print("False Positives: Dog, Cat", FP_RATE)
print()
tfutil.autosummary('Accuracy/Validation', (correct / guesses))
tfutil.save_summaries(summary_log, cur_nimg)
# Save snapshots.
if cur_tick % image_snapshot_ticks == 0 or done:
grid_fakes = Gs.run(grid_latents,
grid_labels,
minibatch_size=sched.minibatch //
config.num_gpus)
misc.save_image_grid(grid_fakes,
os.path.join(
result_subdir, 'fakes%06d.png' %
(cur_nimg // TrainingSpeedInt)),
drange=drange_net,
grid_size=grid_size)
if cur_tick % network_snapshot_ticks == 0 or done:
misc.save_pkl((G, D, Gs),
os.path.join(
result_subdir, 'network-snapshot-%06d.pkl' %
(cur_nimg // TrainingSpeedInt)))
# Record start time of the next tick.
tick_start_time = time.time()
# Write final results.
misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
summary_log.close()
open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
def calc_mnistrgb_histogram(run_id,
num_images=25600,
log='histogram.txt',
minibatch_size=256,
num_evals=10,
eval_reals=True,
final_only=False):
# Load the classification network.
# NOTE: The PKL can be downloaded from https://drive.google.com/open?id=0B4qLcYyJmiz0NHFULTdYc05lX0U
net = network.load_mnist_classifier(
os.path.join(config.data_dir,
'../networks/mnist_classifier_weights.pkl'))
input_var = T.tensor4()
output_expr = lasagne.layers.get_output(net,
inputs=input_var,
deterministic=True)
classify_fn = theano.function([input_var], [output_expr])
# Process folders
print 'Processing directory %s' % (run_id)
result_subdir = misc.locate_result_subdir(run_id)
network_pkls = misc.list_network_pkls(result_subdir)
misc.set_output_log_file(os.path.join(result_subdir, log))
if final_only:
network_pkls = [network_pkls[-1]]
# Histogram calculation.
def calc_histogram(images_all):
scores = []
divergences = []
for i in range(num_evals):
images = images_all[i * num_images:(i + 1) * num_images]
model = [0.] * 1000
for s in range(0, images.shape[0], minibatch_size):
img = images[s:s + minibatch_size].reshape((-1, 1, 32, 32))
res = np.asarray(classify_fn(img)[0])
res = np.argmax(res, axis=1)
res = res.reshape((-1, 3)) * np.asarray([[1, 10, 100]])
res = np.sum(res, axis=1)
for x in res:
model[int(x)] += 1.
model = np.array([b / 25600. for b in model
if b > 0]) # remove empty buckets, normalize
data = np.array([1. / 1000] *
len(model)) # corresponding ideal counts
scores.append(len(model))
divergences.append(np.sum(
model *
np.log(model /
data))) # reverse KL? Metz et al. say KL(model || data)
scores = np.asarray(scores, dtype=np.float32)
return np.mean(scores), np.mean(divergences)
# Load dataset.
training_set, drange_orig = load_dataset_for_previous_run(result_subdir,
shuffle=False)
reals, labels = training_set.get_random_minibatch(num_images * num_evals,
labels=True)
# Evaluate reals.
if eval_reals:
print 'Evaluating histogram for reals...'
time_begin = time.time()
mean, kld = calc_histogram(reals)
print 'Done in %s' % misc.format_time(time.time() - time_begin)
print 'mean %-8.4f kld %-8.4f' % (mean, kld)
# Evaluate each network snapshot.
latents = None
for network_idx, network_pkl in enumerate(network_pkls):
print '%-32s' % os.path.basename(network_pkl),
net = imgapi_load_net(run_id=result_subdir,
snapshot=network_pkl,
num_example_latents=num_images * num_evals)
fakes = imgapi_generate_batch(net,
net.example_latents,
labels,
minibatch_size=minibatch_size,
convert_to_uint8=True)
mean, kld = calc_histogram(fakes)
print 'mean %-8.4f kld %-8.4f' % (mean, kld)
def train_gan(separate_funcs=False,
D_training_repeats=1,
G_learning_rate_max=0.0010,
D_learning_rate_max=0.0010,
G_smoothing=0.999,
adam_beta1=0.0,
adam_beta2=0.99,
adam_epsilon=1e-8,
minibatch_default=16,
minibatch_overrides={},
rampup_kimg=40,
rampdown_kimg=0,
lod_initial_resolution=4,
lod_training_kimg=400,
lod_transition_kimg=400,
total_kimg=10000,
dequantize_reals=False,
gdrop_beta=0.9,
gdrop_lim=0.5,
gdrop_coef=0.2,
gdrop_exp=2.0,
drange_net=[-1, 1],
drange_viz=[-1, 1],
image_grid_size=None,
tick_kimg_default=50,
tick_kimg_overrides={
32: 20,
64: 10,
128: 10,
256: 5,
512: 2,
1024: 1
},
image_snapshot_ticks=4,
network_snapshot_ticks=40,
image_grid_type='default',
resume_network_pkl=None,
resume_kimg=0.0,
resume_time=0.0):
# Load dataset and build networks.
training_set, drange_orig = load_dataset()
if resume_network_pkl:
print 'Resuming', resume_network_pkl
G, D, _ = misc.load_pkl(
os.path.join(config.result_dir, resume_network_pkl))
else:
G = network.Network(num_channels=training_set.shape[1],
resolution=training_set.shape[2],
label_size=training_set.labels.shape[1],
**config.G)
D = network.Network(num_channels=training_set.shape[1],
resolution=training_set.shape[2],
label_size=training_set.labels.shape[1],
**config.D)
Gs = G.create_temporally_smoothed_version(beta=G_smoothing,
explicit_updates=True)
misc.print_network_topology_info(G.output_layers)
misc.print_network_topology_info(D.output_layers)
# Setup snapshot image grid.
if image_grid_type == 'default':
if image_grid_size is None:
w, h = G.output_shape[3], G.output_shape[2]
image_grid_size = np.clip(1920 / w, 3,
16), np.clip(1080 / h, 2, 16)
example_real_images, snapshot_fake_labels = training_set.get_random_minibatch(
np.prod(image_grid_size), labels=True)
snapshot_fake_latents = random_latents(np.prod(image_grid_size),
G.input_shape)
elif image_grid_type == 'category':
W = training_set.labels.shape[1]
H = W if image_grid_size is None else image_grid_size[1]
image_grid_size = W, H
snapshot_fake_latents = random_latents(W * H, G.input_shape)
snapshot_fake_labels = np.zeros((W * H, W),
dtype=training_set.labels.dtype)
example_real_images = np.zeros((W * H, ) + training_set.shape[1:],
dtype=training_set.dtype)
for x in xrange(W):
snapshot_fake_labels[x::W, x] = 1.0
indices = np.arange(
training_set.shape[0])[training_set.labels[:, x] != 0]
for y in xrange(H):
example_real_images[x + y * W] = training_set.h5_lods[0][
np.random.choice(indices)]
else:
raise ValueError('Invalid image_grid_type', image_grid_type)
# Theano input variables and compile generation func.
print 'Setting up Theano...'
real_images_var = T.TensorType('float32', [False] *
len(D.input_shape))('real_images_var')
real_labels_var = T.TensorType(
'float32', [False] * len(training_set.labels.shape))('real_labels_var')
fake_latents_var = T.TensorType('float32', [False] *
len(G.input_shape))('fake_latents_var')
fake_labels_var = T.TensorType(
'float32', [False] * len(training_set.labels.shape))('fake_labels_var')
G_lrate = theano.shared(np.float32(0.0))
D_lrate = theano.shared(np.float32(0.0))
gen_fn = theano.function([fake_latents_var, fake_labels_var],
Gs.eval_nd(fake_latents_var,
fake_labels_var,
ignore_unused_inputs=True),
on_unused_input='ignore')
# Misc init.
resolution_log2 = int(np.round(np.log2(G.output_shape[2])))
initial_lod = max(
resolution_log2 - int(np.round(np.log2(lod_initial_resolution))), 0)
cur_lod = 0.0
min_lod, max_lod = -1.0, -2.0
fake_score_avg = 0.0
if config.D.get('mbdisc_kernels', None):
print 'Initializing minibatch discrimination...'
if hasattr(D, 'cur_lod'): D.cur_lod.set_value(np.float32(initial_lod))
D.eval(real_images_var, deterministic=False, init=True)
init_layers = lasagne.layers.get_all_layers(D.output_layers)
init_updates = [
update for layer in init_layers
for update in getattr(layer, 'init_updates', [])
]
init_fn = theano.function(inputs=[real_images_var],
outputs=None,
updates=init_updates)
init_reals = training_set.get_random_minibatch(500, lod=initial_lod)
init_reals = misc.adjust_dynamic_range(init_reals, drange_orig,
drange_net)
init_fn(init_reals)
del init_reals
# Save example images.
snapshot_fake_images = gen_fn(snapshot_fake_latents, snapshot_fake_labels)
result_subdir = misc.create_result_subdir(config.result_dir,
config.run_desc)
misc.save_image_grid(example_real_images,
os.path.join(result_subdir, 'reals.png'),
drange=drange_orig,
grid_size=image_grid_size)
misc.save_image_grid(snapshot_fake_images,
os.path.join(result_subdir, 'fakes%06d.png' % 0),
drange=drange_viz,
grid_size=image_grid_size)
# Training loop.
cur_nimg = int(resume_kimg * 1000)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
tick_train_out = []
train_start_time = tick_start_time - resume_time
while cur_nimg < total_kimg * 1000:
# Calculate current LOD.
cur_lod = initial_lod
if lod_training_kimg or lod_transition_kimg:
tlod = (cur_nimg / 1000.0) / (lod_training_kimg +
lod_transition_kimg)
cur_lod -= np.floor(tlod)
if lod_transition_kimg:
cur_lod -= max(
1.0 + (np.fmod(tlod, 1.0) - 1.0) *
(lod_training_kimg + lod_transition_kimg) /
lod_transition_kimg, 0.0)
cur_lod = max(cur_lod, 0.0)
# Look up resolution-dependent parameters.
cur_res = 2**(resolution_log2 - int(np.floor(cur_lod)))
minibatch_size = minibatch_overrides.get(cur_res, minibatch_default)
tick_duration_kimg = tick_kimg_overrides.get(cur_res,
tick_kimg_default)
# Update network config.
lrate_coef = misc.rampup(cur_nimg / 1000.0, rampup_kimg)
lrate_coef *= misc.rampdown_linear(cur_nimg / 1000.0, total_kimg,
rampdown_kimg)
G_lrate.set_value(np.float32(lrate_coef * G_learning_rate_max))
D_lrate.set_value(np.float32(lrate_coef * D_learning_rate_max))
if hasattr(G, 'cur_lod'): G.cur_lod.set_value(np.float32(cur_lod))
if hasattr(D, 'cur_lod'): D.cur_lod.set_value(np.float32(cur_lod))
# Setup training func for current LOD.
new_min_lod, new_max_lod = int(np.floor(cur_lod)), int(
np.ceil(cur_lod))
if min_lod != new_min_lod or max_lod != new_max_lod:
print 'Compiling training funcs...'
min_lod, max_lod = new_min_lod, new_max_lod
# Pre-process reals.
real_images_expr = real_images_var
if dequantize_reals:
rnd = theano.sandbox.rng_mrg.MRG_RandomStreams(
lasagne.random.get_rng().randint(1, 2147462579))
epsilon_noise = rnd.uniform(size=real_images_expr.shape,
low=-0.5,
high=0.5,
dtype='float32')
real_images_expr = T.cast(
real_images_expr, 'float32'
) + epsilon_noise # match original implementation of Improved Wasserstein
real_images_expr = misc.adjust_dynamic_range(
real_images_expr, drange_orig, drange_net)
if min_lod > 0: # compensate for shrink_based_on_lod
real_images_expr = T.extra_ops.repeat(real_images_expr,
2**min_lod,
axis=2)
real_images_expr = T.extra_ops.repeat(real_images_expr,
2**min_lod,
axis=3)
# Optimize loss.
G_loss, D_loss, real_scores_out, fake_scores_out = evaluate_loss(
G, D, min_lod, max_lod, real_images_expr, real_labels_var,
fake_latents_var, fake_labels_var, **config.loss)
G_updates = adam(G_loss,
G.trainable_params(),
learning_rate=G_lrate,
beta1=adam_beta1,
beta2=adam_beta2,
epsilon=adam_epsilon).items()
D_updates = adam(D_loss,
D.trainable_params(),
learning_rate=D_lrate,
beta1=adam_beta1,
beta2=adam_beta2,
epsilon=adam_epsilon).items()
# Compile training funcs.
if not separate_funcs:
GD_train_fn = theano.function([
real_images_var, real_labels_var, fake_latents_var,
fake_labels_var
], [G_loss, D_loss, real_scores_out, fake_scores_out],
updates=G_updates + D_updates +
Gs.updates,
on_unused_input='ignore')
else:
D_train_fn = theano.function([
real_images_var, real_labels_var, fake_latents_var,
fake_labels_var
], [G_loss, D_loss, real_scores_out, fake_scores_out],
updates=D_updates,
on_unused_input='ignore')
G_train_fn = theano.function(
[fake_latents_var, fake_labels_var], [],
updates=G_updates + Gs.updates,
on_unused_input='ignore')
# Invoke training funcs.
if not separate_funcs:
assert D_training_repeats == 1
mb_reals, mb_labels = training_set.get_random_minibatch(
minibatch_size,
lod=cur_lod,
shrink_based_on_lod=True,
labels=True)
mb_train_out = GD_train_fn(
mb_reals, mb_labels,
random_latents(minibatch_size, G.input_shape),
random_labels(minibatch_size, training_set))
cur_nimg += minibatch_size
tick_train_out.append(mb_train_out)
else:
for idx in xrange(D_training_repeats):
mb_reals, mb_labels = training_set.get_random_minibatch(
minibatch_size,
lod=cur_lod,
shrink_based_on_lod=True,
labels=True)
mb_train_out = D_train_fn(
mb_reals, mb_labels,
random_latents(minibatch_size, G.input_shape),
random_labels(minibatch_size, training_set))
cur_nimg += minibatch_size
tick_train_out.append(mb_train_out)
G_train_fn(random_latents(minibatch_size, G.input_shape),
random_labels(minibatch_size, training_set))
# Fade in D noise if we're close to becoming unstable
fake_score_cur = np.clip(np.mean(mb_train_out[1]), 0.0, 1.0)
fake_score_avg = fake_score_avg * gdrop_beta + fake_score_cur * (
1.0 - gdrop_beta)
gdrop_strength = gdrop_coef * (max(fake_score_avg - gdrop_lim, 0.0)**
gdrop_exp)
if hasattr(D, 'gdrop_strength'):
D.gdrop_strength.set_value(np.float32(gdrop_strength))
# Perform maintenance operations once per tick.
if cur_nimg >= tick_start_nimg + tick_duration_kimg * 1000 or cur_nimg >= total_kimg * 1000:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
tick_start_time = cur_time
tick_train_avg = tuple(
np.mean(np.concatenate([np.asarray(v).flatten()
for v in vals]))
for vals in zip(*tick_train_out))
tick_train_out = []
# Print progress.
print 'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-9.1f sec/kimg %-6.1f Dgdrop %-8.4f Gloss %-8.4f Dloss %-8.4f Dreal %-8.4f Dfake %-8.4f' % (
(cur_tick, cur_nimg / 1000.0, cur_lod, minibatch_size,
misc.format_time(cur_time - train_start_time), tick_time,
tick_time / tick_kimg, gdrop_strength) + tick_train_avg)
# Visualize generated images.
if cur_tick % image_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
snapshot_fake_images = gen_fn(snapshot_fake_latents,
snapshot_fake_labels)
misc.save_image_grid(snapshot_fake_images,
os.path.join(
result_subdir,
'fakes%06d.png' % (cur_nimg / 1000)),
drange=drange_viz,
grid_size=image_grid_size)
# Save network snapshot every N ticks.
if cur_tick % network_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
misc.save_pkl(
(G, D, Gs),
os.path.join(
result_subdir,
'network-snapshot-%06d.pkl' % (cur_nimg / 1000)))
# Write final results.
misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
training_set.close()
print 'Done.'
with open(os.path.join(result_subdir, '_training-done.txt'), 'wt'):
pass
def train_gan(
separate_funcs=False,
D_training_repeats=1,
G_learning_rate_max=0.0010,
D_learning_rate_max=0.0010,
G_smoothing=0.999,
adam_beta1=0.0,
adam_beta2=0.99,
adam_epsilon=1e-8,
minibatch_default=16,
minibatch_overrides={},
rampup_kimg=40 / speed_factor,
rampdown_kimg=0,
lod_initial_resolution=4,
lod_training_kimg=400 / speed_factor,
lod_transition_kimg=400 / speed_factor,
#lod_training_kimg = 40,
#lod_transition_kimg = 40,
total_kimg=10000 / speed_factor,
dequantize_reals=False,
gdrop_beta=0.9,
gdrop_lim=0.5,
gdrop_coef=0.2,
gdrop_exp=2.0,
drange_net=[-1, 1],
drange_viz=[-1, 1],
image_grid_size=None,
#tick_kimg_default = 1,
tick_kimg_default=50 / speed_factor,
tick_kimg_overrides={
32: 20,
64: 10,
128: 10,
256: 5,
512: 2,
1024: 1
},
image_snapshot_ticks=4,
network_snapshot_ticks=40,
image_grid_type='default',
#resume_network_pkl = '006-celeb128-progressive-growing/network-snapshot-002009.pkl',
resume_network_pkl=None,
resume_kimg=0,
resume_time=0.0):
# Load dataset and build networks.
training_set, drange_orig = load_dataset()
print "*************** test the format of dataset ***************"
print training_set
print drange_orig
# training_set是dataset模块解析h5之后的对象，
# drange_orig 为training_set.get_dynamic_range()
if resume_network_pkl:
print 'Resuming', resume_network_pkl
G, D, _ = misc.load_pkl(
os.path.join(config.result_dir, resume_network_pkl))
else:
G = network.Network(num_channels=training_set.shape[1],
resolution=training_set.shape[2],
label_size=training_set.labels.shape[1],
**config.G)
D = network.Network(num_channels=training_set.shape[1],
resolution=training_set.shape[2],
label_size=training_set.labels.shape[1],
**config.D)
Gs = G.create_temporally_smoothed_version(beta=G_smoothing,
explicit_updates=True)
# G,D对象可以由misc解析pkl之后生成，也可以由network模块构造
print G
print D
#misc.print_network_topology_info(G.output_layers)
#misc.print_network_topology_info(D.output_layers)
# Setup snapshot image grid.
# 设置中途输出图片的格式
if image_grid_type == 'default':
if image_grid_size is None:
w, h = G.output_shape[3], G.output_shape[2]
image_grid_size = np.clip(1920 / w, 3,
16), np.clip(1080 / h, 2, 16)
example_real_images, snapshot_fake_labels = training_set.get_random_minibatch(
np.prod(image_grid_size), labels=True)
snapshot_fake_latents = random_latents(np.prod(image_grid_size),
G.input_shape)
else:
raise ValueError('Invalid image_grid_type', image_grid_type)
# Theano input variables and compile generation func.
print 'Setting up Theano...'
real_images_var = T.TensorType('float32', [False] *
len(D.input_shape))('real_images_var')
# <class 'theano.tensor.var.TensorVariable'>
# print type(real_images_var),real_images_var
real_labels_var = T.TensorType(
'float32', [False] * len(training_set.labels.shape))('real_labels_var')
fake_latents_var = T.TensorType('float32', [False] *
len(G.input_shape))('fake_latents_var')
fake_labels_var = T.TensorType(
'float32', [False] * len(training_set.labels.shape))('fake_labels_var')
# 带有_var的均为输入张量
G_lrate = theano.shared(np.float32(0.0))
D_lrate = theano.shared(np.float32(0.0))
# share语法就是用来设定默认值的，返回复制的对象
gen_fn = theano.function([fake_latents_var, fake_labels_var],
Gs.eval_nd(fake_latents_var,
fake_labels_var,
ignore_unused_inputs=True),
on_unused_input='ignore')
# gen_fn 是一个函数，输入为：[fake_latents_var, fake_labels_var],
# 输出位：Gs.eval_nd(fake_latents_var, fake_labels_var, ignore_unused_inputs=True),
#生成函数
# Misc init.
#读入当前分辨率
resolution_log2 = int(np.round(np.log2(G.output_shape[2])))
#lod 精细度
initial_lod = max(
resolution_log2 - int(np.round(np.log2(lod_initial_resolution))), 0)
cur_lod = 0.0
min_lod, max_lod = -1.0, -2.0
fake_score_avg = 0.0
# Save example images.
snapshot_fake_images = gen_fn(snapshot_fake_latents, snapshot_fake_labels)
result_subdir = misc.create_result_subdir(config.result_dir,
config.run_desc)
misc.save_image_grid(example_real_images,
os.path.join(result_subdir, 'reals.png'),
drange=drange_orig,
grid_size=image_grid_size)
misc.save_image_grid(snapshot_fake_images,
os.path.join(result_subdir, 'fakes%06d.png' % 0),
drange=drange_viz,
grid_size=image_grid_size)
# Training loop.
# 这里才是主训练入口
# 注意在训练过程中不会跳出最外层while循环，因此更换分辨率等操作必然在while循环里
#现有图片数
cur_nimg = int(resume_kimg * 1000)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
tick_train_out = []
train_start_time = tick_start_time - resume_time
while cur_nimg < total_kimg * 1000:
# Calculate current LOD.
#计算当前精细度
cur_lod = initial_lod
if lod_training_kimg or lod_transition_kimg:
tlod = (cur_nimg / (1000.0 / speed_factor)) / (lod_training_kimg +
lod_transition_kimg)
cur_lod -= np.floor(tlod)
if lod_transition_kimg:
cur_lod -= max(
1.0 + (np.fmod(tlod, 1.0) - 1.0) *
(lod_training_kimg + lod_transition_kimg) /
lod_transition_kimg, 0.0)
cur_lod = max(cur_lod, 0.0)
# Look up resolution-dependent parameters.
cur_res = 2**(resolution_log2 - int(np.floor(cur_lod)))
# 当前分辨率
minibatch_size = minibatch_overrides.get(cur_res, minibatch_default)
tick_duration_kimg = tick_kimg_overrides.get(cur_res,
tick_kimg_default)
# Update network config.
# 更新网络结构
lrate_coef = misc.rampup(cur_nimg / 1000.0, rampup_kimg)
lrate_coef *= misc.rampdown_linear(cur_nimg / 1000.0, total_kimg,
rampdown_kimg)
G_lrate.set_value(np.float32(lrate_coef * G_learning_rate_max))
D_lrate.set_value(np.float32(lrate_coef * D_learning_rate_max))
if hasattr(G, 'cur_lod'): G.cur_lod.set_value(np.float32(cur_lod))
if hasattr(D, 'cur_lod'): D.cur_lod.set_value(np.float32(cur_lod))
# Setup training func for current LOD.
new_min_lod, new_max_lod = int(np.floor(cur_lod)), int(
np.ceil(cur_lod))
#print " cur_lod%f\n min_lod %f\n new_min_lod %f\n max_lod %f\n new_max_lod %f\n"%(cur_lod,min_lod,new_min_lod,max_lod,new_max_lod)
if min_lod != new_min_lod or max_lod != new_max_lod:
print 'Compiling training funcs...'
min_lod, max_lod = new_min_lod, new_max_lod
# Pre-process reals.
real_images_expr = real_images_var
if dequantize_reals:
rnd = theano.sandbox.rng_mrg.MRG_RandomStreams(
lasagne.random.get_rng().randint(1, 2147462579))
epsilon_noise = rnd.uniform(size=real_images_expr.shape,
low=-0.5,
high=0.5,
dtype='float32')
real_images_expr = T.cast(
real_images_expr, 'float32'
) + epsilon_noise # match original implementation of Improved Wasserstein
real_images_expr = misc.adjust_dynamic_range(
real_images_expr, drange_orig, drange_net)
if min_lod > 0: # compensate for shrink_based_on_lod
real_images_expr = T.extra_ops.repeat(real_images_expr,
2**min_lod,
axis=2)
real_images_expr = T.extra_ops.repeat(real_images_expr,
2**min_lod,
axis=3)
# Optimize loss.
G_loss, D_loss, real_scores_out, fake_scores_out = evaluate_loss(
G, D, min_lod, max_lod, real_images_expr, real_labels_var,
fake_latents_var, fake_labels_var, **config.loss)
G_updates = adam(G_loss,
G.trainable_params(),
learning_rate=G_lrate,
beta1=adam_beta1,
beta2=adam_beta2,
epsilon=adam_epsilon).items()
D_updates = adam(D_loss,
D.trainable_params(),
learning_rate=D_lrate,
beta1=adam_beta1,
beta2=adam_beta2,
epsilon=adam_epsilon).items()
D_train_fn = theano.function([
real_images_var, real_labels_var, fake_latents_var,
fake_labels_var
], [G_loss, D_loss, real_scores_out, fake_scores_out],
updates=D_updates,
on_unused_input='ignore')
G_train_fn = theano.function([fake_latents_var, fake_labels_var],
[],
updates=G_updates + Gs.updates,
on_unused_input='ignore')
for idx in xrange(D_training_repeats):
mb_reals, mb_labels = training_set.get_random_minibatch(
minibatch_size,
lod=cur_lod,
shrink_based_on_lod=True,
labels=True)
print "******* test minibatch"
print "mb_reals"
print idx, D_training_repeats
print mb_reals.shape, mb_labels.shape
#print mb_reals
print "mb_labels"
#print mb_labels
mb_train_out = D_train_fn(
mb_reals, mb_labels,
random_latents(minibatch_size, G.input_shape),
random_labels(minibatch_size, training_set))
cur_nimg += minibatch_size
tick_train_out.append(mb_train_out)
G_train_fn(random_latents(minibatch_size, G.input_shape),
random_labels(minibatch_size, training_set))
# Fade in D noise if we're close to becoming unstable
fake_score_cur = np.clip(np.mean(mb_train_out[1]), 0.0, 1.0)
fake_score_avg = fake_score_avg * gdrop_beta + fake_score_cur * (
1.0 - gdrop_beta)
gdrop_strength = gdrop_coef * (max(fake_score_avg - gdrop_lim, 0.0)**
gdrop_exp)
if hasattr(D, 'gdrop_strength'):
D.gdrop_strength.set_value(np.float32(gdrop_strength))
# Perform maintenance operations once per tick.
if cur_nimg >= tick_start_nimg + tick_duration_kimg * 1000 or cur_nimg >= total_kimg * 1000:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
tick_start_time = cur_time
tick_train_avg = tuple(
np.mean(np.concatenate([np.asarray(v).flatten()
for v in vals]))
for vals in zip(*tick_train_out))
tick_train_out = []
# Print progress.
print 'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-9.1f sec/kimg %-6.1f Dgdrop %-8.4f Gloss %-8.4f Dloss %-8.4f Dreal %-8.4f Dfake %-8.4f' % (
(cur_tick, cur_nimg / 1000.0, cur_lod, minibatch_size,
misc.format_time(cur_time - train_start_time), tick_time,
tick_time / tick_kimg, gdrop_strength) + tick_train_avg)
# Visualize generated images.
if cur_tick % image_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
snapshot_fake_images = gen_fn(snapshot_fake_latents,
snapshot_fake_labels)
misc.save_image_grid(snapshot_fake_images,
os.path.join(
result_subdir,
'fakes%06d.png' % (cur_nimg / 1000)),
drange=drange_viz,
grid_size=image_grid_size)
# Save network snapshot every N ticks.
if cur_tick % network_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
misc.save_pkl(
(G, D, Gs),
os.path.join(
result_subdir,
'network-snapshot-%06d.pkl' % (cur_nimg / 1000)))
# Write final results.
misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
training_set.close()
print 'Done.'
with open(os.path.join(result_subdir, '_training-done.txt'), 'wt'):
pass
def calc_sliced_wasserstein_scores(run_id,
log='sliced-wasserstein.txt',
resolution_min=16,
resolution_max=1024,
num_images=8192,
nhoods_per_image=64,
nhood_size=7,
dir_repeats=1,
dirs_per_repeat=147,
minibatch_size=16):
import sliced_wasserstein
result_subdir = misc.locate_result_subdir(run_id)
network_pkls = misc.list_network_pkls(result_subdir)
misc.set_output_log_file(os.path.join(result_subdir, log))
# Load dataset.
print 'Loading dataset...'
training_set, drange_orig = load_dataset_for_previous_run(result_subdir)
assert training_set.shape[1] == 3 # RGB
assert num_images % minibatch_size == 0
# Select resolutions.
resolution_full = training_set.shape[3]
resolution_min = min(resolution_min, resolution_full)
resolution_max = min(resolution_max, resolution_full)
base_lod = int(np.log2(resolution_full)) - int(np.log2(resolution_max))
resolutions = [
2**i for i in xrange(int(np.log2(resolution_max)),
int(np.log2(resolution_min)) - 1, -1)
]
# Collect descriptors for reals.
print 'Extracting descriptors for reals...',
time_begin = time.time()
desc_real = [[] for res in resolutions]
desc_test = [[] for res in resolutions]
for minibatch_begin in xrange(0, num_images, minibatch_size):
minibatch = training_set.get_random_minibatch(minibatch_size,
lod=base_lod)
for lod, level in enumerate(
sliced_wasserstein.generate_laplacian_pyramid(
minibatch, len(resolutions))):
desc_real[lod].append(
sliced_wasserstein.get_descriptors_for_minibatch(
level, nhood_size, nhoods_per_image))
desc_test[lod].append(
sliced_wasserstein.get_descriptors_for_minibatch(
level, nhood_size, nhoods_per_image))
print 'done in %s' % misc.format_time(time.time() - time_begin)
# Evaluate scores for reals.
print 'Evaluating scores for reals...',
time_begin = time.time()
scores = []
for lod, res in enumerate(resolutions):
desc_real[lod] = sliced_wasserstein.finalize_descriptors(
desc_real[lod])
desc_test[lod] = sliced_wasserstein.finalize_descriptors(
desc_test[lod])
scores.append(
sliced_wasserstein.sliced_wasserstein(desc_real[lod],
desc_test[lod], dir_repeats,
dirs_per_repeat))
del desc_test
print 'done in %s' % misc.format_time(time.time() - time_begin)
# Print table header.
print
print '%-32s' % 'Case',
for lod, res in enumerate(resolutions):
print '%-12s' % ('%dx%d' % (res, res)),
print 'Average'
print '%-32s' % '---',
for lod, res in enumerate(resolutions):
print '%-12s' % '---',
print '---'
print '%-32s' % 'reals',
for lod, res in enumerate(resolutions):
print '%-12.6f' % scores[lod],
print '%.6f' % np.mean(scores)
# Process each network snapshot.
for network_idx, network_pkl in enumerate(network_pkls):
print '%-32s' % os.path.basename(network_pkl),
net = imgapi_load_net(run_id=result_subdir,
snapshot=network_pkl,
num_example_latents=num_images,
random_seed=network_idx)
# Extract descriptors for generated images.
desc_fake = [[] for res in resolutions]
for minibatch_begin in xrange(0, num_images, minibatch_size):
latents = net.example_latents[minibatch_begin:minibatch_begin +
minibatch_size]
labels = net.example_labels[minibatch_begin:minibatch_begin +
minibatch_size]
minibatch = imgapi_generate_batch(net,
latents,
labels,
minibatch_size=minibatch_size,
convert_to_uint8=True)
minibatch = sliced_wasserstein.downscale_minibatch(
minibatch, base_lod)
for lod, level in enumerate(
sliced_wasserstein.generate_laplacian_pyramid(
minibatch, len(resolutions))):
desc_fake[lod].append(
sliced_wasserstein.get_descriptors_for_minibatch(
level, nhood_size, nhoods_per_image))
# Evaluate scores.
scores = []
for lod, res in enumerate(resolutions):
desc_fake[lod] = sliced_wasserstein.finalize_descriptors(
desc_fake[lod])
scores.append(
sliced_wasserstein.sliced_wasserstein(desc_real[lod],
desc_fake[lod],
dir_repeats,
dirs_per_repeat))
del desc_fake
# Report results.
for lod, res in enumerate(resolutions):
print '%-12.6f' % scores[lod],
print '%.6f' % np.mean(scores)
print
print 'Done.'
def run(self):
self.parent.clear_plots()
if self.data is None:
length, tMin, tMax, fMin, fMax, lMin, lMax, peakF, peakL, peakT = (
'', ) * 10
else:
length = len(self.data)
tMin = format_time(self.extent.tMin, True)
tMax = format_time(self.extent.tMax, True)
fMin = format_precision(self.settings,
freq=self.extent.fMin,
fancyUnits=True)
fMax = format_precision(self.settings,
freq=self.extent.fMax,
fancyUnits=True)
lMin = format_precision(self.settings,
level=self.extent.lMin,
fancyUnits=True)
lMax = format_precision(self.settings,
level=self.extent.lMax,
fancyUnits=True)
peak = self.extent.get_peak_flt()
peakF = format_precision(self.settings,
freq=peak[0],
fancyUnits=True)
peakL = format_precision(self.settings,
level=peak[1],
fancyUnits=True)
peakT = format_time(peak[2], True)
text = [
['Sweeps', '', length],
['Extents', '', ''],
['', 'Start', tMin],
['', 'End', tMax],
['', 'Min frequency', fMin],
['', 'Max frequency', fMax],
['', 'Min level', lMin],
['', 'Max level', lMax],
['Peak', '', ''],
['', 'Level', peakL],
['', 'Frequency', peakF],
['', 'Time', peakT],
]
table = Table(self.axes, loc='center')
table.set_gid('table')
rows = len(text)
cols = len(text[0])
for row in xrange(rows):
for col in xrange(cols):
table.add_cell(row,
col,
text=text[row][col],
width=1.0 / cols,
height=1.0 / rows)
if self.settings.grid:
colour = 'LightGray'
else:
colour = 'w'
set_table_colour(table, colour)
for i in range(3):
table.auto_set_column_width(i)
self.axes.add_table(table)
self.parent.redraw_plot()
self.parent.threadPlot = None
def evaluate_metrics(run_id,
log,
metrics,
num_images,
real_passes,
minibatch_size=None):
metric_class_names = {
'swd': 'metrics.sliced_wasserstein.API',
'fid': 'metrics.frechet_inception_distance.API',
'is': 'metrics.inception_score.API',
'msssim': 'metrics.ms_ssim.API',
}
# Locate training run and initialize logging.
result_subdir = misc.locate_result_subdir(run_id)
snapshot_pkls = misc.list_network_pkls(result_subdir, include_final=False)
assert len(snapshot_pkls) >= 1
log_file = os.path.join(result_subdir, log)
print('Logging output to', log_file)
misc.set_output_log_file(log_file)
# Initialize dataset and select minibatch size.
dataset_obj, mirror_augment = misc.load_dataset_for_previous_run(
result_subdir, verbose=True, shuffle_mb=0)
if minibatch_size is None:
minibatch_size = np.clip(8192 // dataset_obj.shape[1], 4, 256)
# Initialize metrics.
metric_objs = []
for name in metrics:
class_name = metric_class_names.get(name, name)
print('Initializing %s...' % class_name)
class_def = tfutil.import_obj(class_name)
image_shape = [3] + dataset_obj.shape[1:]
obj = class_def(num_images=num_images,
image_shape=image_shape,
image_dtype=np.uint8,
minibatch_size=minibatch_size)
tfutil.init_uninited_vars()
mode = 'warmup'
obj.begin(mode)
for idx in range(10):
obj.feed(
mode,
np.random.randint(0,
256,
size=[minibatch_size] + image_shape,
dtype=np.uint8))
obj.end(mode)
metric_objs.append(obj)
# Print table header.
print()
print('%-10s%-12s' % ('Snapshot', 'Time_eval'), end='')
for obj in metric_objs:
for name, fmt in zip(obj.get_metric_names(),
obj.get_metric_formatting()):
print('%-*s' % (len(fmt % 0), name), end='')
print()
print('%-10s%-12s' % ('---', '---'), end='')
for obj in metric_objs:
for fmt in obj.get_metric_formatting():
print('%-*s' % (len(fmt % 0), '---'), end='')
print()
# Feed in reals.
for title, mode in [('Reals', 'reals'), ('Reals2', 'fakes')][:real_passes]:
print('%-10s' % title, end='')
time_begin = time.time()
labels = np.zeros([num_images, dataset_obj.label_size],
dtype=np.float32)
[obj.begin(mode) for obj in metric_objs]
for begin in range(0, num_images, minibatch_size):
end = min(begin + minibatch_size, num_images)
images, labels[begin:end] = dataset_obj.get_minibatch_np(end -
begin)
if mirror_augment:
images = misc.apply_mirror_augment(images)
if images.shape[1] == 1:
images = np.tile(images, [1, 3, 1, 1]) # grayscale => RGB
[obj.feed(mode, images) for obj in metric_objs]
results = [obj.end(mode) for obj in metric_objs]
print('%-12s' % misc.format_time(time.time() - time_begin), end='')
for obj, vals in zip(metric_objs, results):
for val, fmt in zip(vals, obj.get_metric_formatting()):
print(fmt % val, end='')
print()
# Evaluate each network snapshot.
for snapshot_idx, snapshot_pkl in enumerate(reversed(snapshot_pkls)):
prefix = 'network-snapshot-'
postfix = '.pkl'
snapshot_name = os.path.basename(snapshot_pkl)
assert snapshot_name.startswith(prefix) and snapshot_name.endswith(
postfix)
snapshot_kimg = int(snapshot_name[len(prefix):-len(postfix)])
print('%-10d' % snapshot_kimg, end='')
mode = 'fakes'
[obj.begin(mode) for obj in metric_objs]
time_begin = time.time()
with tf.Graph().as_default(), tfutil.create_session(
config.tf_config).as_default():
G, D, Gs = misc.load_pkl(snapshot_pkl)
for begin in range(0, num_images, minibatch_size):
end = min(begin + minibatch_size, num_images)
latents = misc.random_latents(end - begin, Gs)
images = Gs.run(latents,
labels[begin:end],
num_gpus=config.num_gpus,
out_mul=127.5,
out_add=127.5,
out_dtype=np.uint8)
if images.shape[1] == 1:
images = np.tile(images, [1, 3, 1, 1]) # grayscale => RGB
[obj.feed(mode, images) for obj in metric_objs]
results = [obj.end(mode) for obj in metric_objs]
print('%-12s' % misc.format_time(time.time() - time_begin), end='')
for obj, vals in zip(metric_objs, results):
for val, fmt in zip(vals, obj.get_metric_formatting()):
print(fmt % val, end='')
print()
print()
def train_detector(
minibatch_repeats=4, # Number of minibatches to run before adjusting training parameters.
total_kimg=1, # Total length of the training, measured in thousands of real images.
drange_net=[
-1, 1
], # Dynamic range used when feeding image data to the networks.
snapshot_size=16, # Size of the snapshot image
snapshot_ticks=2**13, # Number of images before maintenance
image_snapshot_ticks=1, # How often to export image snapshots?
network_snapshot_ticks=1, # How often to export network snapshots?
save_tf_graph=True, # Include full TensorFlow computation graph in the tfevents file?
save_weight_histograms=False, # Include weight histograms in the tfevents file?
resume_run_id=None, # Run ID or network pkl to resume training from, None = start from scratch.
resume_snapshot=None, # Snapshot index to resume training from, None = autodetect.
resume_kimg=0.0, # Assumed training progress at the beginning. Affects reporting and training schedule.
resume_time=0.0
): # Assumed wallclock time at the beginning. Affects reporting.
maintenance_start_time = time.time()
# Load the datasets
training_set = dataset.load_dataset(tfrecord=config.tfrecord_train,
verbose=True,
**config.dataset)
testing_set = dataset.load_dataset(tfrecord=config.tfrecord_test,
verbose=True,
repeat=False,
shuffle_mb=0,
**config.dataset)
testing_set_len = len(testing_set)
# TODO: data augmentation
# TODO: testing set
# Construct networks.
with tf.device('/gpu:0'):
if resume_run_id is not None: # TODO: save methods
network_pkl = misc.locate_network_pkl(resume_run_id,
resume_snapshot)
print('Loading networks from "%s"...' % network_pkl)
N = misc.load_pkl(network_pkl)
else:
print('Constructing the network...'
) # TODO: better network (like lod-wise network)
N = tfutil.Network('N',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
**config.N)
N.print_layers()
print('Building TensorFlow graph...')
# Training set up
with tf.name_scope('Inputs'):
lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
# minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
reals, labels, bboxes = training_set.get_minibatch_tf(
) # TODO: increase the size of the batch by several loss computation and mean
N_opt = tfutil.Optimizer(name='TrainN',
learning_rate=lrate_in,
**config.N_opt)
with tf.device('/gpu:0'):
reals, labels, gt_outputs, gt_ref = pre_process(
reals, labels, bboxes, training_set.dynamic_range,
[0, training_set.shape[-2]], drange_net)
with tf.name_scope('N_loss'): # TODO: loss inadapted
N_loss = tfutil.call_func_by_name(N=N,
reals=reals,
gt_outputs=gt_outputs,
gt_ref=gt_ref,
**config.N_loss)
N_opt.register_gradients(tf.reduce_mean(N_loss), N.trainables)
N_train_op = N_opt.apply_updates()
# Testing set up
with tf.device('/gpu:0'):
test_reals_tf, test_labels_tf, test_bboxes_tf = testing_set.get_minibatch_tf(
)
test_reals_tf, test_labels_tf, test_gt_outputs_tf, test_gt_ref_tf = pre_process(
test_reals_tf, test_labels_tf, test_bboxes_tf,
testing_set.dynamic_range, [0, testing_set.shape[-2]], drange_net)
with tf.name_scope('N_test_loss'):
test_loss = tfutil.call_func_by_name(N=N,
reals=test_reals_tf,
gt_outputs=test_gt_outputs_tf,
gt_ref=test_gt_ref_tf,
is_training=False,
**config.N_loss)
print('Setting up result dir...')
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
summary_log = tf.summary.FileWriter(result_subdir)
if save_tf_graph:
summary_log.add_graph(tf.get_default_graph())
if save_weight_histograms:
N.setup_weight_histograms()
test_reals, _, test_bboxes = testing_set.get_minibatch_np(snapshot_size)
misc.save_img_bboxes(test_reals,
test_bboxes,
os.path.join(result_subdir, 'reals.png'),
snapshot_size,
adjust_range=False)
test_reals = misc.adjust_dynamic_range(test_reals,
training_set.dynamic_range,
drange_net)
test_preds, _ = N.run(test_reals, minibatch_size=snapshot_size)
misc.save_img_bboxes(test_reals, test_preds,
os.path.join(result_subdir, 'fakes.png'),
snapshot_size)
print('Training...')
if resume_run_id is None:
tfutil.run(tf.global_variables_initializer())
cur_nimg = 0
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
train_start_time = tick_start_time
# Choose training parameters and configure training ops.
sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
training_set.configure(sched.minibatch)
_train_loss = 0
while cur_nimg < total_kimg * 1000:
# Run training ops.
# for _ in range(minibatch_repeats):
_, loss = tfutil.run([N_train_op, N_loss], {lrate_in: sched.N_lrate})
_train_loss += loss
cur_nimg += sched.minibatch
# Perform maintenance tasks once per tick.
if (cur_nimg >= total_kimg * 1000) or (cur_nimg % snapshot_ticks == 0
and cur_nimg > 0):
cur_tick += 1
cur_time = time.time()
# tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
_train_loss = _train_loss / (cur_nimg - tick_start_nimg)
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
total_time = cur_time - train_start_time
maintenance_time = tick_start_time - maintenance_start_time
maintenance_start_time = cur_time
testing_set.configure(sched.minibatch)
_test_loss = 0
# testing_set_len = 1 # TMP
for _ in range(0, testing_set_len, sched.minibatch):
_test_loss += tfutil.run(test_loss)
_test_loss /= testing_set_len
# Report progress. # TODO: improved report display
print(
'tick %-5d kimg %-6.1f time %-10s sec/tick %-3.1f maintenance %-7.2f train_loss %.4f test_loss %.4f'
% (tfutil.autosummary('Progress/tick', cur_tick),
tfutil.autosummary('Progress/kimg', cur_nimg / 1000),
misc.format_time(
tfutil.autosummary('Timing/total_sec', total_time)),
tfutil.autosummary('Timing/sec_per_tick', tick_time),
tfutil.autosummary('Timing/maintenance', maintenance_time),
tfutil.autosummary('TrainN/train_loss', _train_loss),
tfutil.autosummary('TrainN/test_loss', _test_loss)))
tfutil.autosummary('Timing/total_hours',
total_time / (60.0 * 60.0))
tfutil.autosummary('Timing/total_days',
total_time / (24.0 * 60.0 * 60.0))
tfutil.save_summaries(summary_log, cur_nimg)
if cur_tick % image_snapshot_ticks == 0:
test_bboxes, test_refs = N.run(test_reals,
minibatch_size=snapshot_size)
misc.save_img_bboxes_ref(
test_reals, test_bboxes, test_refs,
os.path.join(result_subdir,
'fakes%06d.png' % (cur_nimg // 1000)),
snapshot_size)
if cur_tick % network_snapshot_ticks == 0:
misc.save_pkl(
N,
os.path.join(
result_subdir,
'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))
_train_loss = 0
# Record start time of the next tick.
tick_start_time = time.time()
# Write final results.
# misc.save_pkl(N, os.path.join(result_subdir, 'network-final.pkl'))
summary_log.close()
open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
def train_progressive_gan(
G_smoothing=0.999, # Exponential running average of generator weights.
D_repeats=1, # How many times the discriminator is trained per G iteration.
minibatch_repeats=4, # Number of minibatches to run before adjusting training parameters.
reset_opt_for_new_lod=True, # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
total_kimg=15000, # Total length of the training, measured in thousands of real images.
mirror_augment=False, # Enable mirror augment?
drange_net=[
-1, 1
], # Dynamic range used when feeding image data to the networks.
image_snapshot_ticks=1, # How often to export image snapshots?
network_snapshot_ticks=10, # How often to export network snapshots?
save_tf_graph=False, # Include full TensorFlow computation graph in the tfevents file?
save_weight_histograms=False, # Include weight histograms in the tfevents file?
resume_run_id=None, # Run ID or network pkl to resume training from, None = start from scratch.
resume_snapshot=None, # Snapshot index to resume training from, None = autodetect.
resume_kimg=0.0, # Assumed training progress at the beginning. Affects reporting and training schedule.
resume_time=0.0
): # Assumed wallclock time at the beginning. Affects reporting.
maintenance_start_time = time.time()
training_set = dataset.load_dataset(data_dir=config.data_dir,
verbose=True,
**config.dataset)
# Construct networks.
with tf.device('/gpu:0'):
if resume_run_id is not None:
network_pkl = misc.locate_network_pkl(resume_run_id,
resume_snapshot)
print('Loading networks from "%s"...' % network_pkl)
G, D, Gs, E = misc.load_pkl(network_pkl)
else:
print('Constructing networks...')
G = tfutil.Network('G',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.G)
D = tfutil.Network('D',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.D)
E = tfutil.Network('E',
num_channels=training_set.shape[0],
resolution=training_set.shape[1],
label_size=training_set.label_size,
**config.E)
Gs = G.clone('Gs')
Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
G.print_layers()
D.print_layers()
E.print_layers()
print('Building TensorFlow graph...')
with tf.name_scope('Inputs'):
lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
minibatch_split = minibatch_in // config.num_gpus
reals, labels = training_set.get_minibatch_tf()
reals_split = tf.split(reals, config.num_gpus)
labels_split = tf.split(labels, config.num_gpus)
G_opt = tfutil.Optimizer(name='TrainG',
learning_rate=lrate_in,
**config.G_opt)
D_opt = tfutil.Optimizer(name='TrainD',
learning_rate=lrate_in,
**config.D_opt)
E_opt = tfutil.Optimizer(name='TrainE',
learning_rate=lrate_in,
**config.E_opt)
for gpu in range(config.num_gpus):
with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
E_gpu = E if gpu == 0 else E.clone(E.name + '_shadow')
lod_assign_ops = [
tf.assign(G_gpu.find_var('lod'), lod_in),
tf.assign(D_gpu.find_var('lod'), lod_in),
tf.assign(E_gpu.find_var('lod'), lod_in)
]
reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment,
training_set.dynamic_range, drange_net)
labels_gpu = labels_split[gpu]
with tf.name_scope('G_loss'), tf.control_dependencies(
lod_assign_ops):
G_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
E=E_gpu,
opt=G_opt,
training_set=training_set,
minibatch_size=minibatch_split,
**config.G_loss)
with tf.name_scope('D_loss'), tf.control_dependencies(
lod_assign_ops):
D_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
E=E_gpu,
opt=D_opt,
training_set=training_set,
minibatch_size=minibatch_split,
reals=reals_gpu,
labels=labels_gpu,
**config.D_loss)
with tf.name_scope('E_loss'), tf.control_dependencies(
lod_assign_ops):
E_loss = tfutil.call_func_by_name(
G=G_gpu,
D=D_gpu,
E=E_gpu,
opt=E_opt,
training_set=training_set,
reals=reals_gpu,
minibatch_size=minibatch_split,
**config.E_loss)
G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
E_opt.register_gradients(tf.reduce_mean(E_loss), E_gpu.trainables)
G_train_op = G_opt.apply_updates()
D_train_op = D_opt.apply_updates()
E_train_op = E_opt.apply_updates()
#sys.exit(0)
print('Setting up snapshot image grid...')
grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(
G, training_set, **config.grid)
sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
grid_fakes = Gs.run(grid_latents,
grid_labels,
minibatch_size=sched.minibatch // config.num_gpus)
print('Setting up result dir...')
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
misc.save_image_grid(grid_reals,
os.path.join(result_subdir, 'reals.png'),
drange=training_set.dynamic_range,
grid_size=grid_size)
misc.save_image_grid(grid_fakes,
os.path.join(result_subdir, 'fakes%06d.png' % 0),
drange=drange_net,
grid_size=grid_size)
summary_log = tf.summary.FileWriter(result_subdir)
if save_tf_graph:
summary_log.add_graph(tf.get_default_graph())
if save_weight_histograms:
G.setup_weight_histograms()
D.setup_weight_histograms()
print('Training...')
cur_nimg = int(resume_kimg * 1000)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
train_start_time = tick_start_time - resume_time
prev_lod = -1.0
while cur_nimg < total_kimg * 1000:
# Choose training parameters and configure training ops.
sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
training_set.configure(sched.minibatch, sched.lod)
if reset_opt_for_new_lod:
if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
sched.lod) != np.ceil(prev_lod):
G_opt.reset_optimizer_state()
D_opt.reset_optimizer_state()
E_opt.reset_optimizer_state()
prev_lod = sched.lod
# Run training ops.
for repeat in range(minibatch_repeats):
for _ in range(D_repeats):
tfutil.run(
[D_train_op, Gs_update_op], {
lod_in: sched.lod,
lrate_in: sched.D_lrate,
minibatch_in: sched.minibatch
})
cur_nimg += sched.minibatch
tfutil.run(
[G_train_op, E_train_op], {
lod_in: sched.lod,
lrate_in: sched.G_lrate,
minibatch_in: sched.minibatch
})
# Perform maintenance tasks once per tick.
done = (cur_nimg >= total_kimg * 1000)
if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
total_time = cur_time - train_start_time
maintenance_time = tick_start_time - maintenance_start_time
maintenance_start_time = cur_time
# Report progress.
print(
'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f'
% (tfutil.autosummary('Progress/tick', cur_tick),
tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
tfutil.autosummary('Progress/lod', sched.lod),
tfutil.autosummary('Progress/minibatch', sched.minibatch),
misc.format_time(
tfutil.autosummary('Timing/total_sec', total_time)),
tfutil.autosummary('Timing/sec_per_tick', tick_time),
tfutil.autosummary('Timing/sec_per_kimg',
tick_time / tick_kimg),
tfutil.autosummary('Timing/maintenance_sec',
maintenance_time)))
tfutil.autosummary('Timing/total_hours',
total_time / (60.0 * 60.0))
tfutil.autosummary('Timing/total_days',
total_time / (24.0 * 60.0 * 60.0))
tfutil.save_summaries(summary_log, cur_nimg)
# Save snapshots.
if cur_tick % image_snapshot_ticks == 0 or done:
grid_fakes = Gs.run(grid_latents,
grid_labels,
minibatch_size=sched.minibatch //
config.num_gpus)
misc.save_image_grid(grid_fakes,
os.path.join(
result_subdir,
'fakes%06d.png' % (cur_nimg // 1000)),
drange=drange_net,
grid_size=grid_size)
misc.save_all_res(training_set.shape[1],
Gs,
result_subdir,
50,
minibatch_size=sched.minibatch //
config.num_gpus)
if cur_tick % network_snapshot_ticks == 0 or done:
misc.save_pkl(
(G, D, Gs, E),
os.path.join(
result_subdir,
'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))
# Record start time of the next tick.
tick_start_time = time.time()
# Write final results.
misc.save_pkl((G, D, Gs, E),
os.path.join(result_subdir, 'network-final.pkl'))
summary_log.close()
open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
def run(self):
self.parent.clear_plots()
if self.data is None:
length, tMin, tMax, fMin, fMax, lMin, lMax, peakF, peakL, peakT = ('-',) * 10
else:
length = len(self.data)
tMin = format_time(self.extent.tMin, True)
tMax = format_time(self.extent.tMax, True)
fMin = format_precision(self.settings, freq=self.extent.fMin,
fancyUnits=True)
fMax = format_precision(self.settings, freq=self.extent.fMax,
fancyUnits=True)
lMin = format_precision(self.settings, level=self.extent.lMin,
fancyUnits=True)
lMax = format_precision(self.settings, level=self.extent.lMax,
fancyUnits=True)
peak = self.extent.get_peak_flt()
peakF = format_precision(self.settings, freq=peak[0],
fancyUnits=True)
peakL = format_precision(self.settings, level=peak[1],
fancyUnits=True)
peakT = format_time(peak[2], True)
text = [['Sweeps', '', length],
['Extent', '', ''],
['', 'Start', tMin],
['', 'End', tMax],
['', 'Min frequency', fMin],
['', 'Max frequency', fMax],
['', 'Min level', lMin],
['', 'Max level', lMax],
['Peak', '', ''],
['', 'Level', peakL],
['', 'Frequency', peakF],
['', 'Time', peakT],
]
table = Table(self.axes, loc='center')
table.set_gid('table')
rows = len(text)
cols = len(text[0])
fontProperties = FontProperties()
fontProperties.set_weight('semibold')
for row in xrange(rows):
for col in xrange(cols):
fp = fontProperties if col == 0 else None
table.add_cell(row, col,
text=text[row][col],
fontproperties=fp,
width=1.0 / cols, height=1.0 / rows)
if self.settings.grid:
colour = 'LightGray'
else:
colour = 'w'
set_table_colour(table, colour)
for i in range(2):
table.auto_set_column_width(i)
self.axes.add_table(table)
self.parent.redraw_plot()
self.parent.threadPlot = None