def __init__(self,
_,
gan=None,
config=None,
g_optimizer=None,
d_optimizer=None,
name="GANOptimizer"):
super().__init__(config.learn_rate, name=name)
self.gan = gan
self.config = config
def create_optimizer(klass, options):
options['gan'] = self.gan
options['config'] = options
defn = {
k: v
for k, v in options.items()
if k in inspect.getargspec(klass).args
}
return klass(options.learn_rate, **defn)
d_optimizer = hc.lookup_functions(d_optimizer)
g_optimizer = hc.lookup_functions(g_optimizer)
self.d_optimizer = create_optimizer(d_optimizer["class"], d_optimizer)
self.g_optimizer = create_optimizer(g_optimizer["class"], g_optimizer)
def __init__(self, learning_rate=0.001, p=0.1, gan=None, config=None, use_locking=False, name="GigaWolfOptimizer", optimizer=None, optimizer2=None):
super().__init__(use_locking, name)
self.gan = gan
self.config = config
self._lr_t = learning_rate
optimizer = hc.lookup_functions(optimizer)
self.optimizer = self.gan.create_optimizer(optimizer)
optimizer2 = hc.lookup_functions(optimizer2)
self.optimizer2 = self.gan.create_optimizer(optimizer2)
def __init__(self, learning_rate=0.001, p=0.1, gan=None, config=None, use_locking=False, name="GigaWolfOptimizer", optimizer=None, optimizer2=None):
super().__init__(use_locking, name)
self.gan = gan
self.config = config
self._lr_t = learning_rate
optimizer = hc.lookup_functions(optimizer)
self.optimizer = self.gan.create_optimizer(optimizer)
optimizer2 = hc.lookup_functions(optimizer2)
self.optimizer2 = self.gan.create_optimizer(optimizer2)
def create(self):
config = self.config
g_lr = config.g_learn_rate
d_lr = config.d_learn_rate
self.create_called = True
self.global_step = tf.train.get_global_step()
self.d_lr = d_lr
self.g_lr = g_lr
for hook_config in (config.hooks or []):
hook_config = hc.lookup_functions(hook_config.copy())
defn = {
k: v
for k, v in hook_config.items()
if k in inspect.getargspec(hook_config['class']).args
}
defn['gan'] = self.gan
defn['config'] = hook_config
defn['trainer'] = self
hook = hook_config["class"](**defn)
self.gan.components += [hook]
losses = hook.losses()
if losses[0] is not None:
self.gan.loss.sample[0] += losses[0]
if losses[1] is not None:
self.gan.loss.sample[1] += losses[1]
self.train_hooks.append(hook)
result = self._create()
for hook in self.train_hooks:
hook.after_create()
def __init__(self,
gan,
config,
d_vars=None,
g_vars=None,
loss=None,
name="BaseTrainer"):
self.current_step = 0
self.g_vars = g_vars
self.d_vars = d_vars
self.loss = loss
self.d_shake = None
self.g_shake = None
self.train_hooks = []
for hook_config in (config.hooks or []):
hook_config = hc.lookup_functions(hook_config.copy())
defn = {
k: v
for k, v in hook_config.items()
if k in inspect.getargspec(hook_config['class']).args
}
defn['gan'] = gan
defn['config'] = hook_config
defn['trainer'] = self
hook = hook_config["class"](**defn)
losses = hook.losses()
if losses[0] is not None:
self.loss.sample[0] += losses[0]
if losses[1] is not None:
self.loss.sample[1] += losses[1]
self.train_hooks.append(hook)
GANComponent.__init__(self, gan, config, name=name)
def __init__(self,
learning_rate=0.001,
p=0.1,
gan=None,
config=None,
use_locking=False,
name="SOSOptimizer",
optimizer=None,
alpha=1):
super().__init__(use_locking, name)
self._alpha = alpha
self.gan = gan
self.config = config
self._lr_t = learning_rate
def create_optimizer(klass, options):
options['gan'] = self.gan
options['config'] = options
defn = {
k: v
for k, v in options.items()
if k in inspect.getargspec(klass).args
}
learn_rate = options.learn_rate or options.learning_rate
if 'learning_rate' in options:
del defn['learning_rate']
return klass(learn_rate, **defn)
optimizer = hc.lookup_functions(optimizer)
self.optimizer = create_optimizer(optimizer['class'], optimizer)
def create(self):
config = self.config
g_lr = config.g_learn_rate
d_lr = config.d_learn_rate
self.create_called = True
self.global_step = tf.train.get_global_step()
self.d_lr = d_lr
self.g_lr = g_lr
for hook_config in (config.hooks or []):
hook_config = hc.lookup_functions(hook_config.copy())
defn = {k: v for k, v in hook_config.items() if k in inspect.getargspec(hook_config['class']).args}
defn['gan']=self.gan
defn['config']=hook_config
defn['trainer']=self
hook = hook_config["class"](**defn)
self.gan.components += [hook]
losses = hook.losses()
if losses[0] is not None:
self.gan.loss.sample[0] += losses[0]
if losses[1] is not None:
self.gan.loss.sample[1] += losses[1]
self.train_hooks.append(hook)
result = self._create()
for hook in self.train_hooks:
hook.after_create()
def __init__(self,
learning_rate=0.001,
decay=0.9,
gan=None,
config=None,
use_locking=False,
name="EmaOptimizer",
optimizer=None):
super().__init__(use_locking, name)
self._decay = decay
self.gan = gan
self.config = config
self.name = name
def create_optimizer(klass, options):
options['gan'] = self.gan
options['config'] = options
defn = {
k: v
for k, v in options.items()
if k in inspect.getargspec(klass).args
}
return klass(options.learn_rate, **defn)
optimizer = hc.lookup_functions(optimizer)
self.optimizer = create_optimizer(optimizer['class'], optimizer)
def discriminator(self, x, f,z,g,gz):
config = self.gan.config
batch_size = config['batch_size']*2
single_batch_size = config['batch_size']
channels = (config['channels'])
# combine to one batch, per Ian's "Improved GAN"
xs = [x]
gs = g
set_tensor("xs", xs)
set_tensor("gs", gs)
g = g[-1]
for i in gs:
resized = tf.image.resize_images(xs[-1],[int(xs[-1].get_shape()[1]//2),int(xs[-1].get_shape()[2]//2)], 1)
xs.append(resized)
xs.pop()
gs.reverse()
# careful on order. See https://arxiv.org/pdf/1606.00704v1.pdf
z = tf.concat(0, [z, gz])
discriminators = []
for i, discriminator in enumerate(config['discriminators']):
discriminator = hc.lookup_functions(discriminator)
discriminators.append(discriminator['create'](self.gan, discriminator, x, g, xs, gs,prefix="d_"+str(i)))
net = tf.concat(1, discriminators)
last_layer = net
last_layer = tf.reshape(last_layer, [batch_size, -1])
last_layer = tf.slice(last_layer, [single_batch_size, 0], [single_batch_size, -1])
d_real = tf.reshape(net, [batch_size, -1])
d_real = tf.slice(net, [0, 0], [single_batch_size, -1])
d_fake = tf.reshape(net, [batch_size, -1])
d_fake = tf.slice(net, [single_batch_size, 0], [single_batch_size, -1])
if config['y_dims'] == 1:
dr_class=None
dr_logits=None
df_class=None
df_logits=None
else:
num_classes = config['y_dims']+1
net = linear(net, num_classes, scope="d_fc_end", stddev=0.003)
net = layer_norm_1(batch_size*2, name='d_bn_end')(net)
class_logits = tf.slice(net, [0,1], [single_batch_size*2,num_classes-1])
gan_logits = tf.squeeze(tf.slice(net, [0,0], [single_batch_size*2,1]))
dr_class=tf.slice(class_logits, [0, 0], [single_batch_size, num_classes-1])
dr_logits=tf.slice(gan_logits, [0], [single_batch_size])
df_class=tf.slice(class_logits, [single_batch_size, 0], [single_batch_size, num_classes-1])
df_logits=tf.slice(gan_logits, [single_batch_size], [single_batch_size]),
return [dr_class,dr_logits,df_class,df_logits, last_layer, d_real, d_fake]
def create_z_encoding(self):
self.gan.graph.z = []
encoders = []
for i, encoder in enumerate(self.gan.config.encoders):
encoder = hc.Config(hc.lookup_functions(encoder))
zs, z_base = encoder.create(encoder, self.gan)
encoders.append(zs)
self.gan.graph.z.append(z_base)
z_encoded = tf.concat(axis=1, values=encoders)
self.gan.graph.z_encoded = z_encoded
return z_encoded
def create_optimizer(self, options):
options = hc.lookup_functions(options)
klass = options['class']
newopts = options.copy()
newopts['gan']=self.gan
newopts['config']=options
defn = {k: v for k, v in newopts.items() if k in inspect.getargspec(klass).args}
learn_rate = options.learn_rate or options.learning_rate
if 'learning_rate' in options:
del defn['learning_rate']
gan_component = klass(learn_rate, **defn)
self.components.append(gan_component)
return gan_component
def create_optimizer(self, options):
options = hc.lookup_functions(options)
klass = options['class']
newopts = options.copy()
newopts['gan']=self.gan
newopts['config']=options
defn = {k: v for k, v in newopts.items() if k in inspect.getargspec(klass).args}
learn_rate = options.learn_rate or options.learning_rate
if 'learning_rate' in options:
del defn['learning_rate']
gan_component = klass(learn_rate, **defn)
self.components.append(gan_component)
return gan_component
def generator(self, z, reuse=False):
config = self.gan.config
x_dims = config.x_dims
output_channels = config.channels
batch_size = config.batch_size
with (tf.variable_scope("generator", reuse=reuse)):
if 'y' in self.gan.graph:
z = tf.concat(axis=1, values=[z, self.gan.graph.y])
generator = hc.Config(hc.lookup_functions(config.generator))
nets = generator.create(generator, self.gan, z)
return nets
def discriminator(self, x, f, z, g, gz):
config = self.gan.config
graph = self.gan.graph
batch_size = config.batch_size * 2
single_batch_size = config.batch_size
channels = config.channels
# combine to one batch, per Ian's "Improved GAN"
xs = [x]
gs = g
graph.xs = xs
graph.gs = gs
g = g[-1]
if len(gs) > 1:
for i in gs:
resized = tf.image.resize_images(xs[-1], [
int(xs[-1].get_shape()[1] // 2),
int(xs[-1].get_shape()[2] // 2)
], 1)
xs.append(resized)
xs.pop()
gs.reverse()
discriminators = []
for i, discriminator in enumerate(config.discriminators):
discriminator = hc.Config(hc.lookup_functions(discriminator))
with (tf.variable_scope("discriminator")):
discriminators.append(
discriminator.create(self.gan,
discriminator,
x,
g,
xs,
gs,
prefix="d_" + str(i)))
def split_d(net, i):
net = tf.slice(net, [single_batch_size * i, 0],
[single_batch_size, -1])
return net
d_reals = [split_d(x, 0) for x in discriminators]
d_fakes = [split_d(x, 1) for x in discriminators]
net = tf.concat(axis=1, values=discriminators)
d_real = split_d(net, 0)
d_fake = split_d(net, 1)
return [d_real, d_fake, d_reals, d_fakes]
def step(self):
trainer = hc.Config(hc.lookup_functions(self.config['trainer']))
d_loss, g_loss = trainer.run(self.gan)
if (self.steps > 1 and (self.steps % self.args.sample_every == 0)):
sample_file = "samples/%06d.png" % (self.sampled)
self.create_path(sample_file)
print(str(self.steps) + ":", "Sample created " + sample_file)
sample_list = self.sample(sample_file)
if self.args.use_hc_io:
hc.io.sample(self.config, sample_list)
self.sampled += 1
self.steps += 1
return True
def discriminator(config, x, f,z,g,gz):
batch_size = config['batch_size']*2
single_batch_size = config['batch_size']
channels = (config['channels'])
# combine to one batch, per Ian's "Improved GAN"
xs = [x]
gs = g
set_tensor("xs", xs)
set_tensor("gs", gs)
g = g[-1]
for i in gs:
resized = tf.image.resize_images(xs[-1],[int(xs[-1].get_shape()[1]//2),int(xs[-1].get_shape()[2]//2)], 1)
xs.append(resized)
xs.pop()
gs.reverse()
# careful on order. See https://arxiv.org/pdf/1606.00704v1.pdf
z = tf.concat(0, [z, gz])
discriminators = []
for i, discriminator in enumerate(config['discriminators']):
discriminator = hc.lookup_functions(discriminator)
discriminators.append(discriminator['create'](config, discriminator, x, g, xs, gs,prefix="d_"+str(i)))
net = tf.concat(1, discriminators)
last_layer = net
last_layer = tf.reshape(last_layer, [batch_size, -1])
last_layer = tf.slice(last_layer, [single_batch_size, 0], [single_batch_size, -1])
num_classes = config['y_dims']+1
if config['y_dims'] == 1:
net = linear(net, 1, scope="d_fc_end", stddev=0.003)
class_logits = net
gan_logits = tf.squeeze(net)
else:
net = linear(net, num_classes, scope="d_fc_end", stddev=0.003)
class_logits = tf.slice(net, [0,1], [single_batch_size*2,num_classes-1])
gan_logits = tf.squeeze(tf.slice(net, [0,0], [single_batch_size*2,1]))
return [tf.slice(class_logits, [0, 0], [single_batch_size, num_classes-1]),
tf.slice(gan_logits, [0], [single_batch_size]),
tf.slice(class_logits, [single_batch_size, 0], [single_batch_size, num_classes-1]),
tf.slice(gan_logits, [single_batch_size], [single_batch_size]),
last_layer]
def __init__(self,
learning_rate=0.001,
p=0.1,
gan=None,
config=None,
use_locking=False,
name="CurlOptimizer",
optimizer=None,
rho=1,
beta=1,
gamma=1):
super().__init__(use_locking, name)
self._beta = beta
self._rho = rho
self._gamma = gamma
self.gan = gan
self.config = config
self._lr_t = learning_rate
self.g_rho = gan.configurable_param(self.config.g_rho)
self.d_rho = gan.configurable_param(self.config.d_rho)
if tf.contrib.framework.is_tensor(self.g_rho):
self.gan.add_metric("g_rho", self.g_rho)
if tf.contrib.framework.is_tensor(self.d_rho):
self.gan.add_metric("d_rho", self.d_rho)
def create_optimizer(klass, options):
options['gan'] = self.gan
options['config'] = options
defn = {
k: v
for k, v in options.items()
if k in inspect.getargspec(klass).args
}
learn_rate = options.learn_rate or options.learning_rate
if 'learning_rate' in options:
del defn['learning_rate']
return klass(learn_rate, **defn)
optimizer = hc.lookup_functions(optimizer)
self.optimizer = create_optimizer(optimizer['class'], optimizer)
def get_config_value(self, symbol):
if symbol in self.config:
config = hc.Config(hc.lookup_functions(self.config[symbol]))
return config
return None
def _create(self):
gan = self.gan
config = self.config
d_vars = gan.d_vars()
g_vars = gan.g_vars()
d_vars = list(set(d_vars).intersection(tf.trainable_variables()))
g_vars = list(set(g_vars).intersection(tf.trainable_variables()))
loss = self.gan.loss
d_loss, g_loss = loss.sample
allloss = d_loss + g_loss
allvars = d_vars + g_vars
d_grads = tf.gradients(d_loss, d_vars)
g_grads = tf.gradients(g_loss, g_vars)
grads = d_grads + g_grads
self.d_log = -tf.log(tf.abs(d_loss+TINY))
for g, d_v in zip(grads,d_vars):
if g is None:
print("!!missing gradient")
print(d_v)
return
apply_vec = []
apply_vec_d = []
apply_vec_g = []
for (i, grad, v) in zip(range(len(grads)), grads, allvars):
if grad == None:
print("WARNING: grad none", grad, v)
else:
apply_vec.append((grad, v))
if v in d_vars:
apply_vec_d.append((grad, v))
else:
apply_vec_g.append((grad, v))
optimizer = hc.lookup_functions(config.optimizer)
optimizer['gan']=self.gan
optimizer['config']=optimizer
defn = {k: v for k, v in optimizer.items() if k in inspect.getargspec(optimizer['class']).args}
lr = optimizer.learn_rate or optimizer.learning_rate
if 'learning_rate' in optimizer:
del defn['learning_rate']
tr = optimizer['class'](lr, **defn)
self.optimizer = tr
self.gan.trainer = self
self.g_loss = g_loss
self.d_loss = d_loss
self.gan.optimizer = tr
optimize_t = tr.apply_gradients(apply_vec, global_step=self.global_step)
d_optimize_t = tr.apply_gradients(apply_vec_d, global_step=self.global_step)
self.past_weights = []
self.g_loss = g_loss
self.d_loss = d_loss
self.slot_vars = tr.variables()
def _slot_var(x, g_vars):
for g in g_vars:
if x.name.startswith(g.name.split(":")[0]):
return True
return False
self.slot_vars_g = [x for x in self.slot_vars if _slot_var(x, g_vars)]
self.slot_vars_d = [x for x in self.slot_vars if _slot_var(x, d_vars)]
self.optimize_t = optimize_t
self.d_optimize_t = d_optimize_t
self.min_fitness=None
print("CONFIG ", config)
if config.fitness_type is not None:
mean = tf.zeros([1])
used_grads = d_grads
if config.grad_type == "sum":
for g in used_grads:
mean += tf.reduce_sum(tf.abs(g))
else:
for g in used_grads:
mean += tf.reduce_mean(tf.abs(g))
mean/=len(used_grads)
self.mean=mean
#self.mean=mean*100
if config.fitness_type == 'g_loss':
self.g_fitness = g_loss - (config.diversity_importance or 1) * tf.log(tf.abs(self.mean + d_loss - g_loss))
elif(config.fitness_type == 'gradient-only'):
self.g_fitness = -tf.log(reg)
elif(config.fitness_type == 'grads'):
self.g_fitness = mean
elif(config.fitness_type == 'point'):
self.g_fitness = mean - 1000*d_loss + 1000*g_loss
elif(config.fitness_type == 'fail'):
self.g_fitness = -mean
elif(config.fitness_type == 'fail2'):
self.g_fitness = -loss.d_fake
elif(config.fitness_type == 'fail3'):
self.g_fitness = -g_loss
elif(config.fitness_type == 'fail2-reverse'):
self.g_fitness = loss.d_fake
elif(config.fitness_type == 'ls'):
a,b,c = loss.config.labels
self.g_fitness = tf.square(loss.d_fake-a)
elif(config.fitness_type == 'ls-r'):
a,b,c = loss.config.labels
self.g_fitness = -tf.square(loss.d_fake-a)
elif(config.fitness_type == 'ls2'):
a,b,c = loss.config.labels
self.g_fitness = tf.square(loss.d_fake-c)
elif(config.fitness_type == 'ls2-r'):
a,b,c = loss.config.labels
self.g_fitness = -tf.square(loss.d_fake-c)
elif(config.fitness_type == 'std'):
self.g_fitness = -tf.nn.sigmoid(loss.d_fake)
elif(config.fitness_type == 'ls3'):
self.g_fitness = 1-loss.d_fake
elif(config.fitness_type == 'ls4'):
self.g_fitness = loss.d_real-loss.d_fake
elif(config.fitness_type == 'ls5'):
self.g_fitness = tf.square(loss.d_real)-tf.square(loss.d_fake)
elif(config.fitness_type == 'fq1'):
lam = 0.1
self.g_fitness = -loss.d_fake-lam*mean
elif(config.fitness_type == 'fq2'):
lam = 0.1
self.g_fitness = loss.d_real-loss.d_fake-lam*mean
elif(config.fitness_type == 'fq3'):
lam = 1
self.g_fitness = loss.d_real-loss.d_fake+lam*mean
elif(config.fitness_type == 'fq4'):
lam = 1
self.g_fitness = -loss.d_fake+lam*mean
elif(config.fitness_type == 'fq5'):
lam = 1
self.g_fitness = -loss.d_fake-lam*tf.norm(mean)
elif(config.fitness_type == 'fq6'):
lam = 0.1
self.g_fitness = -loss.d_fake-lam*tf.norm(mean+d_loss)
elif(config.fitness_type == 'fq7'):
lam = 0.1
self.g_fitness = -loss.d_fake-lam*tf.norm(-mean-d_loss)
elif(config.fitness_type == 'fq8'):
lam = 0.1
self.g_fitness = -tf.norm(mean+d_loss)
elif(config.fitness_type == 'fq9'):
lam = 0.1
self.g_fitness = lam*mean
elif(config.fitness_type == 'fq10'):
lam = 0.1
self.g_fitness = tf.norm(mean+d_loss)
elif(config.fitness_type == 'fq11'):
lam = 100.00
self.fq = -loss.d_fake
self.fd = lam * mean
self.g_fitness = -loss.d_fake + lam * mean
elif(config.fitness_type == 'ls3-fail'):
self.g_fitness = -(1-loss.d_fake)
elif(config.fitness_type == 'gldl'):
self.g_fitness = -d_loss + g_loss
elif(config.fitness_type == 'df'):
self.g_fitness = tf.abs(loss.d_fake) - tf.abs(loss.d_real)
elif(config.fitness_type == 'standard'):
self.g_fitness = tf.reduce_mean(g_loss) - (config.diversity_importance or 1)* tf.log(tf.abs(self.mean - tf.log(TINY+tf.sigmoid(d_loss)) - \
tf.log(1.0-tf.sigmoid(g_loss)+TINY)))
else:
self.g_fitness = tf.reduce_mean(loss.d_fake) - (config.diversity_importance or 1)* tf.log(tf.abs(self.mean + tf.reduce_mean(loss.d_real) - tf.reduce_mean(loss.d_fake)))
self.g_fitness = tf.reduce_mean(self.g_fitness)
return optimize_t, optimize_t
def train(self):
trainer = hc.Config(hc.lookup_functions(self.config.trainer))
return trainer.run(self)
def get_config_value(self, symbol):
if symbol in self.config:
config = hc.Config(hc.lookup_functions(self.config[symbol]))
return config
return None
def create(self, graph):
x = graph.x
f = graph.f
config = self.gan.config
# This is a hack to set dtype across ops.py, since each tensorflow instruction needs a dtype argument
# TODO refactor
set_ops_globals(config.dtype, config.batch_size)
batch_size = config.batch_size
g_losses = []
extra_g_loss = []
d_losses = []
z = self.create_z_encoding()
# create generator
g = self.generator(z)
g_sample = g
d_real, d_fake, d_reals, d_fakes = self.discriminator(x, f, None, g, z)
self.gan.graph.d_real = d_real
self.gan.graph.d_fake = d_fake
self.gan.graph.d_reals = d_reals
self.gan.graph.d_fakes = d_fakes
for i, loss in enumerate(config.losses):
loss = hc.Config(hc.lookup_functions(loss))
d_loss, g_loss = loss.create(loss, self.gan)
if (d_loss is not None):
d_losses.append(tf.squeeze(d_loss))
if (g_loss is not None):
g_losses.append(tf.squeeze(g_loss))
g_reg_losses = [
var
for var in tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
if 'g_' in var.name
]
d_reg_losses = [
var
for var in tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
if 'd_' in var.name
]
extra_g_loss += g_reg_losses
g_loss = tf.reduce_mean(tf.add_n(g_losses))
for extra in extra_g_loss:
g_loss += extra
d_loss = tf.reduce_mean(tf.add_n(d_losses))
#for extra in d_reg_losses:
# d_loss += extra
joint_loss = tf.reduce_mean(tf.add_n(g_losses + d_losses))
summary = tf.global_variables()
def summary_reduce(s):
if (len(s.get_shape()) == 0):
return s
while (len(s.get_shape()) > 1):
s = tf.reduce_mean(s, 1)
#s=tf.squeeze(s)
return tf.reduce_mean(s, 0)
summary = [(s.get_shape(), s.name, s.dtype, summary_reduce(s))
for s in summary]
graph.d_loss = d_loss
graph.d_log = -tf.log(tf.abs(d_loss + TINY))
graph.f = f
graph.g = g_sample
graph.g_loss = g_loss
graph.hc_summary = summary
graph.joint_loss = joint_loss
g_vars = [var for var in tf.trainable_variables() if 'g_' in var.name]
d_vars = [var for var in tf.trainable_variables() if 'd_' in var.name]
v_vars = [var for var in tf.trainable_variables() if 'v_' in var.name]
g_vars += v_vars
trainer = hc.Config(hc.lookup_functions(config.trainer))
g_optimizer, d_optimizer = trainer.create(trainer, self.gan, d_vars,
g_vars)
graph.d_optimizer = d_optimizer
graph.g_optimizer = g_optimizer
