def do_matops(shape=(100,100)):
A = uniform(shape)
B = uniform(shape)
C = matmul(A,B)
D = matmul(B,A)
eig(A);eig(B);eig(C);eig(D)
svd(A);svd(B);svd(C);eig(D)
def _interpolate(a, b=None):
if b is None: # interpolation in DRAGAN
beta = random.uniform(shape=shape(a), minval=0., maxval=1.)
b = a + 0.5 * math.reduce_std(a) * beta
shape_ = [shape(a)[0]] + [1] * (a.shape.ndims - 1)
alpha = random.uniform(shape=shape_, minval=0., maxval=1.)
inter = a + alpha * (b - a)
inter.set_shape(a.shape)
return inter
def get_initial_weights(K, N, L):
from tensorflow import random as tfrandom, int32 as tfint32
return {
'Alice': tfrandom.uniform((K, N),
minval=-L,
maxval=L + 1,
dtype=tfint32),
'Bob': tfrandom.uniform((K, N), minval=-L, maxval=L + 1,
dtype=tfint32),
# TODO: doesn't work for probabilistic:
'Eve': tfrandom.uniform((K, N), minval=-L, maxval=L + 1, dtype=tfint32)
}
def do_gradops(opt,loss,model,insh,t=None):
with GradientTape() as tape:
y = do_eval(model, insh)[1]
if t == None:
t = uniform(shape(y))
l = loss(t,y)
grad = tape.gradient(l,model.trainable_weights)
opt.apply_gradients(zip(grad,model.trainable_weights))
def train_g(self):
z = random.uniform((self.batch_size, 1, 1, self.z_dim))
with tf.GradientTape() as t:
x_fake = self.G(z, training=True)
fake_logits = self.D(x_fake, training=True)
loss = ops.g_loss_fn(fake_logits)
grad = t.gradient(loss, self.G.trainable_variables)
self.g_opt.apply_gradients(zip(grad, self.G.trainable_variables))
return loss
def trainable(config, reporter):
"""
Args:
config (dict): Parameters provided from the search algorithm
or variant generation.
"""
if not isinstance(config['update_rule'], str):
update_rule = update_rules[int(config['update_rule'])]
else:
update_rule = config['update_rule']
K, N, L = int(config['K']), int(config['N']), int(config['L'])
run_name = f"run-{get_session_num(logdir)}"
run_logdir = join(logdir, run_name)
# for each attack, the TPMs should start with the same weights
initial_weights_tensors = get_initial_weights(K, N, L)
training_steps_ls = {}
eve_scores_ls = {}
losses_ls = {}
# for each attack, the TPMs should use the same inputs
seed = tfrandom.uniform([],
minval=0,
maxval=tfint64.max,
dtype=tfint64).numpy()
for attack in ['none', 'geometric']:
initial_weights = {
tpm: weights_tensor_to_variable(weights, tpm)
for tpm, weights in initial_weights_tensors.items()
}
tfrandom.set_seed(seed)
if tensorboard:
attack_logdir = join(run_logdir, attack)
attack_writer = tensorflow.summary.create_file_writer(
attack_logdir)
with attack_writer.as_default():
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
else:
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
training_steps_ls[attack] = training_steps
eve_scores_ls[attack] = sync_scores
losses_ls[attack] = loss
avg_training_steps = tensorflow.math.reduce_mean(
list(training_steps_ls.values()))
avg_eve_score = tensorflow.math.reduce_mean(
list(eve_scores_ls.values()))
mean_loss = tensorflow.math.reduce_mean(list(losses_ls.values()))
reporter(
avg_training_steps=avg_training_steps.numpy(),
avg_eve_score=avg_eve_score.numpy(),
mean_loss=mean_loss.numpy(),
done=True,
)
def gradient_penalty(self, f, real, fake):
alpha = random.uniform([self.batch_size, 1, 1, 1], 0., 1.)
diff = fake - real
inter = real + (alpha * diff)
with tf.GradientTape() as t:
t.watch(inter)
pred = f(inter)
grad = t.gradient(pred, [inter])[0]
slopes = tf.sqrt(tf.reduce_sum(tf.square(grad), axis=[1, 2, 3]))
gp = tf.reduce_mean((slopes - 1.)**2)
return gp
def build_model_dense(architecture=None):
if architecture == None:
depth = int(uniform([1],2,10))
units = [2**int(uniform([1],2,9)) for _ in range(depth)]
activations = ['elu','exponential','linear','relu','selu','swish','tanh']
architecture = []
for i in range(depth):
j = int(uniform([1],0,len(activations)))
doNorm = int(uniform([1],0,2))
architecture.append((units[i],activations[j],doNorm))
inp = Input(architecture[0][0])
x = Dense(architecture[0][0],activation=architecture[-0][1])(inp)
for units,acti,doNorm in architecture[1:-1]:
x = Dense(units,activation=acti)(x)
if doNorm:
x = BatchNormalization()(x)
out = Dense(architecture[-1][0],activation=architecture[-1][1])(x)
loss = 'categorical_crossentropy'
optimizers = ['adadelta','adagrad','adam','adamax','ftrl','nadam','rmsprop','sgd']
optimizer = optimizers[int(uniform([1],0,len(optimizers)))]
return Model(inputs=inp,outputs=out),architecture[0][0],loss,optimizer
def train_d(self, x_real):
z = random.uniform((self.batch_size, 1, 1, self.z_dim))
with tf.GradientTape() as t:
x_fake = self.G(z, training=True)
fake_logits = self.D(x_fake, training=True)
real_logits = self.D(x_real, training=True)
cost = ops.d_loss_fn(fake_logits, real_logits)
gp = self.gradient_penalty(partial(self.D, training=True), x_real,
x_fake)
cost += self.grad_penalty_weight * gp
grad = t.gradient(cost, self.D.trainable_variables)
self.d_opt.apply_gradients(zip(grad, self.D.trainable_variables))
return cost
def apply_phaseshuffle(args):
x, rad = args
pad_type = 'reflect'
b, x_len, nch = x.get_shape().as_list()
phase = random.uniform([], minval=-rad, maxval=rad + 1, dtype=int32)
pad_l = maximum(phase, 0)
pad_r = maximum(-phase, 0)
phase_start = pad_r
x = pad(x, [[0, 0], [pad_l, pad_r], [0, 0]], mode=pad_type)
x = x[:, phase_start:phase_start + x_len]
x.set_shape([b, x_len, nch])
return x
def test(generator, mapping_net):
"""
Test the model.
:param generator: generator model
:return: None
"""
img = np.array(
generator(adain_net,
mapping_net(uniform(batch_size, z_dim), minval=-1, maxval=1),
np.random.randint(num_genres + 1, size=(batch_size, ))))
### Below, we've already provided code to save these generated images to files on disk
# Rescale the image from (-1, 1) to (0, 255)
img = ((img / 2) - 0.5) * 255
# Convert to uint8
img = img.astype(np.uint8)
# Save images to disk
for i in range(0, batch_size):
img_i = img[i]
s = out_dir + '/' + str(i) + '.png'
imwrite(s, img_i)
from tensorflow.compat.v1 import logging, placeholder, global_variables_initializer, Session
from tensorflow import Variable, random, zeros, name_scope, nn, matmul, sigmoid, reduce_mean, Graph
from tensorflow.compat.v1.train import GradientDescentOptimizer, Saver
from numpy import array, square
logging.set_verbosity(logging.ERROR) #Не показываем лишние предупреждения
with Graph().as_default(): #Открываем граф как главный
X = placeholder("float32", shape=[4, 2],
name='X') #Создаём объект для хранения входа
Y = placeholder("float32", shape=[4, 1],
name='Y') #Создаём объект для хранения выхода
W = Variable(
random.uniform([2, 2], -1, 1), name="W"
) #Создаём переменную (наклон функции) с рандомным значением между -1 и 1 для входа
w = Variable(
random.uniform([2, 1], -1, 1), name="w"
) #Создаём переменную (наклон функции) с рандомным значением между -1 и 1 для выхода
c = Variable(
zeros([4, 2]), name="c"
) #Создаём переменную (смещение по оси координат) с нулями для входа
b = Variable(
zeros([4, 1]), name="b"
) #Создаём переменную (смещение по оси координат) с нулями для выхода
with name_scope("hidden_layer") as scope:
h = nn.relu(
matmul(X, W) + c
def train(
generator,
discriminator,
dataset,
genre_labels,
manager,
mapping_net,
noise_net,
adain_net,
):
"""
Train the model for one epoch. Save a checkpoint every 500 or so batches.
:param generator: generator model
:param discriminator: discriminator model
:param dataset: list of all album covers
:param manager: the manager that handles saving checkpoints by calling save()
:return: The average FID score over the epoch
"""
sum_fid = 0
indices = tf.random.shuffle(tf.range(len(genre_labels)))
num_examples = len(indices)
# Loop over our data until we run out
for i in range(num_examples):
batch = tf.gather(
dataset, indices[i:i + batch_size if i +
batch_size < num_examples else num_examples])
labels = tf.gather(
genre_labels, indices[i:i + batch_size if i +
batch_size < num_examples else num_examples])
z = uniform((batch_size, z_dim), minval=-1, maxval=1)
with GradientTape(persistent=True) as tape:
w = mapping_net(z)
# generated images
G_sample = generator(adain_net, w, labels)
# test discriminator against real images
logits_real = discriminator(batch, labels)
# re-use discriminator weights on new inputs
logits_fake = discriminator(G_sample, labels)
g_loss = generator_loss(logits_fake)
# g_loss = tf.reduce_sum(p)
#g_loss = tf.reduce_sum(G_sample)
d_loss = discriminator_loss(logits_real, logits_fake)
map_grads = tape.gradient(g_loss, mapping_net.trainable_variables
) # success measured by same parameters
map_optimizer.apply_gradients(
zip(map_grads, mapping_net.trainable_variables))
a_grads = tape.gradient(g_loss, adain_net.trainable_variables
) # success measured by same parameters
adain_optimizer.apply_gradients(
zip(a_grads, adain_net.trainable_variables))
# optimize the generator and the discriminator
gen_gradients = tape.gradient(g_loss, generator.trainable_variables)
generator_optimizer.apply_gradients(
zip(gen_gradients, generator.trainable_variables))
if (i % num_gen_updates == 0):
disc_gradients = tape.gradient(d_loss,
discriminator.trainable_variables)
discriminator_optimizer.apply_gradients(
zip(disc_gradients, discriminator.trainable_variables))
# Save
if i % args.save_every == 0:
manager.save()
# Calculate inception distance and track the fid in order
# to return the average
if i % 500 == 0:
fid_ = fid_function(batch, G_sample)
print('**** D_LOSS: %g ****' % d_loss)
print('**** G_LOSS: %g ****' % g_loss)
print('**** INCEPTION DISTANCE: %g ****' % fid_)
sum_fid += fid_
return sum_fid / (i // 500)
def do_eval(model, insh):
x = uniform(insh)
y = model(x)
return (x,y)
def do_prepops(shape=(10000,50,50)):
ds = uniform(shape)
permutation = shuffle(range(shape[0]))
permuted_ds = gather(ds, permutation, axis=0)