def test_weights_clip(self):
"""WeightsClip should restrict values after optimizer updates"""
constraint = WeightsClip(min_value=-0.001, max_value=0.001)
initializer = Constant(1.0)
layer = Dense(10,
kernel_constraint=constraint,
bias_constraint=constraint,
kernel_initializer=initializer,
bias_initializer=initializer)
with tf.GradientTape() as tape:
data = tf.random.normal(shape=(1, 5))
pred = layer(data)
true_lb = tf.ones_like(pred)
loss = MeanSquaredError()(true_lb, pred)
# before update, weights should equal to initial values
for weight in layer.weights:
assert np.allclose(weight.numpy(), 1.0)
optimizer = SGD(learning_rate=0.0001)
grad = tape.gradient(loss, layer.trainable_variables)
optimizer.apply_gradients(zip(grad, layer.trainable_variables))
# after update, weights should within constraint ranges
for weight in layer.weights:
assert np.all(-0.001 <= weight.numpy())
assert np.all(weight.numpy() <= 0.001)
class DQNAgent:
def __init__(self, num_actions=8, epsilon=1, epsilon_min=0.1, epsilon_decay=0.9995, load=load):
self.epsilon = epsilon
self.epsilon_min = epsilon_min
self.epsilon_decay = epsilon_decay
self.discount_factor = discount_factor
self.num_actions = num_actions
self.optimizer = SGD(learning_rate)
if load:
print("Loading model from: ", model_savefolder)
self.dqn = tf.keras.models.load_model(model_savefolder)
else:
self.dqn = DQN(self.num_actions)
self.target_net = DQN(self.num_actions)
def update_target_net(self):
self.target_net.set_weights(self.dqn.get_weights())
def choose_action(self, state):
if self.epsilon < np.random.uniform(0,1):
action = int(tf.argmax(self.dqn(tf.reshape(state, (1,30,45,1))), axis=1))
else:
action = np.random.choice(range(self.num_actions), 1)[0]
return action
def train_dqn(self, samples):
screen_buf, actions, rewards, next_screen_buf, dones = split_tuple(samples)
row_ids = list(range(screen_buf.shape[0]))
ids = extractDigits(row_ids, actions)
done_ids = extractDigits(np.where(dones)[0])
with tf.GradientTape() as tape:
tape.watch(self.dqn.trainable_variables)
Q_prev = tf.gather_nd(self.dqn(screen_buf), ids)
Q_next = self.target_net(next_screen_buf)
Q_next = tf.gather_nd(Q_next, extractDigits(row_ids, tf.argmax(agent.dqn(next_screen_buf), axis=1)))
q_target = rewards + self.discount_factor * Q_next
if len(done_ids)>0:
done_rewards = tf.gather_nd(rewards, done_ids)
q_target = tf.tensor_scatter_nd_update(tensor=q_target, indices=done_ids, updates=done_rewards)
td_error = tf.keras.losses.MSE(q_target, Q_prev)
gradients = tape.gradient(td_error, self.dqn.trainable_variables)
self.optimizer.apply_gradients(zip(gradients, self.dqn.trainable_variables))
if self.epsilon > self.epsilon_min:
self.epsilon *= self.epsilon_decay
else:
self.epsilon = self.epsilon_min
def generate_adversarial(
model,
x_0,
adv_radius,
adv_type,
training_type,
adv_policy,
step_size=gin.REQUIRED,
max_iter=gin.REQUIRED,
n_stop=gin.REQUIRED,
l1_regularization=gin.REQUIRED
): # l1 special case of intermdiate activation => merge into ManifoldSampler
if adv_policy == 'adv':
noise = tf.random.uniform(shape=x_0.shape,
minval=-adv_radius,
maxval=adv_radius)
noise = ball_project(noise, x_0, adv_radius)
delta = tf.Variable(initial_value=noise, trainable=True)
elif adv_policy == 'gan':
noise = tf.random.uniform(shape=x_0.shape, minval=0., maxval=1.)
x = tf.Variable(initial_value=noise, trainable=True)
lr = step_size * (adv_radius / max_iter)
optimizer = SGD(learning_rate=lr) # sufficient considering loss landscape
manifold = ManifoldForward(model, n_stop)
for _ in range(max_iter):
with tf.GradientTape(watch_accessed_variables=False) as tape:
if adv_policy == 'adv':
tape.watch(delta)
x = x_0 + delta
elif adv_policy == 'gan':
tape.watch(x)
y, z_map = manifold.sample(x)
loss = goal(training_type, adv_type, y)
if z_map is not None:
z_loss = manifold.representation_goal(z_map)
loss = loss + z_loss
if l1_regularization is not None:
l1 = l1_penalty(x) if adv_policy == 'gan' else l1_penalty(
delta)
loss = loss + l1_regularization * l1
diff_target = x if adv_policy == 'gan' else delta
grad_f = tape.gradient(loss, [diff_target])
grad_f = renormalize_grads(grad_f)
optimizer.apply_gradients(zip(grad_f, [diff_target]))
if adv_policy == 'adv':
delta.assign(ball_project(
delta, x_0, adv_radius)) # project back to ball in manifold
elif adv_policy == 'gan':
x.assign(tf.clip_by_value(x, 0., 1.)) # image set
if adv_policy == 'adv':
x = x_0 + delta.value()
x = tf.clip_by_value(x, 0., 1.)
return x
elif adv_policy == 'gan':
return x.value()
def styleTransfer(sourcepath, stylepath):
path = 'static/uploads'
base_image_path = os.path.join(path, "source.png")
style_reference_image_path = os.path.join(path, "style.png")
result_prefix = "static/results/result"
global content_weight
global total_variation_weight
global style_weight
total_variation_weight = 1e-6
style_weight = 3e-6
content_weight = 5e-7
width, height = image.load_img(base_image_path).size
global img_nrows
global img_ncols
img_nrows = 400
img_ncols = int(width * img_nrows / height)
content_img = preprocess_image(base_image_path)
shape = content_img.shape[1:]
model = VGG19(weights="imagenet", include_top=False, input_shape=shape)
outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])
global feature_extractor
feature_extractor = Model(inputs=model.inputs, outputs=outputs_dict)
global style_layer_names
style_layer_names = [
"block1_conv1",
"block2_conv1",
"block3_conv1",
"block4_conv1",
"block5_conv1",
]
global content_layer_name
content_layer_name = "block5_conv2"
optimizer = SGD(
schedules.ExponentialDecay(initial_learning_rate=100.0,
decay_steps=100,
decay_rate=0.96))
base_image = preprocess_image(base_image_path)
style_reference_image = preprocess_image(style_reference_image_path)
combination_image = tf.Variable(preprocess_image(base_image_path))
iterations = 100
for i in range(1, iterations + 1):
loss, grads = compute_loss_and_grads(combination_image, base_image,
style_reference_image)
optimizer.apply_gradients([(grads, combination_image)])
print("Iteration %d: loss=%.2f" % (i, (loss)))
print(combination_image)
img = deprocess_image(combination_image.numpy())
fname = result_prefix + ".png"
image.save_img(fname, img)
def train(model: Model, loss: Loss, optimizer: SGD, x: tf.Tensor, y: tf.Tensor) -> float:
# Forward
with tf.GradientTape() as tape:
y_pred = model(x)
output = tf.reduce_mean(loss(y, y_pred))
# Backward
grad = tape.gradient(output, model.trainable_variables)
# Update parameters
optimizer.apply_gradients(zip(grad, model.trainable_variables))
return output.numpy()
def train(model: Sequential, loss: Loss, optimizer: SGD, x: tf.Tensor,
y: tf.Tensor) -> float:
with tf.GradientTape() as tape:
# Forward
x = tf.reshape(x, shape=(x.shape[0], 1))
y_pred = model(x)
y_pred = tf.squeeze(y_pred)
output = tf.reduce_mean(loss(y, y_pred))
# Backward
grad = tape.gradient(output, model.trainable_variables)
# Update parameters
optimizer.apply_gradients(zip(grad, model.trainable_variables))
return output.numpy()
def train_experimental_model(train_images, train_labels, test_images,
test_labels):
logging.info('Training network with two output layers...')
model = utils.create_model('VGG_8_simple_with_two_outputs')
train_dataset = tf.data.Dataset.from_tensor_slices(
(train_images, train_labels)).batch(64)
test_dataset = tf.data.Dataset.from_tensor_slices(
(test_images, test_labels)).batch(64)
loss_object = tf.keras.losses.CategoricalCrossentropy(from_logits=True)
optimizer = SGD(lr=0.001, momentum=0.9)
train_loss_results = []
train_accuracy_results = []
num_epochs = 200
for epoch in range(num_epochs):
epoch_loss_avg = tf.keras.metrics.Mean()
epoch_accuracy = tf.keras.metrics.CategoricalAccuracy()
for x, y in train_dataset:
# Optimize the model
loss_value, grads = grad(model, x, y, loss_object)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
# Track progress
epoch_loss_avg.update_state(loss_value) # Add current batch loss
y1_, y2_ = model(x, training=True)
epoch_accuracy.update_state(y, y1_)
# End epoch
train_loss_results.append(epoch_loss_avg.result())
train_accuracy_results.append(epoch_accuracy.result())
logging.info(
f'Epoch {epoch:03d}: Loss: {epoch_loss_avg.result():.3f}, '
f'Accuracy: {epoch_accuracy.result():.3%}')
# Do a test after some epochs, and also at the end
if epoch % 10 == 0 or epoch == num_epochs - 1:
test_current_accuracy(model, test_dataset)
def test_discriminator_weight_updates(self):
"""Discriminator weights should be restrcited into constraint range"""
inputs = Input((64, 64, 3))
discriminator = WDiscriminator()(inputs)
discriminator = Model(inputs=inputs, outputs=discriminator)
with tf.GradientTape() as tape:
data = tf.random.normal((1, 64, 64, 3))
pred = discriminator(data, training=True)
true_lb = tf.ones_like(pred)
loss = MeanSquaredError()(true_lb, pred)
optimizer = SGD(learning_rate=1.0)
grad = tape.gradient(loss, discriminator.trainable_variables)
optimizer.apply_gradients(zip(grad, discriminator.trainable_variables))
# after update, weights should within constraint ranges
for weight in discriminator.trainable_variables:
assert np.all(-0.01 <= weight.numpy())
assert np.all(weight.numpy() <= 0.01)
class AVNet(Model):
def __init__(self, n_features, n_candidates, n_voters, inp_shape, opt,
l_rate, arch):
'''
Simple version: 1 input layer (relu), 1 hidden layer (relu), 1 output layer (softmax)
Extras:
- Dropout
- BatchNormalization
'''
super(AVNet, self).__init__()
# This architecture variable allows us to run concurrent experiments automatically
self.arch = arch
# Define layers
self.input_layer = Dense(n_features,
activation='relu',
input_shape=inp_shape)
if self.arch == 1:
self.mid_layer = Dense(n_candidates * n_features,
activation='relu')
self.last_layer = Dense(n_candidates * n_voters, activation='relu')
elif self.arch == 2:
self.mid_layer = Dense(n_candidates * n_voters, activation='relu')
self.last_layer = Dense(n_candidates * n_features,
activation='relu')
elif self.arch == 3:
self.mid_layer = Dense(n_candidates * n_voters, activation='relu')
self.last_layer = Dense(n_candidates * n_features,
activation='linear')
elif self.arch == 4:
self.mid_layer = Dense(n_candidates * n_features,
activation='relu')
self.last_layer = Dense(n_candidates * n_voters,
activation='linear')
elif self.arch == 5:
self.mid_layer = Dense(n_candidates * n_voters,
activation='linear')
self.last_layer = Dense(n_candidates * n_features,
activation='linear')
elif self.arch == 6:
self.mid_layer = Dense(n_candidates * n_features,
activation='linear')
self.last_layer = Dense(n_candidates * n_voters,
activation='linear')
elif self.arch == 7:
self.mid_layer = Dense(128, activation='relu')
self.last_layer = Dense(256, activation='relu')
elif self.arch == 8:
self.mid_layer = Dense(128, activation='linear')
self.last_layer = Dense(256, activation='linear')
elif self.arch == 9:
self.mid_layer = Dense(n_candidates * n_features,
activation='relu')
self.last_layer = Dense(n_candidates * n_features,
activation='relu')
elif self.arch == 10:
self.mid_layer = Dense(n_candidates * n_features,
activation='linear')
self.last_layer = Dense(n_candidates * n_features,
activation='linear')
elif self.arch == 11:
self.mid_layer = Dense(n_candidates * n_features,
activation='relu')
self.last_layer = Dense(n_candidates * n_features,
activation='linear')
# Voter level, candidate level
elif self.arch == 12:
self.mid_layer = Dense(n_candidates * n_features,
activation='relu')
self.extra_layer = Dense(n_candidates * n_voters,
activation='relu')
self.last_layer = Dense(n_candidates * n_features,
activation='relu')
elif self.arch == 13:
self.mid_layer = Dense(n_candidates * n_features,
activation='relu')
self.extra_layer = Dense(n_candidates * n_voters,
activation='linear')
self.last_layer = Dense(n_candidates * n_features,
activation='relu')
elif self.arch == 14:
self.mid_layer = Dense(n_candidates * n_features,
activation='linear')
self.extra_layer = Dense(n_candidates * n_voters,
activation='linear')
self.last_layer = Dense(n_candidates * n_features,
activation='linear')
else:
print("Must enter a valid network architecture! [1-8]")
sys.exit(1)
self.leaky_relu = LeakyReLU(alpha=0.3)
self.dropout = Dropout(0.2)
self.batch_norm = BatchNormalization()
self.scorer = Dense(n_candidates, activation='softmax')
if opt == "Adam":
self.optimizer = Adam(learning_rate=l_rate)
elif opt == "Adagrad":
self.optimizer = Adagrad(learning_rate=l_rate)
else:
self.optimizer = SGD(learning_rate=l_rate)
self.CCE = CategoricalCrossentropy()
self.av_losses = []
def reset_scores(self):
# Scoring functions
self.total_condorcet = 0
self.total_majority = 0
self.total_plurality = 0
self.total_IM = 0
self.condorcet_score = 0
self.majority_score = 0
self.plurality_score = 0
self.IM_score = 0
def call(self, inputs, **kwargs):
""" Inputs is some tensor version of the ballots in an AVProfile
For testing purposes we will use AVProfile.rank_matrix, which represents
the count of each candidate per rank """
x = self.input_layer(inputs)
if self.arch in [1, 2, 7, 8, 9]:
x = self.mid_layer(x)
x = self.dropout(x)
x = self.last_layer(x)
x = self.dropout(x)
elif self.arch in [3, 4, 11]:
x = self.mid_layer(x)
x = self.dropout(x)
x = self.last_layer(x)
x = self.leaky_relu(x)
x = self.dropout(x)
elif self.arch in [5, 6, 10]:
x = self.mid_layer(x)
x = self.leaky_relu(x)
x = self.dropout(x)
x = self.last_layer(x)
x = self.leaky_relu(x)
x = self.dropout(x)
elif self.arch == 12:
x = self.mid_layer(x)
x = self.dropout(x)
x = self.extra_layer(x)
x = self.dropout(x)
x = self.last_layer(x)
# x = self.dropout(x)
elif self.arch == 13:
x = self.mid_layer(x)
x = self.dropout(x)
x = self.extra_layer(x)
x = self.leaky_relu(x)
x = self.dropout(x)
x = self.last_layer(x)
# x = self.dropout(x)
elif self.arch == 14:
x = self.mid_layer(x)
x = self.leaky_relu(x)
x = self.dropout(x)
x = self.extra_layer(x)
x = self.leaky_relu(x)
x = self.dropout(x)
x = self.last_layer(x)
x = self.leaky_relu(x)
# x = self.dropout(x)
return self.scorer(x)
def av_loss(self, profile, verbose=False):
""" An av_loss function will take an instance of <profile, condorcet_winner, majority_winner, simulator_list>, where
- profile = AVProfile.rank_matrix (shape: n_candidates x n_candidates)
- condorcet_winner = AVProfile.condorcet_winner (one-hot vector of size n_candidates)
- majority_winner = AVProfile.majority_winner (one-hot vector of size n_candidates)
- (IIA or IM)_simulator = AVProfile.IIA_profiles, AVProfile.IM_profiles (list of matrices of shape: n_candidates x n_candidates)
Idea: For 1 profile, the av_loss will be the sum of all the loss components according to the constraints above.
-> When there is a one-hot vector, we will use cross entropy
-> When there is a simulator list, we will add up the cross entropy losses after "running the election"
Extras:
-> Add a regularization term for the sum of cross entropy in the simulator list case
"""
# Original profile rank matrix
profile_matrix = profile.flatten_rank_matrix()
# Winners as strings
condorcet_w = profile.condorcet_w
majority_w = profile.majority_w
plurality_w = profile.plurality_w
# One-hot vectors
condorcet_w_vec = profile.condorcet_w_vector
majority_w_vec = profile.majority_w_vector
plurality_w_vec = profile.plurality_w_vector
# Make a forward pass to the profile
predictions = self.call(profile_matrix)
pred_w = get_winner(predictions, profile.candidates)
# Simulated preferences according to the predicted winner
alternative_profiles = profile.IM_rank_matrices[get_winner(
predictions, profile.candidates, out_format="idx")]
alternative_profiles = [
profile.flatten_rank_matrix(alt_profile)
for alt_profile in alternative_profiles
]
# Keep track of scores
if condorcet_w != "No Condorcet Winner":
self.total_condorcet += 1
if pred_w == condorcet_w:
self.condorcet_score += 1
if majority_w != "No Majority Winner":
self.total_majority += 1
if pred_w == majority_w:
self.majority_score += 1
else: # We will use Plurality if no Condorcet or Majority winner
self.total_plurality += 1
if pred_w == plurality_w:
self.plurality_score += 1
# Interpret the results
if verbose:
print(
"------------------------------------------------------------")
# print("Profile rank matrix:", profile.rank_matrix)
print("Condorcet winner:", condorcet_w)
print("Majority winner:", majority_w)
print("Plurality winner:", plurality_w)
print("-----------------")
print("Predicted winner:",
get_winner(predictions, profile.candidates))
print("Predicted winner quality scores:", predictions)
print(
"------------------------------------------------------------")
def loss(plurality_c, pred_c):
alt_profiles_score = 0
# Calculate IM score first - sum of crossentropy for all alternative winners given an IM profile
IM_score = 0
for IM_alt_profile in alternative_profiles:
alt_winner = self.call(IM_alt_profile)
pred_c_vec = get_winner(pred_c,
profile.candidates,
out_format="one-hot")
# Keep track of when winners match between altered profiles and original profile
if get_winner(alt_winner, profile.candidates) == get_winner(
pred_c, profile.candidates):
alt_profiles_score += 1
IM_score += self.CCE(pred_c_vec, alt_winner)
# Store scores
self.total_IM += len(alternative_profiles)
self.IM_score += alt_profiles_score
if verbose:
print(
f"IM score: {alt_profiles_score}/{len(alternative_profiles)}"
)
# Regularize IM score by the number of simulated profiles
IM_score /= len(alternative_profiles)
# If there isn't a Condorcet or a Majority winner, just use the Plurality winner
if count_nonzero(condorcet_w_vec) == 0 and count_nonzero(
majority_w_vec) == 0:
plurality_score = self.CCE(plurality_c, pred_c)
return plurality_score + IM_score
else:
condorcet_score = self.CCE(condorcet_w_vec, pred_c)
majority_score = self.CCE(majority_w_vec, pred_c)
if count_nonzero(condorcet_w_vec) != 0 and count_nonzero(
majority_w_vec) == 0:
return condorcet_score + IM_score
if count_nonzero(condorcet_w_vec
) == 0 and count_nonzero(majority_w_vec) != 0:
return majority_score + IM_score
else:
return majority_score + condorcet_score + IM_score
# Return loss function
return loss(plurality_c=plurality_w_vec, pred_c=predictions)
def calculate_grad(self, profile, verbose=False):
with GradientTape() as tape:
loss_value = self.av_loss(profile, verbose)
return loss_value, tape.gradient(loss_value, self.trainable_variables)
def train(self, profiles, epochs):
for _ in tqdm(range(epochs)):
self.reset_scores()
# print(f"Epoch {epoch} ==============================================================")
# Iterate through the list of profiles, not datasets
for i, profile in enumerate(profiles):
# Perform forward pass + calculate gradients
# if i % 33 == 0:
# loss_value, grads = self.calculate_grad(profile, verbose=True)
# print(f"Step: {self.optimizer.iterations.numpy()}, Loss: {loss_value}")
# else:
loss_value, grads = self.calculate_grad(profile)
self.av_losses.append(loss_value)
self.optimizer.apply_gradients(
zip(grads, self.trainable_variables))
# print("\n\n")
def get_results(self):
print("===================================================")
print("AVNet Results")
print("---------------------------------------------------")
results = dict()
if self.total_condorcet != 0:
print(
f"Condorcet Score: {self.condorcet_score}/{self.total_condorcet} = {self.condorcet_score / self.total_condorcet}"
)
results["Condorcet Score"] = (
f"{self.condorcet_score}/{self.total_condorcet}",
self.condorcet_score / self.total_condorcet)
else:
print("No Condorcet Winners")
results["Condorcet Score"] = (0, 0)
if self.total_majority != 0:
print(
f"Majority Score: {self.majority_score}/{self.total_majority} = {self.majority_score / self.total_majority}"
)
results["Majority Score"] = (
f"{self.majority_score}/{self.total_majority}",
self.majority_score / self.total_majority)
else:
print("No Majority Winners")
results["Majority Score"] = (0, 0)
if self.total_plurality != 0:
print(
f"Plurality Score: {self.plurality_score}/{self.total_plurality} = {self.plurality_score / self.total_plurality}"
)
results["Plurality Score"] = (
f"{self.plurality_score}/{self.total_plurality}",
self.plurality_score / self.total_plurality)
else:
print("No Plurality Winners were used")
results["Plurality Score"] = (0, 0)
print(
f"IM Score: {self.IM_score}/{self.total_IM} = {self.IM_score / self.total_IM}"
)
results["IM Score"] = (f"{self.IM_score}/{self.total_IM}",
self.IM_score / self.total_IM)
print(f"IM CCE Mean:", mean(self.av_losses))
results["IM CCE Score"] = round(mean(self.av_losses), 3)
return results
metrics = {'acc': 0.0, 'loss': 0.0, 'val_acc': 0.0, 'val_loss': 0.0, 'lr': lr}
for epoch in range(n_epochs):
# Iterate through the training set
print("\epoch {}/{}".format(epoch+1,n_epochs))
progress_bar = Progbar(train_size, stateful_metrics=list(metrics.keys()))
train_gen = train_ds.take(train_size//batch_size)
for batch_idx, (x, y) in enumerate(train_gen):
with tf.GradientTape() as tape:
y_pred = model(x, training=True)
loss = loss_fn(y, y_pred)
gradients = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(zip(gradients, model.trainable_weights))
acc_metric.update_state(y, y_pred)
train_step += 1
progress_bar.update(batch_idx*batch_size, values=[('acc',acc_metric.result()),
('loss', loss)])
# print(batch_idx, " / ", len(train_gen))
with train_writer.as_default():
tf.summary.scalar("Loss", loss, step=epoch)
tf.summary.scalar(
"Accuracy", acc_metric.result(), step=epoch
)
# reset accuracy between epochs (and for testing and test)
acc_metric.reset_states()
x = tf.random.uniform(shape=(n, ))
print(x.shape)
noise = tf.random.normal(shape=(len(x), ))
print(noise.shape)
y = m * x + b + noise
return x, y
m = 1
b = 2
Xtrain, ytrain = MakeNoisyData(m, b)
model = tf.keras.Sequential([
Dense(12, activation="relu", input_shape=(1, )),
Dense(12, activation="relu"),
Dense(1)
])
loss = MeanSquaredError()
print(model.summary())
optimizer = SGD(learning_rate=0.05, momentum=0.9)
for i in range(50):
with tf.GradientTape() as tape:
current_loss = loss(ytrain, model(Xtrain))
grad = tape.gradient(current_loss, model.trainable_variables)
optimizer.apply_gradients(zip(grad, model.trainable_variables))
print(current_loss)
def update(self):
#update v
st = time()
X = np.reshape(self.states_m, (1, -1))
next_v = self._predict_v(self.next_states_m) if not self.dones_m else 0
Y = np.reshape(self.rewards_m + self.discount * next_v, (1, 1))
self.v.fit(x=X, y=Y, batch_size=1, verbose=0)
#update pi
st = time()
this_v = self._predict_v(self.states_m)
delta = self.rewards_m + self.discount * next_v - this_v
self.I *= self.discount
if self.time == 1: self.I = 1 #I = gamma^t since epsiode started
if self.dones_m: self.time = 0 #Is incremented in observe
s = self.states_m
#Using the formulas found in Reinforcement Learning : An Introduction
if self.with_keras:
if self.bias:
s = np.concatenate([self.states_m, [1]])
with tf.GradientTape() as tape:
#print(self.pi_trainer.output, self.pi_trainer.trainable_variables)
ln_pi = self.pi_trainer(
[s.reshape((1, -1)),
np.reshape(self.actions_m, (1, 1))],
training=True)
ln_pi_grads = tape.gradient(ln_pi,
self.pi_trainer.trainable_variables)
grads = -self.I * delta * ln_pi_grads
optimizer = SGD(lr=self.learning_rate_v, clipvalue=10)
optimizer.apply_gradients(
zip(grads, self.pi_trainer.trainable_variables))
else:
mu, sig = self._get_mu_sig(s) #contains the "add bias"
if self.bias:
s = np.concatenate([self.states_m, [1]])
grad_ln_pi_mu = (self.actions_m - mu) / (sig**2) * s
grad_ln_pi_sig = ((self.actions_m - mu)**2 / (sig**2) - 1) * s
#gradient clipping
grad_ln_pi_mu = np.clip(grad_ln_pi_mu, -10, 10)
grad_ln_pi_sig = np.clip(grad_ln_pi_sig, -10, 10)
if self.bias:
self.theta_mu += np.reshape(
self.learning_rate_pi * self.I * delta * grad_ln_pi_mu,
(self.state_nb + 1))
self.theta_sig += np.reshape(
self.learning_rate_pi * self.I * delta * grad_ln_pi_sig,
(self.state_nb + 1))
else:
self.theta_mu += np.reshape(
self.learning_rate_pi * self.I * delta * grad_ln_pi_mu,
(self.state_nb))
self.theta_sig += np.reshape(
self.learning_rate_pi * self.I * delta * grad_ln_pi_sig,
(self.state_nb))
class DeepARLearner:
def __init__(
self,
ts_obj: TimeSeriesTrain,
output_dim: int = 1,
emb_dim: int = 128,
lstm_dim: int = 128,
dropout: float = 0.1,
optimizer: str = "adam",
lr: float = 0.001,
batch_size: int = 16,
# scheduler=None,
train_window: int = 20,
verbose: int = 0,
# mask_value: int=0,
random_seed: int = None,
hparams: typing.Dict[str, typing.Union[int, float]] = None,
):
""" initialize DeepAR model
Arguments:
ts_obj {TimeSeriesTrain} -- training dataset object
Keyword Arguments:
output_dim {int} -- dimension of output variables (default: {1})
emb_dim {int} -- dimension of categorical embeddings (default: {128})
lstm_dim {int} -- dimension of lstm cells (default: {128})
dropout {float} -- dropout (default: {0.1})
optimizer {str} -- options are "adam" and "sgd" (default: {"adam"})
lr {float} -- learning rate (default: {0.001})
batch_size {int} -- number of time series to sample in each batch (default: {16})
train_window {int} -- the length of time series sampled for training. consistent throughout (default: {20})
verbose {int} -- (default: {0})
random_seed {int} -- optional random seed to control randomness throughout learner (default: {None})
hparams {typing.Dict[str, typing.Union[int, float]]} --
dict of hyperparameters (keys) and hp domain (values) over which to hp search (default: {None})
"""
if random_seed is not None:
tf.random.set_seed(random_seed)
assert verbose == 0 or verbose == 1, "argument verbose must be 0 or 1"
if verbose == 1:
logger.setLevel(logging.INFO)
self._verbose = verbose
if hparams is None:
self._lr = lr
self._train_window = train_window
self._batch_size = batch_size
else:
self._lr = hparams["learning_rate"]
self._train_window = hparams["window_size"]
self._batch_size = hparams["batch_size"]
emb_dim = hparams["emb_dim"]
lstm_dim = hparams["lstm_dim"]
dropout = hparams["lstm_dropout"]
# Invariance - Window size must be <= max_series_length + negative observations to respect covariates
if self._train_window > ts_obj.max_age:
logger.info(
f"Training set with max observations {ts_obj.max_age} does not support train window size of "
+
f"{self._train_window}, resetting train window to {ts_obj.max_age}"
)
self._train_window = ts_obj.max_age
# self.scheduler = scheduler
self._ts_obj = ts_obj
# define model
self._model = self._create_model(
self._ts_obj.num_cats,
self._ts_obj.features,
output_dim=output_dim,
emb_dim=emb_dim,
lstm_dim=lstm_dim,
dropout=dropout,
count_data=self._ts_obj.count_data,
)
# define optimizer
if optimizer == "adam":
self._optimizer = Adam(learning_rate=self._lr)
elif optimizer == "sgd":
self._optimizer = SGD(lr=self._lr, momentum=0.1, nesterov=True)
else:
raise ValueError("Optimizer must be one of `adam` and `sgd`")
# define loss function
if self._ts_obj.count_data:
self._loss_fn = NegativeBinomialLogLikelihood(
mask_value=self._ts_obj.mask_value)
else:
self._loss_fn = GaussianLogLikelihood(
mask_value=self._ts_obj.mask_value)
def save_weights(self, filepath: str):
""" saves model's current weights to filepath
Arguments:
filepath {str} -- filepath
"""
self._model.save_weights(filepath)
def load_weights(self, filepath: str):
""" loads model's current weights from filepath
Arguments:
filepath {str} -- filepath
"""
self._model.load_weights(filepath)
def _create_model(
self,
num_cats: int,
num_features: int,
output_dim: int = 1,
emb_dim: int = 128,
lstm_dim: int = 128,
batch_size: int = 16,
dropout: float = 0.1,
count_data: bool = False,
) -> Model:
"""
util function
creates model architecture (Keras Sequential) with arguments specified in constructor
"""
cont_inputs = Input(shape=(self._train_window, num_features),
batch_size=self._batch_size)
cat_inputs = Input(shape=(self._train_window, ),
batch_size=self._batch_size)
embedding = Embedding(num_cats, emb_dim)(cat_inputs)
concatenate = Concatenate()([cont_inputs, embedding])
lstm_out = LSTMResetStateful(
lstm_dim,
return_sequences=True,
stateful=True,
dropout=dropout,
recurrent_dropout=dropout,
unit_forget_bias=True,
name="lstm",
)(concatenate)
mu = Dense(
output_dim,
kernel_initializer="glorot_normal",
bias_initializer="glorot_normal",
name="mu",
)(lstm_out)
sigma = Dense(
output_dim,
kernel_initializer="glorot_normal",
bias_initializer="glorot_normal",
name="sigma",
)(lstm_out)
model = Model(inputs=[cont_inputs, cat_inputs], outputs=[mu, sigma])
return model
def _training_loop(
self,
filepath: str,
train_gen: train_ts_generator, # can name of function be type?
val_gen: train_ts_generator,
epochs: int = 100,
steps_per_epoch: int = 50,
early_stopping: int = True,
stopping_patience: int = 5,
stopping_delta: int = 1,
) -> typing.Tuple[tf.Tensor, int]:
"""
util function
iterates over batches, updates gradients, records metrics, writes to tb, checkpoints, early stopping
"""
# set up metrics to track during training
batch_loss_avg = Mean()
epoch_loss_avg = Mean()
eval_loss_avg = Mean()
eval_mae = MeanAbsoluteError()
eval_rmse = RootMeanSquaredError()
# set up early stopping callback
early_stopping_cb = EarlyStopping(patience=stopping_patience,
active=early_stopping,
delta=stopping_delta)
# setup table for unscaling
self._lookup_table = build_tf_lookup(self._ts_obj.target_means)
# Iterate over epochs.
best_metric = math.inf
for epoch in range(epochs):
logger.info(f"Start of epoch {epoch}")
start_time = time.time()
for batch, (x_batch_train, cat_labels,
y_batch_train) in enumerate(train_gen):
# compute loss
with tf.GradientTape(persistent=True) as tape:
mu, scale = self._model(x_batch_train, training=True)
# softplus parameters
scale = softplus(scale)
if self._ts_obj.count_data:
mu = softplus(mu)
mu, scale = unscale(mu, scale, cat_labels,
self._lookup_table)
loss_value = self._loss_fn(y_batch_train, (mu, scale))
# sgd
if self._tb:
tf.summary.scalar("train_loss", loss_value,
epoch * steps_per_epoch + batch)
batch_loss_avg(loss_value)
epoch_loss_avg(loss_value)
grads = tape.gradient(loss_value,
self._model.trainable_weights)
self._optimizer.apply_gradients(
zip(grads, self._model.trainable_weights))
# Log 5x per epoch.
if batch % (steps_per_epoch // 5) == 0 and batch != 0:
logger.info(
f"Epoch {epoch}: Avg train loss over last {(steps_per_epoch // 5)} steps: {batch_loss_avg.result()}"
)
batch_loss_avg.reset_states()
# Run each epoch batches times
epoch_loss_avg_result = epoch_loss_avg.result()
if batch == steps_per_epoch:
logger.info(
f"Epoch {epoch} took {round(time.time() - start_time, 0)}s : Avg train loss: {epoch_loss_avg_result}"
)
break
# validation
if val_gen is not None:
logger.info(f"End of epoch {epoch}, validating...")
start_time = time.time()
for batch, (x_batch_val, cat_labels,
y_batch_val) in enumerate(val_gen):
# compute loss, doesn't need to be persistent bc not updating weights
with tf.GradientTape() as tape:
# treat as training -> reset lstm states inbetween each batch
mu, scale = self._model(x_batch_val, training=True)
# softplus parameters
mu, scale = self._softplus(mu, scale)
# unscale parameters
mu, scale = unscale(mu, scale, cat_labels,
self._lookup_table)
# calculate loss
loss_value = self._loss_fn(y_batch_val, (mu, scale))
# log validation metrics (avg loss, avg MAE, avg RMSE)
eval_mae(y_batch_val, mu)
eval_rmse(y_batch_val, mu)
eval_loss_avg(loss_value)
if batch == steps_per_epoch:
break
# logging
eval_mae_result = eval_mae.result()
logger.info(
f"Validation took {round(time.time() - start_time, 0)}s")
logger.info(
f"Epoch {epoch}: Val loss on {steps_per_epoch} steps: {eval_loss_avg.result()}"
)
logger.info(
f"Epoch {epoch}: Val MAE: {eval_mae_result}, RMSE: {eval_rmse.result()}"
)
if self._tb:
tf.summary.scalar("val_loss", eval_loss_avg.result(),
epoch)
tf.summary.scalar("val_mae", eval_mae_result, epoch)
tf.summary.scalar("val_rmse", eval_rmse.result(), epoch)
new_metric = eval_mae_result
# early stopping
if early_stopping_cb(eval_mae_result):
break
# reset metric states
eval_loss_avg.reset_states()
eval_mae.reset_states()
eval_rmse.reset_states()
else:
if early_stopping_cb(epoch_loss_avg_result):
break
new_metric = epoch_loss_avg_result
# update best_metric and save new checkpoint if improvement
if new_metric < best_metric:
best_metric = new_metric
if filepath is not None:
self._checkpointer.save(file_prefix=filepath)
else:
self.save_weights("model_best_weights.h5")
# reset epoch loss metric
epoch_loss_avg.reset_states()
# load in best weights before returning if not using checkpointer
if filepath is None:
self.load_weights("model_best_weights.h5")
os.remove("model_best_weights.h5")
return best_metric, epoch + 1
def fit(
self,
checkpoint_dir: str = None,
validation: bool = True,
steps_per_epoch: int = 50,
epochs: int = 100,
early_stopping: bool = True,
stopping_patience: int = 5,
stopping_delta: int = 1,
tensorboard: bool = True,
) -> typing.Tuple[tf.Tensor, int]:
""" fits DeepAR model for steps_per_epoch * epochs iterations
Keyword Arguments:
checkpoint_dir {str} -- directory to save checkpoint and tensorboard files (default: {None})
validation {bool} -- whether to perform validation. If True will automatically try
to use validation data sequestered in construction of time series object(default: {True})
steps_per_epoch {int} -- number of steps to process each epoch (default: {50})
epochs {int} -- number of epochs (default: {100})
early_stopping {bool} -- whether to include early stopping callback (default: {True})
stopping_patience {int} -- early stopping callback patience, measured in epochs (default: {5})
stopping_delta {int} -- early stopping delta, the comparison to determine stopping will be
previous metric vs. new metric +- stopping_delta (default: {1})
tensorboard {bool} -- whether to write output to tensorboard logs (default: {True})
Returns:
Tuple(float, int) -- 1) final_metric best (train loss or eval MAE) after fitting
2) number of iterations completed (might have been impacted by stopping criterion)
"""
self._epochs = epochs
# try to load previously saved checkpoint from filepath
self._checkpointer = tf.train.Checkpoint(optimizer=self._optimizer,
model=self._model)
if checkpoint_dir is not None:
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
filepath = os.path.join(checkpoint_dir, "{epoch:04d}.ckpt")
latest_ckpt = tf.train.latest_checkpoint(checkpoint_dir)
if latest_ckpt:
self._checkpointer.restore(latest_ckpt)
elif tensorboard:
# create tensorboard log files in default location
checkpoint_dir = "./tb/"
tb_writer = tf.summary.create_file_writer(checkpoint_dir)
tb_writer.set_as_default()
else:
filepath = None
self._tb = tensorboard
# train generator
train_gen = train_ts_generator(
self._model,
self._ts_obj,
self._batch_size,
self._train_window,
verbose=self._verbose,
)
# validation generator
if validation:
val_gen = train_ts_generator(
self._model,
self._ts_obj,
self._batch_size,
self._train_window,
verbose=self._verbose,
val_set=True,
)
else:
val_gen = None
# Iterate over epochs.
return self._training_loop(
filepath,
train_gen,
val_gen,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
early_stopping=early_stopping,
stopping_patience=stopping_patience,
stopping_delta=stopping_delta,
)
def _add_prev_target(
self, x_test: tf.Variable,
prev_target: tf.Tensor) -> typing.Tuple[tf.Variable, tf.Tensor]:
""" private util function that replaces the previous target column in input data with
dynamically generated last target
"""
x_test_new = x_test[0][:, :1, -1:].assign(prev_target)
return [x_test_new, x_test[1]]
def _softplus(
self,
mu: tf.Tensor,
scale: tf.Tensor,
) -> typing.Tuple[tf.Tensor, tf.Tensor]:
"""
private util function that applies softplus transformation to various parameters
depending on type of data
"""
scale = softplus(scale)
if self._ts_obj.count_data:
mu = softplus(mu)
return mu, scale
def _squeeze(
self,
mu: tf.Tensor,
scale: tf.Tensor,
squeeze_dims: typing.List[int] = [2]
) -> typing.Tuple[tf.Tensor, tf.Tensor]:
"""
private util function that squeezes predictions along certain dimensions depending on whether
we are predicting in-sample or out-of-sample
"""
return tf.squeeze(mu, squeeze_dims), tf.squeeze(scale, squeeze_dims)
def _negative_binomial(self,
mu: tf.Tensor,
scale: tf.Tensor,
samples: int = 1) -> np.ndarray:
"""
private util function that draws n samples from a negative binomial distribution parameterized
by mu and scale parameters
based on implementation from GluonTS library:
https://gluon-ts.mxnet.io/_modules/gluonts/distribution/neg_binomial.html#NegativeBinomial
"""
tol = 1e-5
r = 1.0 / scale
theta = scale * mu
r = tf.math.minimum(tf.math.maximum(tol, r), 1e10)
theta = tf.math.minimum(tf.math.maximum(tol, theta), 1e10)
p = 1 / (theta + 1)
return np.random.negative_binomial(r, p, samples)
def _draw_samples(
self,
mu_tensor: tf.Tensor,
scale_tensor: tf.Tensor,
point_estimate: int = False,
samples: int = 1,
) -> typing.List[typing.List[np.ndarray]]:
"""
private util function that draws samples from appropriate distribution
"""
# shape : [# test groups, samples]
if point_estimate:
return [np.repeat(mu, samples) for mu in mu_tensor]
elif self._ts_obj.count_data:
return [
list(self._negative_binomial(mu, scale, samples))
for mu, scale in zip(mu_tensor, scale_tensor)
]
else:
return [
list(np.random.normal(mu, scale, samples))
for mu, scale in zip(mu_tensor, scale_tensor)
]
def _reset_lstm_states(self):
""" private util function to reset lstm states, dropout mask, and recurrent dropout mask
"""
def predict(
self,
test_ts_obj: TimeSeriesTest,
point_estimate: bool = True,
horizon: int = None,
samples: int = 100,
include_all_training: bool = False,
return_in_sample_predictions: bool = True,
) -> np.ndarray:
""" predict horizon steps into the future
Arguments:
test_ts_obj {TimeSeriesTest} -- time series object for prediction
Keyword Arguments:
point_estimate {bool} -- if True, always sample mean of distributions,
otherwise sample from (mean, scale) parameters (default: {True})
horizon {int} -- optional, can specify prediction horizon into future (default: {None})
samples {int} -- how many samples to draw to calculate confidence intervals (default: {100})
include_all_training {bool} -- whether to start calculating hidden states from beginning
of training data, alternative is from t_0 - train_window (default: {False})
return_in_sample_predictions {bool} --
whether to return in sample predictions as well as future predictions (default: {True})
Returns:
np.ndarray -- predictions, shape is (# unique test groups, horizon, # samples)
"""
assert samples > 0, "The number of samples to draw must be positive"
# test generator
test_gen = test_ts_generator(
self._model,
test_ts_obj,
self._batch_size,
self._train_window,
include_all_training=include_all_training,
verbose=self._verbose,
)
# reset lstm states before prediction
self._model.get_layer("lstm").reset_states()
# make sure horizon is legitimate value
if horizon is None or horizon > test_ts_obj.horizon:
horizon = test_ts_obj.horizon
# forward
test_samples = [[] for _ in range(len(test_ts_obj.test_groups))]
start_time = time.time()
prev_iteration_index = 0
for batch_idx, batch in enumerate(test_gen):
x_test, scale_keys, horizon_idx, iteration_index = batch
if iteration_index is None:
break
if horizon_idx == horizon:
test_ts_obj.batch_idx = 0
test_ts_obj.iterations += 1
continue
# reset lstm states for new sequence of predictions through time
if iteration_index != prev_iteration_index:
self._model.get_layer("lstm").reset_states()
# don't need to replace for first test batch bc have tgt from last training example
if horizon_idx > 1:
# add one sample from previous predictions to test batches
# all dim 0, first row of dim 1, last col of dim 3
x_test = self._add_prev_target(x_test, mu[:, :1, :])
# make predictions
mu, scale = self._model(x_test)
mu, scale = self._softplus(mu, scale)
# unscale
scaled_mu, scaled_scale = unscale(
mu[:scale_keys.shape[0]],
scale[:scale_keys.shape[0]],
scale_keys,
self._lookup_table,
)
# in-sample predictions (ancestral sampling)
if horizon_idx <= 0:
if horizon_idx % 5 == 0:
logger.info(
f"Making in-sample predictions with ancestral sampling. {-horizon_idx} batches remaining"
)
# squeeze 2nd dim - output_dim
scaled_mu, scaled_scale = self._squeeze(
scaled_mu, scaled_scale)
for mu, scale, sample_list in zip(
scaled_mu,
scaled_scale,
test_samples[iteration_index *
self._batch_size:(iteration_index + 1) *
self._batch_size],
):
draws = self._draw_samples(mu,
scale,
point_estimate=point_estimate,
samples=samples)
sample_list.extend(draws)
# draw samples from learned distributions for test samples
else:
if horizon_idx % 5 == 0:
logger.info(
f"Making future predictions. {horizon-horizon_idx} batches remaining"
)
# get first column predictions (squeeze 1st dim - horizon)
scaled_mu, scaled_scale = scaled_mu[:, :
1, :], scaled_scale[:, :
1, :]
# slice at number of unique ts and squeeze 1st dim - horizon, 2nd dim - output_dim)
squeezed_mu, squeezed_scale = self._squeeze(
scaled_mu,
scaled_scale,
squeeze_dims=[1, 2],
)
# concatenate
draws = self._draw_samples(
squeezed_mu,
squeezed_scale,
point_estimate=point_estimate,
samples=samples,
)
for draw_list, sample_list in zip(
draws,
test_samples[iteration_index *
self._batch_size:(iteration_index + 1) *
self._batch_size],
):
sample_list.append(draw_list)
# reset batch idx and iterations_index so we can call predict() multiple times
test_ts_obj.batch_idx = 0
test_ts_obj.iterations = 0
test_ts_obj.batch_test_data_prepared = False
# filter test_samples depending on return_in_sample_predictions param
if return_in_sample_predictions:
pred_samples = np.array(test_samples)[:, -(self._ts_obj.max_age +
horizon):, :]
else:
pred_samples = np.array(test_samples)[:, -horizon:, :]
logger.info(
f"Inference ({samples} sample(s), {horizon} timesteps) took {round(time.time() - start_time, 0)}s"
)
# Shape [# test_groups, horizon, samples]
# TODO [# test_groups, horizon, samples, output_dim] not supported yet
return pred_samples
class CycleGAN(object):
def __init__(self, args):
self.batch_size = args.batch_size
self.time_step = args.time_step # number of time steps
self.pitch_range = args.pitch_range # number of pitches
self.input_c_dim = args.input_nc # number of input image channels
self.output_c_dim = args.output_nc # number of output image channels
self.lr = args.lr
self.L1_lambda = args.L1_lambda
self.gamma = args.gamma
self.sigma_d = args.sigma_d
self.dataset_A_dir = args.dataset_A_dir
self.dataset_B_dir = args.dataset_B_dir
self.d_loss_path = args.d_loss_path
self.g_loss_path = args.g_loss_path
self.cycle_loss_path = args.cycle_loss_path
self.sample_dir = args.sample_dir
self.model = args.model
self.discriminator = build_discriminator
self.generator = build_generator
self.criterionGAN = mae_criterion
OPTIONS = namedtuple(
"OPTIONS",
"batch_size "
"time_step "
"input_nc "
"output_nc "
"pitch_range "
"gf_dim "
"df_dim "
"is_training",
)
self.options = OPTIONS._make(
(
args.batch_size,
args.time_step,
args.pitch_range,
args.input_nc,
args.output_nc,
args.ngf,
args.ndf,
args.phase == "train",
)
)
self.now_datetime = get_now_datetime()
self.pool = ImagePool(args.max_size)
self._build_model(args)
print("Initialized model.")
def _build_model(self, args):
# Generator
self.generator_A2B = self.generator(self.options, name="Generator_A2B")
self.generator_B2A = self.generator(self.options, name="Generator_B2A")
# Discriminator
self.discriminator_A = self.discriminator(self.options, name="Discriminator_A")
self.discriminator_B = self.discriminator(self.options, name="Discriminator_B")
if self.model != "base":
self.discriminator_A_all = self.discriminator(
self.options, name="Discriminator_A_all"
)
self.discriminator_B_all = self.discriminator(
self.options, name="Discriminator_B_all"
)
# Discriminator and Generator Optimizer
if args.optimizer == "adam":
self.DA_optimizer = Adam(self.lr, beta_1=args.beta1)
self.DB_optimizer = Adam(self.lr, beta_1=args.beta1)
self.GA2B_optimizer = Adam(self.lr, beta_1=args.beta1)
self.GB2A_optimizer = Adam(self.lr, beta_1=args.beta1)
elif args.optimizer == "rmsprop":
self.DA_optimizer = RMSprop(self.lr, momentum=args.beta1)
self.DB_optimizer = RMSprop(self.lr, momentum=args.beta1)
self.GA2B_optimizer = RMSprop(self.lr, momentum=args.beta1)
self.GB2A_optimizer = RMSprop(self.lr, momentum=args.beta1)
elif args.optimizer == "sgd":
self.DA_optimizer = SGD(self.lr, momentum=args.beta1)
self.DB_optimizer = SGD(self.lr, momentum=args.beta1)
self.GA2B_optimizer = SGD(self.lr, momentum=args.beta1)
self.GB2A_optimizer = SGD(self.lr, momentum=args.beta1)
# Checkpoints
model_name = "cyclegan.model"
model_dir = args.model_dir
self.checkpoint_dir = os.path.join(args.checkpoint_dir, model_dir, model_name)
if not os.path.exists(self.checkpoint_dir):
os.makedirs(self.checkpoint_dir)
self.checkpoint = tf.train.Checkpoint(
generator_A2B_optimizer=self.GA2B_optimizer,
generator_B2A_optimizer=self.GB2A_optimizer,
discriminator_A_optimizer=self.DA_optimizer,
discriminator_B_optimizer=self.DB_optimizer,
generator_A2B=self.generator_A2B,
generator_B2A=self.generator_B2A,
discriminator_A=self.discriminator_A,
discriminator_B=self.discriminator_B,
)
self.checkpoint_manager = tf.train.CheckpointManager(
self.checkpoint, self.checkpoint_dir, max_to_keep=5
)
print("Built model.")
def train(self, args):
# Data from domain A and B, and mixed dataset for partial and full models.
dataA = glob("./{}/train/*.*".format(self.dataset_A_dir))
dataB = glob("./{}/train/*.*".format(self.dataset_B_dir))
if args.continue_train:
if self.checkpoint.restore(self.checkpoint_manager.latest_checkpoint):
print(" [*] Load checkpoint succeeded!")
else:
print(" [!] Load checkpoint failed.")
counter = 1
start_time = time.time()
d_loss_list = []
g_loss_list = []
cycle_loss_list = []
print("Training start...")
for epoch in range(args.epoch):
# Shuffle training data
np.random.shuffle(dataA)
np.random.shuffle(dataB)
# Get the proper number of batches
batch_idxs = min(len(dataA), len(dataB)) // self.batch_size
# learning rate starts to decay when reaching the threshold
self.lr = (
self.lr
if epoch < args.epoch_step
else self.lr * (args.epoch - epoch) / (args.epoch - args.epoch_step)
)
for idx in range(batch_idxs):
# To feed real_data
batch_files = list(
zip(
dataA[idx * self.batch_size : (idx + 1) * self.batch_size],
dataB[idx * self.batch_size : (idx + 1) * self.batch_size],
)
)
batch_samples = [
load_npy_data(batch_file) for batch_file in batch_files
]
batch_samples = np.array(batch_samples).astype(
np.float32
) # batch_size * 64 * 84 * 2
real_A, real_B = batch_samples[:, :, :, 0], batch_samples[:, :, :, 1]
real_A = tf.expand_dims(real_A, -1)
real_B = tf.expand_dims(real_B, -1)
# generate gaussian noise for robustness improvement
gaussian_noise = np.abs(
np.random.normal(
0,
self.sigma_d,
[
self.batch_size,
self.time_step,
self.pitch_range,
self.input_c_dim,
],
)
).astype(np.float32)
with tf.GradientTape(persistent=True) as gen_tape, tf.GradientTape(
persistent=True
) as disc_tape:
fake_B = self.generator_A2B(real_A, training=True)
cycle_A = self.generator_B2A(fake_B, training=True)
fake_A = self.generator_B2A(real_B, training=True)
cycle_B = self.generator_A2B(fake_A, training=True)
[fake_A_sample, fake_B_sample] = self.pool([fake_A, fake_B])
DA_real = self.discriminator_A(
real_A + gaussian_noise, training=True
)
DB_real = self.discriminator_B(
real_B + gaussian_noise, training=True
)
DA_fake = self.discriminator_A(
fake_A + gaussian_noise, training=True
)
DB_fake = self.discriminator_B(
fake_B + gaussian_noise, training=True
)
DA_fake_sample = self.discriminator_A(
fake_A_sample + gaussian_noise, training=True
)
DB_fake_sample = self.discriminator_B(
fake_B_sample + gaussian_noise, training=True
)
# Generator loss
cycle_loss = self.L1_lambda * (
abs_criterion(real_A, cycle_A) + abs_criterion(real_B, cycle_B)
)
g_A2B_loss = (
self.criterionGAN(DB_fake, tf.ones_like(DB_fake)) + cycle_loss
)
g_B2A_loss = (
self.criterionGAN(DA_fake, tf.ones_like(DA_fake)) + cycle_loss
)
g_loss = g_A2B_loss + g_B2A_loss - cycle_loss
# Discriminator loss
d_A_loss_real = self.criterionGAN(DA_real, tf.ones_like(DA_real))
d_A_loss_fake = self.criterionGAN(
DA_fake_sample, tf.zeros_like(DA_fake_sample)
)
d_A_loss = (d_A_loss_real + d_A_loss_fake) / 2
d_B_loss_real = self.criterionGAN(DB_real, tf.ones_like(DB_real))
d_B_loss_fake = self.criterionGAN(
DB_fake_sample, tf.zeros_like(DB_fake_sample)
)
d_B_loss = (d_B_loss_real + d_B_loss_fake) / 2
d_loss = d_A_loss + d_B_loss
# Calculate the gradients for generator and discriminator
generator_A2B_gradients = gen_tape.gradient(
target=g_A2B_loss, sources=self.generator_A2B.trainable_variables,
)
generator_B2A_gradients = gen_tape.gradient(
target=g_B2A_loss, sources=self.generator_B2A.trainable_variables,
)
discriminator_A_gradients = disc_tape.gradient(
target=d_A_loss, sources=self.discriminator_A.trainable_variables,
)
discriminator_B_gradients = disc_tape.gradient(
target=d_B_loss, sources=self.discriminator_B.trainable_variables,
)
# Apply the gradients to the optimizer
self.GA2B_optimizer.apply_gradients(
zip(
generator_A2B_gradients, self.generator_A2B.trainable_variables,
)
)
self.GB2A_optimizer.apply_gradients(
zip(
generator_B2A_gradients, self.generator_B2A.trainable_variables,
)
)
self.DA_optimizer.apply_gradients(
zip(
discriminator_A_gradients,
self.discriminator_A.trainable_variables,
)
)
self.DB_optimizer.apply_gradients(
zip(
discriminator_B_gradients,
self.discriminator_B.trainable_variables,
)
)
print(
"================================================================="
)
print(
(
"Epoch: [%2d] [%4d/%4d] time: %4.4f D_loss: %6.2f, G_loss: %6.2f, cycle_loss: %6.2f"
% (
epoch,
idx,
batch_idxs,
time.time() - start_time,
d_loss,
g_loss,
cycle_loss,
)
)
)
# ADDED
d_loss_list.append(d_loss)
g_loss_list.append(g_loss)
cycle_loss_list.append(cycle_loss)
counter += 1
# generate samples during training to track the learning process
if np.mod(counter, args.print_freq) == 1:
sample_dir = os.path.join(
self.sample_dir,
"{}2{}_{}_{}_{}".format(
self.dataset_A_dir,
self.dataset_B_dir,
self.now_datetime,
self.model,
self.sigma_d,
),
)
if not os.path.exists(sample_dir):
os.makedirs(sample_dir)
# to binary, 0 denotes note off, 1 denotes note on
samples = [
to_binary(real_A, 0.5),
to_binary(fake_B, 0.5),
to_binary(cycle_A, 0.5),
to_binary(real_B, 0.5),
to_binary(fake_A, 0.5),
to_binary(cycle_B, 0.5),
]
self.sample_model(
samples=samples, sample_dir=sample_dir, epoch=epoch, idx=idx
)
if np.mod(counter, args.save_freq) == 1:
self.checkpoint_manager.save(counter)
pickle_loss_list(d_loss_list, self.d_loss_path)
pickle_loss_list(g_loss_list, self.g_loss_path)
pickle_loss_list(cycle_loss_list, self.cycle_loss_path)
def test(self, args):
"""
--test_dir
--checkpoint_dir
--which_direction
--model_dir
"""
if args.which_direction == "AtoB":
sample_files = glob("./{}/test/*.*".format(self.dataset_A_dir))
elif args.which_direction == "BtoA":
sample_files = glob("./{}/test/*.*".format(self.dataset_B_dir))
else:
raise Exception("--which_direction must be AtoB or BtoA")
sample_files.sort(
key=lambda x: int(os.path.splitext(os.path.basename(x))[0].split("_")[-1])
)
checkpoint_filepath = tf.train.latest_checkpoint(args.checkpoint_dir)
if checkpoint_filepath:
status = self.checkpoint.restore(checkpoint_filepath)
print(" [*] Load checkpoint succeeded!", checkpoint_filepath)
else:
print(" [!] Load checkpoint failed...")
test_dir_mid = os.path.join(
args.test_dir, "{}/{}/mid".format(args.model_dir, args.which_direction),
)
if not os.path.exists(test_dir_mid):
os.makedirs(test_dir_mid)
test_dir_npy = os.path.join(
args.test_dir, "{}/{}/npy".format(args.model_dir, args.which_direction),
)
if not os.path.exists(test_dir_npy):
os.makedirs(test_dir_npy)
for idx in range(len(sample_files)):
print("Processing midi: ", sample_files[idx])
sample_npy = np.load(sample_files[idx]) * 1.0
# save midis
origin = sample_npy.reshape(1, sample_npy.shape[0], sample_npy.shape[1], 1)
midi_path_origin = os.path.join(
test_dir_mid, "{}_origin.mid".format(idx + 1)
)
midi_path_transfer = os.path.join(
test_dir_mid, "{}_transfer.mid".format(idx + 1)
)
midi_path_cycle = os.path.join(test_dir_mid, "{}_cycle.mid".format(idx + 1))
if args.which_direction == "AtoB":
transfer = self.generator_A2B(origin, training=False)
cycle = self.generator_B2A(transfer, training=False)
else:
transfer = self.generator_B2A(origin, training=False)
cycle = self.generator_A2B(transfer, training=False)
save_midis(origin, midi_path_origin)
save_midis(transfer, midi_path_transfer)
save_midis(cycle, midi_path_cycle)
# save npy files
npy_path_origin = os.path.join(test_dir_npy, "origin")
npy_path_transfer = os.path.join(test_dir_npy, "transfer")
npy_path_cycle = os.path.join(test_dir_npy, "cycle")
if not os.path.exists(npy_path_origin):
os.makedirs(npy_path_origin)
if not os.path.exists(npy_path_transfer):
os.makedirs(npy_path_transfer)
if not os.path.exists(npy_path_cycle):
os.makedirs(npy_path_cycle)
np.save(
os.path.join(npy_path_origin, "{}_origin.npy".format(idx + 1)), origin
)
np.save(
os.path.join(npy_path_transfer, "{}_transfer.npy".format(idx + 1)),
transfer,
)
np.save(os.path.join(npy_path_cycle, "{}_cycle.npy".format(idx + 1)), cycle)
def sample_model(self, samples, sample_dir, epoch, idx):
print("Generating samples during learning...")
if not os.path.exists(os.path.join(sample_dir, "B2A")):
os.makedirs(os.path.join(sample_dir, "B2A"))
if not os.path.exists(os.path.join(sample_dir, "A2B")):
os.makedirs(os.path.join(sample_dir, "A2B"))
save_midis(
samples[0],
"./{}/A2B/{:02d}_{:04d}_origin.mid".format(sample_dir, epoch, idx),
)
save_midis(
samples[1],
"./{}/A2B/{:02d}_{:04d}_transfer.mid".format(sample_dir, epoch, idx),
)
save_midis(
samples[2],
"./{}/A2B/{:02d}_{:04d}_cycle.mid".format(sample_dir, epoch, idx),
)
save_midis(
samples[3],
"./{}/B2A/{:02d}_{:04d}_origin.mid".format(sample_dir, epoch, idx),
)
save_midis(
samples[4],
"./{}/B2A/{:02d}_{:04d}_transfer.mid".format(sample_dir, epoch, idx),
)
save_midis(
samples[5],
"./{}/B2A/{:02d}_{:04d}_cycle.mid".format(sample_dir, epoch, idx),
)
class Train:
def __init__(self,
seqs,
adj,
nodes_features,
epochs,
key,
use_gcn,
batch_size,
use_gru=True):
self.epochs = epochs
self.seqs = seqs.astype('float32')
self.seqs_noised = seqs.copy().astype('float32')
self.max_s = seqs[key].max()
self.seqs_noised[key] = np.random.normal(
self.max_s / 2.0, self.max_s / 10.0,
size=(seqs.shape[0])).astype('float32')
self.key = key
self.gen_optimizer = SGD(lr, adam_beta_1)
self.desc_optimizer = SGD(lr, adam_beta_1)
self.adj = normalized_laplacian(adj.astype('float32'))
self.adj_expanded = tf.expand_dims(normalized_laplacian(
adj.astype('float32')),
axis=0)
self.nodes_features = nodes_features.astype('float32')
self.nodes_f_expanded = tf.expand_dims(
nodes_features.astype('float32'), axis=0)
self.generator = model.make_generator('generator', batch_size,
self.adj, self.nodes_features,
use_gcn, use_gru)
self.discriminator = model.make_discriminator('discriminator',
batch_size, self.adj,
self.nodes_features,
use_gcn, use_gru)
self.d_loss_fn, self.g_loss_fn = losses.get_wasserstein_losses_fn()
self.wsst_hist = []
self.cos_simi = []
self.var_hist = []
self.rmse_hist = []
self.mae_hist = []
self.r2_hist = []
self.g_loss_hist = []
self.d_loss_hist = []
def __call__(self,
epochs=None,
save_path='generated/',
batch_size=96,
monitor=True):
if not os.path.exists(save_path):
os.makedirs(save_path)
if epochs is None:
epochs = self.epochs
time_len = self.seqs.shape[0]
total_batch = int(time_len / batch_size)
time_consumed_total = 0.
final_epoch = epochs
stable = 0
for epoch in range(1, epochs + 1):
start = time.time()
total_gen_loss = 0
total_disc_loss = 0
for week in range(0, total_batch):
current_seqs = self.seqs[week * batch_size:week * batch_size +
batch_size]
seqs_noised = self.seqs_noised[week *
batch_size:week * batch_size +
batch_size]
max_s = current_seqs[self.key].max()
seqs_noised[self.key] = np.random.normal(
max_s / 2.0, max_s / 10.0,
size=(current_seqs.shape[0])).astype('float32')
gen_loss, disc_loss = self.train_step(current_seqs,
seqs_noised, batch_size)
total_gen_loss += gen_loss
total_disc_loss += disc_loss
time_consumed = time.time() - start
time_consumed_total += time_consumed
time_consumed_agv = time_consumed_total / epoch
epochs_last = epochs - epoch
estimate_time_last = epochs_last * time_consumed_agv
if epoch % 1 == 0:
metrics_ = self.evaluate(save_path,
batch_size,
epoch,
plot_compare=False)
self.wsst_hist.append(metrics_.get('WSSTD'))
self.cos_simi.append(metrics_.get('COS_SIMI'))
self.rmse_hist.append(metrics_.get('RMSE'))
self.mae_hist.append(metrics_.get('MAE'))
self.r2_hist.append(metrics_.get('R^2'))
self.var_hist.append(metrics_.get('Var'))
self.g_loss_hist.append(
round(float(total_gen_loss / total_batch), 3))
self.d_loss_hist.append(
round(float(total_disc_loss / total_batch), 3))
if epoch % evaluate_interval == 0:
metrics_ = self.evaluate(save_path, batch_size, epoch)
metrics_['gen_loss'] = round(
float(total_gen_loss / total_batch), 3)
metrics_['disc_loss'] = round(
float(total_disc_loss / total_batch), 3)
print('epoch {}/{}({})-{}, to finish： {}'.format(
epoch, epochs, round(time_consumed_total, 2),
json.dumps(metrics_, ensure_ascii=False),
round(estimate_time_last, 2)))
if not metrics_.keys().__contains__('RMSE'):
metrics_['crash'] = True
break
if monitor and metrics_['WSSTD'] < 0.02 and metrics_[
'RMSE'] < 0.09:
stable += 1
if stable > 2:
final_epoch = epoch
break
else:
stable = 0
matrix_hist = pd.DataFrame()
matrix_hist['wsst_hist'] = self.wsst_hist
matrix_hist['cos_simi'] = self.cos_simi
matrix_hist['rmse_hist'] = self.rmse_hist
matrix_hist['mae_hist'] = self.mae_hist
matrix_hist['r2_hist'] = self.r2_hist
matrix_hist['var_hist'] = self.var_hist
matrix_hist['g_loss_hist'] = self.g_loss_hist
matrix_hist['d_loss_hist'] = self.d_loss_hist
matrix_hist.to_csv(save_path + '/matrix_hist.csv',
encoding='utf_8_sig',
index=False)
self.save_model(save_path, time_consumed_total)
return time_consumed_total, final_epoch
@tf.function
def train_step(self, seqs, seqs_noised, batch_size):
with tf.GradientTape(persistent=True) as tape:
real_output = self.call_model(self.discriminator, seqs)
generated = self.call_model(self.generator, seqs_noised)
left = tf.slice(seqs, [0, 0], [batch_size, self.key])
right = tf.slice(seqs, [0, self.key + 1], [batch_size, -1])
combined = tf.concat([left, generated[0], right], 1)
generated_output = self.call_model(self.discriminator, combined)
loss_g = self.g_loss_fn(generated_output)
loss_d = self.d_loss_fn(real_output, generated_output)
loss_dif = abs(loss_d - loss_g)
if loss_dif <= 10:
self.apply_grad(self.generator, tape, loss_g)
self.apply_grad(self.discriminator, tape, loss_d)
else:
if loss_d > loss_g:
self.apply_grad(self.discriminator, tape, loss_d)
else:
self.apply_grad(self.generator, tape, loss_g)
return loss_g, loss_d
def apply_grad(self, d_or_g, tape, loss):
grad = tape.gradient(loss, d_or_g.trainable_variables)
self.desc_optimizer.apply_gradients(
zip(grad, d_or_g.trainable_variables))
def call_model(self, model_, seqs):
return model_(inputs=[
tf.expand_dims(seqs, axis=0), self.nodes_f_expanded,
self.adj_expanded
])
def generate(self, real_seqs):
seqs_replace = real_seqs.copy()
max_s = seqs_replace[self.key].max()
seqs_replace[self.key] = np.random.normal(
max_s / 2.0, max_s / 10.0,
size=(seqs_replace.shape[0])).astype('float32')
gen_data = tf.squeeze(self.generator(tf.expand_dims(seqs_replace,
axis=0),
training=False),
axis=0)
return pd.DataFrame(gen_data.numpy())
def get_compare(self, start_day, batch_size):
real_seqs = self.seqs[start_day:start_day + batch_size]
self.seqs_noised[self.key] = np.random.normal(
self.max_s / 2.0, self.max_s / 10.0,
size=(self.seqs.shape[0])).astype('float32')
noise_seq = self.seqs_noised[start_day:start_day + batch_size]
generated_seqs = self.call_model(self.generator, noise_seq).numpy()[0]
return real_seqs[self.key].values, generated_seqs
def evaluate(self,
save_path,
batch_size,
name=None,
restore_model=False,
plot_compare=True):
if restore_model:
self.load_model(save_path)
start_day = 0
real, generated = self.get_compare(start_day, batch_size)
if math.isnan(generated[0][0]):
return dict()
if plot_compare:
utils.compare_plot(name, save_path, real, generated)
return metrics.get_common_metrics(
real.reshape(1, -1)[0],
generated.reshape(1, -1)[0])
def load_model(self, save_path):
print('try recover models from ' + save_path + '. ')
self.generator.load_weights(save_path + '/model_generator_weight')
self.discriminator.load_weights(save_path +
'/model_discriminator_weight')
print('models from ' + save_path + ' recovered. ')
def save_model(self, save_path, time_consumed_total):
self.generator.save_weights(save_path + '/model_generator_weight')
self.discriminator.save_weights(save_path +
'/model_discriminator_weight')
print('models saved into path: ' + save_path +
', total time consumed: %s' % time_consumed_total)
class DINETrainer(object):
def __init__(self, model, data, config):
self.loss_fn = DVContinuousLoss(name="nll_loss")
self.model = model
self.param_num = tf.cast(model['train'].count_params(), tf.float64)
self.data = data
self.config = config
self.learning_rate = ConstantScheduler(config.learning_rate)
self.optimizer = SGD(config.learning_rate)
self.global_step = tf.Variable(0, trainable=False, dtype=tf.int64)
def build_metrics():
metric_pool = list()
metric_pool.append(
dict({
"metric": DVContinuous("D_xy"),
"name": "mi",
"weight_fn": mK.nan_mask,
"target_fn": mK.split_identity_0,
"pred_fn": mK.split_identity_0
}))
metric_pool.append(
dict({
"metric": DVContinuous("D_y"),
"name": "mi",
"weight_fn": mK.nan_mask,
"target_fn": mK.split_identity_1,
"pred_fn": mK.split_identity_1
}))
metric_pool.append(
dict({
"metric": DI("di"),
"name": "mi",
"weight_fn": mK.nan_mask,
"target_fn": mK.identity,
"pred_fn": mK.identity
}))
return metric_pool
self.metrics = {
"train":
CustomMetrics(build_metrics(), config.train_writer, name='train'),
"eval":
CustomMetrics(build_metrics(), config.test_writer, name='eval')
}
def build_figures():
figure_pool = dict()
figure_pool["di"] = Plot(name='di_progress')
return figure_pool
self.visualizer = Visualization(
build_figures(), os.path.join(config.tensor_board_dir, "visual"))
@staticmethod
def metrics_data(t, t_):
return t, t_
@staticmethod
def visualize_aggr_data(t, t_):
return None
@staticmethod
def visualize_plot_data(*args):
visual_data = dict({"di": args[-1]})
return visual_data
def reset_model_states(self):
def reset_recursively(models):
for model in models.values():
if isinstance(model, dict):
reset_recursively(model)
else:
model.reset_states()
reset_recursively(self.model)
def sync_eval_model(self):
w = self.model['train'].get_weights()
self.model['eval'].set_weights(w)
# @tf.function
def compute_grads(self, x, y):
with tf.GradientTape(persistent=True) as tape:
T, T_ = self.model['train']([x, y], training=True)
loss = self.loss_fn(T, T_)
loss_xy, loss_y = tf.split(loss, num_or_size_splits=2)
gradients_xy = tape.gradient(loss_xy,
self.model['train'].trainable_weights)
gradients_y = tape.gradient(loss_y,
self.model['train'].trainable_weights)
gradients = [
g_xy + g_y for g_xy, g_y in zip(gradients_xy, gradients_y)
]
gradients, grad_norm = tf.clip_by_global_norm(
gradients, self.config.clip_grad_norm)
with self.config.train_writer.as_default():
tf.summary.scalar("loss_xy", tf.squeeze(loss_xy), self.global_step)
tf.summary.scalar("loss_y", tf.squeeze(loss_y), self.global_step)
tf.summary.scalar("loss",
tf.squeeze(loss_xy) - tf.squeeze(loss_y),
self.global_step)
tf.summary.scalar("grad_norm", grad_norm, self.global_step)
tf.summary.scalar("lr", self.learning_rate(self.global_step),
self.global_step)
self.global_step.assign_add(1)
return gradients, T, T_
# @tf.function
def apply_grads(self, gradients):
self.optimizer.apply_gradients(
zip(gradients, self.model['train'].trainable_weights))
# @tf.function
def train_step(self, x, y):
gradients, T, T_ = self.compute_grads(x, y)
self.apply_grads(gradients)
return T, T_
def eval_step(self, x):
T, T_ = self.model['eval'](x, training=False)
return T, T_
def train_epoch(self, epoch):
self.metrics["train"].reset_states()
self.reset_model_states()
for x, y in self.data["train"]:
if x is None:
self.reset_model_states()
continue
T, T_ = self.train_step(x, y)
self.metrics["train"].update_state(T, T_)
self.metrics["train"].log_metrics(epoch)
def evaluate(self, epoch, iterator="eval"):
self.metrics["eval"].reset_states()
self.visualizer.reset_states()
self.sync_eval_model()
self.reset_model_states()
for k, (x, y) in enumerate(self.data[iterator]):
T, T_ = self.eval_step([x, y])
self.metrics["eval"].update_state(*self.metrics_data(T, T_))
self.visualizer.update_state(self.visualize_aggr_data(T, T_))
results = self.metrics["eval"].log_metrics(epoch)
self.visualizer.update_state(self.visualize_plot_data(*results))
self.visualizer.plot(epoch, save=True)
def train(self):
for epoch in range(self.config.num_epochs):
if epoch % self.config.eval_freq == 0:
self.evaluate(epoch)
self.train_epoch(epoch)
self.evaluate(self.config.num_epochs, iterator="long_eval")
class POTrainer:
def __init__(self, actor, env, config):
self.actor = actor
self.env = env
self.config = config
self.T = config.unroll_steps
self.input_dim = config.channel_cardinality - 1
self.model = self._build_training_model()
lr_sche = keras.optimizers.schedules.ExponentialDecay(
self.config.learning_rate,
decay_steps=self.config.num_epochs //
self.config.learning_rate_decay_steps,
decay_rate=self.config.learning_rate_decay,
staircase=False)
self.optimizer = SGD(learning_rate=lr_sche)
self.model.compile(optimizer=self.optimizer, loss=loss)
self.mean_metric = Mean()
@tf.function
def split_rewards_and_state(self, x):
return tf.split(x, axis=-1, num_or_size_splits=[1, self.input_dim])
def _build_training_model(self):
def name(title, idx):
return "{}_{:d}".format(title, idx)
rewards = list()
states = list()
self.input = z = Input(shape=[self.input_dim])
for t in range(self.T):
u = self.actor.model(z, training=True)
z_u = Concatenate(axis=-1, name=name("concat_z_u", t))([z, u])
r, z_prime = self.env.model(z_u)
rewards.append(r)
z = z_prime
states.append(z)
self.rewards = Concatenate(axis=-1, name="concat_rewards")(rewards)
self.states = Lambda(tf.stack,
arguments={
'axis': 1,
'name': "concat_states"
})(states)
return Model(inputs=self.input, outputs=[self.rewards, self.states])
@tf.function
def train_epoch(self):
raise NotImplementedError
@tf.function
def train_step(self, z):
with tf.GradientTape() as tape:
# training=True is only needed if there are layers with different
# behavior during training versus inference (e.g. Dropout).
rewards, states = self.model(z)
loss = -K.mean(rewards)
gradients = tape.gradient(loss, self.actor.model.trainable_weights)
self.optimizer.apply_gradients(
zip(gradients, self.actor.model.trainable_weights))
return -loss, states[:, -1, :]
# train_loss(loss)
# train_accuracy(labels, predictions)
def train(self):
def lr(k):
return self.config.learning_rate * (
self.config.learning_rate_decay
**(k // (self.config.num_epochs //
self.config.learning_rate_decay_steps)))
template = "Epoch: {:05d}\tLearning Rate: {:2.2e}\tAverage Reward: {:8.5f} "
z = tf.random.uniform([self.config.batch_size, self.input_dim])
for k in range(self.config.num_epochs):
if k % self.config.eval_freq == 0:
average_reward, state_histogram = self.test(
self.config.eval_len)
print(template.format(k, lr(k), average_reward))
mlflow.log_metric("average_reward", float(average_reward), k)
I, z = self.train_step(z)
average_reward, state_histogram = self.test(self.config.eval_long_len)
mlflow.log_metric("average_reward", float(average_reward),
self.config.num_epochs)
print("Epoch: {}\tAverage Reward: {:8.5f} ".format(
"Final", average_reward))
state_clusters = KMeans(
n_clusters=self.config.n_clusters).fit(state_histogram)
with open(os.path.join(self.config.summary_dir, "log.txt"), 'a') as f:
f.write("Clusters:\n")
f.writelines(
['{}\n'.format(x) for x in state_clusters.cluster_centers_])
print(*['{}\n'.format(x) for x in state_clusters.cluster_centers_])
def test(self, eval_len):
state_histogram = list()
self.mean_metric.reset_states()
z = tf.random.uniform([self.config.batch_size_eval, self.input_dim])
for k in range(eval_len):
r, next_states = self.model.predict(z)
z = tf.squeeze(next_states[:, -1, :])
if k > eval_len // 10:
self.mean_metric(r)
state_histogram.append(next_states)
return self.mean_metric.result(), tf.reshape(
tf.concat(state_histogram, axis=0), [-1, self.input_dim])
class ClassificationTrainer:
def __init__(self, model, data, config):
self.model_train = model['train']
self.model_test = model['eval']
self.data = data
self.config = config
self.loss_fn = ClassificationLoss(config)
self.optimizer = SGD(learning_rate=self.config.learning_rate)
self.metric_train = ClassificationMetrics(config.train_writer,
name='train')
self.metric_train_eval = ClassificationMetrics(config.train_writer,
name='train_eval')
self.metric_test = ClassificationMetrics(config.test_writer,
name='test')
self.global_step = tf.Variable(0, trainable=False, dtype=tf.int64)
@tf.function
def compute_grads(self, samples, targets):
with tf.GradientTape() as tape:
predictions = self.model_train(samples, training=True)
''' generate the targets and apply the corresponding loss function '''
loss = self.loss_fn(targets, predictions)
gradients = tape.gradient(loss, self.model_train.trainable_weights)
gradients, grad_norm = tf.clip_by_global_norm(
gradients, self.config.clip_grad_norm)
with self.config.train_writer.as_default():
tf.summary.scalar("grad_norm", grad_norm, self.global_step)
self.global_step.assign_add(1)
return gradients, predictions
@tf.function
def apply_grads(self, gradients):
self.optimizer.apply_gradients(
zip(gradients, self.model_train.trainable_weights))
def sync_eval_model(self):
model_weights = self.model_train.get_weights()
ma_weights = self.model_test.get_weights()
alpha = self.config.moving_average_coefficient
self.model_test.set_weights([
ma * alpha + w * (1 - alpha)
for ma, w in zip(ma_weights, model_weights)
])
@tf.function
def train_step(self, samples, targets):
gradients, predictions = self.compute_grads(samples, targets)
self.apply_grads(gradients)
return predictions
@tf.function
def eval_step(self, samples):
predictions = self.model_test(samples, training=False)
return predictions
def train_epoch(self, epoch):
self.metric_train.reset_states()
self.model_train.reset_states()
for samples, targets in self.data['train']:
predictions = self.train_step(samples, targets)
self.metric_train.update_state(targets, predictions)
self.sync_eval_model()
self.metric_train.print(epoch)
self.metric_train.log_metrics(epoch)
def evaluate_train(self, epoch):
self.metric_train_eval.reset_states()
self.model_test.reset_states()
for samples, targets in self.data['train_eval']:
predictions = self.eval_step(samples)
self.metric_train_eval.update_state(targets, predictions)
self.metric_train_eval.print(epoch)
self.metric_train_eval.log_metrics(epoch)
def evaluate_test(self, epoch):
self.metric_test.reset_states()
self.model_test.reset_states()
for samples, targets in self.data['test']:
predictions = self.eval_step(samples)
self.metric_test.update_state(targets, predictions)
self.metric_test.print(epoch)
self.metric_test.log_metrics(epoch)
def train(self):
for epoch in range(self.config.num_epochs):
self.train_epoch(epoch)
if epoch % self.config.eval_freq == 0:
self.evaluate_train(epoch)
self.evaluate_test(epoch)
def generate_adversarial(model,
x_0,
scale,
margin,
true_negative,
max_iter=gin.REQUIRED,
w_weight=gin.REQUIRED,
border=gin.REQUIRED,
h_x_0=gin.REQUIRED,
h_x=gin.REQUIRED,
mult=gin.REQUIRED,
logloss=gin.REQUIRED,
reversedlogloss=gin.REQUIRED):
learning_rate = (mult * margin) / max_iter
if true_negative:
learning_rate = learning_rate * model._get_coef()
optimizer = SGD(
learning_rate=learning_rate
) # no momentum required due to smooth optimization landscape
# x_init is perturbed x_0, with atmost 10% of a gradient step (which can be admittely quite high)
x_init = x_0 + 0.1 * learning_rate * tf.math.l2_normalize(
tf.random.uniform(x_0.shape, -1., 1.))
x = tf.Variable(initial_value=x_init, trainable=True)
for _ in range(max_iter):
with tf.GradientTape() as tape:
try:
y = model(x)
except tf.python.framework.errors_impl.InvalidArgumentError as e:
norm = tf.reduce_sum(x**2.)
print('adversarial', norm, x)
raise e
if logloss: # binary cross-entropy
zeros = tf.zeros([int(y.shape[0]), 1]) # seek frontiere
ones = tf.ones([int(y.shape[0]), 1])
if reversedlogloss:
zeros, ones = ones, zeros
if true_negative: # where f is already negative, add examples
ce = tf.nn.sigmoid_cross_entropy_with_logits(
zeros + 0.5, y)
else: # otherwise f is positive but really shouldn't (most of the time), we remove those parts
ce = tf.nn.sigmoid_cross_entropy_with_logits(ones, y)
adversarial_score = tf.reduce_mean(ce)
if reversedlogloss:
adversarial_score = -adversarial_score
else: # wasserstein bro: sign is flipped because minimization of loss instead maximization of cost
if true_negative:
adversarial_score = tf.reduce_mean(
y) # seek x of f(x) negative
else:
adversarial_score = -tf.reduce_mean(
y) # seek x of f(x) positive
loss = w_weight * adversarial_score
if (h_x_0 + h_x + border) > 0.:
fidelity = -h_x_0 * ball_repulsion(
x, x_0, scale, margin) # avoid true positive, irrelevant
dispersion = -h_x * metric_entropy(
x, x, scale, margin) # regularization to cover space
frontiere_score = border * frontiere_distance(y, margin)
loss = loss + dispersion + fidelity + frontiere_score # minimize loss
grad_f_x = tape.gradient(loss, [x])
grad_f_x = renormalize_grads(grad_f_x) # keep good learning rate
optimizer.apply_gradients(zip(grad_f_x, [x]))
return x.value()
