class Regressor:
"""Neural network class for the Boston pricing house problem.
Attributes:
model: Keras Sequential model
"""
def __init__(self, dim: int):
self.model = Sequential()
self.model.add(Dense(144, input_dim=dim, activation="relu"))
self.model.add(Dense(72, activation="relu"))
self.model.add(Dense(18, activation="relu"))
self.model.add(Dense(1, activation="linear"))
self.model.compile(optimizer="adam", loss="mean_squared_error")
def train_n_epochs(self, n_epochs: int, x_train: pd.DataFrame,
y_train: pd.DataFrame) -> None:
"""Training function for the built in model.
Args:
n_epochs (int): Number of epochs to be trained.
x_train (~pd.dataframe): Features dataset for training.
y_train(~pd.dataframe): Labels for training.
"""
self.model.fit(x_train, y_train, epochs=n_epochs, verbose=0)
def evaluate_on_test(self, x_test: pd.DataFrame,
y_test: pd.DataFrame) -> Tuple[float, float]:
"""Evaluating on testset.
Args:
x_test (dataframe): Feature set for evaluation.
y_test (dataframe): Dependent variable for evaluation.
Returns:
test_loss: Value of the testing loss.
r_squared: Value of R-squared,
to be shown as 'accuracy' metric to the Coordinator
"""
y_pred: np.ndarray = self.model.predict(x_test)
r_squared: float = r2_score(y_test, y_pred)
test_loss: float = self.model.evaluate(x_test, y_test)
return test_loss, r_squared
def get_shapes(self) -> List[Tuple[int, ...]]:
return [weight.shape for weight in self.model.get_weights()]
def get_weights(self) -> np.ndarray:
return np.concatenate(self.model.get_weights(), axis=None)
def set_weights(self, weights: np.ndarray) -> None:
shapes = self.get_shapes()
# expand the flat weights
indices: np.ndarray = np.cumsum([np.prod(shape) for shape in shapes])
tensorflow_weights: List[np.ndarray] = np.split(
weights, indices_or_sections=indices)
tensorflow_weights = [
np.reshape(weight, newshape=shape)
for weight, shape in zip(tensorflow_weights, shapes)
]
# apply the weights to the tensorflow model
self.model.set_weights(tensorflow_weights)
# This is a sample Python script.
# Press ⌃R to execute it or replace it with your code.
# Press Double ⇧ to search everywhere for classes, files, tool windows, actions, and settings.
import tensorflow as tf
import numpy as np
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
model = Sequential([Dense(units=1, input_shape=[1])])
model.compile(optimizer='sgd', loss='mean_squared_error')
xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float)
ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float)
model.fit(xs, ys, epochs=500)
print(model.predict([10.0]))
print("Here is what i have learned: {}".format(model.get_weights()))
y_p = prices
# 缩小数据方便后续计算
features = features/100
prices = prices/100
# 定义模型
model = Sequential()
model.add(Dense(1, kernel_initializer='random_normal', bias_initializer='zeros'))
# 编译模型
model.compile(optimizer=optimizers.SGD(lr=learning_rate), loss='mse', metrics=['acc'])
# 训练模型
batch_size = 10
for epoch in range(num_epochs):
history = model.fit(x=features, y=prices, batch_size=batch_size, shuffle=True)
print('epoch ', epoch+1, ',loss = ', history.history['loss'])
print("w, b=", model.get_weights())
[w, b] = model.get_weights()
# 画图
# 青色点为数据散点图
# 蓝色方块为预期直线
# 红色圆圈为预测直线
x = np.arange(0, 200, 2)
plt.scatter(x_f, y_p, 3, hold=True, c='c')
plt.scatter(x, 6.7*x-24.42, 13, hold=True, c='b', marker='s')
plt.scatter(x, w*x+b*100, 7, c='r', marker='o')
plt.show()
def trainningNetwork(bagOfWords, y):
tiempo_i = time.time()
Errores = np.ones(10)
# Sens = np.zeros(10)
# Espec = np.zeros(10)
Precision = np.zeros(10)
Recall = np.zeros(10)
F1score = np.zeros(10)
j = 0
kf = KFold(n_splits=10, shuffle=True)
for train_index, test_index in kf.split(bagOfWords):
#print("TRAIN:", train_index, "TEST:", test_index)
X_train, X_test = bagOfWords[train_index], bagOfWords[test_index]
y_train, y_test = y[train_index], y[test_index]
#Instanciamos el modelo MLP
model = Sequential()
model.add(
Dense(units=15, activation='relu', input_dim=bagOfWords.shape[1]))
#Dropout
model.add(Dropout(0.25))
model.add(Dense(units=50, activation='relu'))
model.add(Dense(units=20, activation='relu'))
model.add(Dense(units=50, activation='relu'))
model.add(Dense(units=40, activation='relu'))
model.add(Dense(units=40, activation='relu'))
model.add(Dense(units=20, activation='relu'))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# Model config
model.get_config()
# List all weight tensors
model.get_weights()
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#Train process
model.fit(X_train, y_train, epochs=30)
# dump(model, 'mlpModel.joblib')
# Test
ypred = model.predict(X_test)
y_pred = []
for yp, yt in zip(ypred, y_test):
if yp <= 0.5:
yp = 0
else:
yp = 1
y_pred.append(yp)
#print(yp, '\t', yt)
y_pred = np.asarray(y_pred)
Errores[j] = classification_error(y_pred, y_test)
#print('Error en la iteración: ', Errores[j])
precision, recall, f1score = error_measures(y_pred, y_test)
# Sens[j] = sens
# Espec[j] = esp
Precision[j] = precision
Recall[j] = recall
F1score[j] = f1score
j += 1
return model, Errores, Precision, Recall, F1score, time.time() - tiempo_i
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Flatten, Dense, Dropout
import numpy as np
# 模型定义
model = Sequential([
Flatten(input_shape=(28, 28)),
Dense(units=200, activation='tanh'),
Dropout(0.4),
Dense(units=100, activation='tanh'),
Dropout(0.4),
Dense(units=10, activation='softmax')
])
# 保存模型参数
model.save_weights('my_model/model_weights')
# 获取模型参数
weights = model.get_weights()
# 把list转变成array
weights = np.array(weights)
# 循环每一层权值
# enumerate相当于循环计数器，记录当前循环次数
# weights保存的数据可以对照print输出查看
for i, w in enumerate(weights):
if i % 2 == 0:
print('{}:w_shape:{}'.format(int(i / 2 + 1), w.shape))
else:
print('{}:b_shape:{}'.format(int(i / 2 + 0.5), w.shape))
class Agent:
def __init__(self, env, optimizer, batch_size):
# general info
self.state_size = env.observation_space.shape[
0] # number of factors in the state; e.g: velocity, position, etc
self.action_size = env.action_space.n
self.optimizer = optimizer
self.batch_size = batch_size
# allow large replay exp space
self.replay_exp = deque(maxlen=1000000)
self.gamma = 0.99
self.epsilon = 1.0 # initialize with high exploration, which will decay later
# Build Policy Network
self.brain_policy = Sequential()
self.brain_policy.add(
Dense(128, input_dim=self.state_size, activation="relu"))
self.brain_policy.add(Dense(128, activation="relu"))
self.brain_policy.add(Dense(self.action_size, activation="linear"))
self.brain_policy.compile(loss="mse", optimizer=self.optimizer)
# Build Target Network
self.brain_target = Sequential()
self.brain_target.add(
Dense(128, input_dim=self.state_size, activation="relu"))
self.brain_target.add(Dense(128, activation="relu"))
self.brain_target.add(Dense(self.action_size, activation="linear"))
self.brain_target.compile(loss="mse", optimizer=self.optimizer)
self.update_brain_target()
# add new experience to the replay exp
def memorize_exp(self, state, action, reward, next_state, done):
self.replay_exp.append((state, action, reward, next_state, done))
"""
# agent's brain
def build_model(self):
# a NN with 2 fully connected hidden layers
model = Sequential()
model.add(Dense(128, input_dim = self.state_size, activation = "relu"))
model.add(Dense(128 , activation = "relu"))
model.add(Dense(self.action_size, activation = "linear"))
model.compile(loss = "mse", optimizer = self.optimizer)
return model
"""
def update_brain_target(self):
return self.brain_target.set_weights(self.brain_policy.get_weights())
def choose_action(self, state):
if np.random.uniform(0.0, 1.0) < self.epsilon: # exploration
action = np.random.choice(self.action_size)
else:
state = np.reshape(state, [1, state_size])
qhat = self.brain_policy.predict(
state) # output Q(s,a) for all a of current state
action = np.argmax(
qhat[0]
) # because the output is m * n, so we need to consider the dimension [0]
return action
# update params in NN
def learn(self):
"""
sample = random.choices(self.replay_exp, k = min(len(self.replay_exp), self.batch_size))
states, actions, rewards, next_states, dones = map(list, zip(sample))
# add exp to replay exp
qhats_next = self.brain_target(next_states)
# set all value actions of terminal state to 0
qhats_next[dones] = np.zeros((self.action_size))
q_targets = rewards + self.gamma * np.max(qhats_next, axis=1) # update greedily
self.brain.update_nn(self.sess, states, actions, q_targets)
"""
# take a mini-batch from replay experience
cur_batch_size = min(len(self.replay_exp), self.batch_size)
mini_batch = random.sample(self.replay_exp, cur_batch_size)
# batch data
sample_states = np.ndarray(
shape=(cur_batch_size,
self.state_size)) # replace 128 with cur_batch_size
sample_actions = np.ndarray(shape=(cur_batch_size, 1))
sample_rewards = np.ndarray(shape=(cur_batch_size, 1))
sample_next_states = np.ndarray(shape=(cur_batch_size,
self.state_size))
sample_dones = np.ndarray(shape=(cur_batch_size, 1))
temp = 0
for exp in mini_batch:
sample_states[temp] = exp[0]
sample_actions[temp] = exp[1]
sample_rewards[temp] = exp[2]
sample_next_states[temp] = exp[3]
sample_dones[temp] = exp[4]
temp += 1
sample_qhat_next = self.brain_target.predict(sample_next_states)
# set all Q values terminal states to 0
sample_qhat_next = sample_qhat_next * (
np.ones(shape=sample_dones.shape) - sample_dones)
# choose max action for each state
sample_qhat_next = np.max(sample_qhat_next, axis=1)
sample_qhat = self.brain_policy.predict(sample_states)
for i in range(cur_batch_size):
a = sample_actions[i, 0]
sample_qhat[
i,
int(a)] = sample_rewards[i] + self.gamma * sample_qhat_next[i]
q_target = sample_qhat
self.brain_policy.fit(sample_states, q_target, epochs=1, verbose=0)
"""
class DDQNAgent:
def __init__(self, env, optimizer, gamma, batch_size):
# general info
self.state_size = env.observation_space.shape[
0] # number of factors in the state; e.g: velocity, position, etc
_, self.action_size = quantize(None)
self.batch_size = batch_size
# allow large replay exp space
# self.replay_exp = deque(maxlen=BUFFER_SIZE)
self.replay_exp = ExperienceReplay(type=REPLAY_TYPE)
self.gamma = gamma
self.epsilon = 1.0 # initialize with high exploration, which will decay later
# Build Policy Network
self.brain_policy = Sequential()
self.brain_policy.add(
Dense(256, input_dim=self.state_size, activation="relu"))
self.brain_policy.add(Dense(256, activation="relu"))
self.brain_policy.add(Dense(64, activation="relu"))
self.brain_policy.add(Dense(self.action_size, activation="linear"))
self.brain_policy.compile(loss="mse", optimizer=optimizer)
# Build Target Network
self.brain_target = Sequential()
self.brain_target.add(
Dense(256, input_dim=self.state_size, activation="relu"))
self.brain_target.add(Dense(256, activation="relu"))
self.brain_target.add(Dense(64, activation="relu"))
self.brain_target.add(Dense(self.action_size, activation="linear"))
self.brain_target.compile(loss="mse", optimizer=optimizer)
self.update_brain_target()
# # add new experience to the replay exp
# def memorize_exp(self, state, action, reward, next_state, done):
# self.replay_exp.append((state, action, reward, next_state, done))
def update_brain_target(self):
return self.brain_target.set_weights(self.brain_policy.get_weights())
def choose_action(self, state):
if self._should_do_exploration():
action = np.random.choice(self.action_size)
else:
state = np.reshape(state, [1, self.state_size])
qhat = self.brain_policy.predict(
state) # output Q(s,a) for all a of current state
action = np.argmax(
qhat[0]
) # because the output is m * n, so we need to consider the dimension [0]
return action
def learn(self, sample=None):
# take a mini-batch from replay experience
if sample is None:
if self.replay_exp.is_prioritized():
cur_batch_size = min(self.replay_exp.size, self.batch_size)
mini_batch = self.replay_exp.replay_exp.sample(cur_batch_size)
else:
cur_batch_size = min(self.replay_exp.size, self.batch_size)
mini_batch = random.sample(self.replay_exp.replay_exp,
cur_batch_size)
else:
cur_batch_size = 1
mini_batch = [(0, sample)]
# batch data
sample_states = np.ndarray(shape=(cur_batch_size, self.state_size))
sample_actions = np.ndarray(shape=(cur_batch_size, 2))
sample_rewards = np.ndarray(shape=(cur_batch_size, 1))
sample_next_states = np.ndarray(shape=(cur_batch_size,
self.state_size))
sample_dones = np.ndarray(shape=(cur_batch_size, 1))
for index, exp in enumerate(mini_batch):
if self.replay_exp.is_prioritized():
sample_states[index] = exp[1][0]
sample_actions[index] = exp[1][1]
sample_rewards[index] = exp[1][2]
sample_next_states[index] = exp[1][3]
sample_dones[index] = exp[1][4]
else:
sample_states[index] = exp[0]
sample_actions[index] = exp[1]
sample_rewards[index] = exp[2]
sample_next_states[index] = exp[3]
sample_dones[index] = exp[4]
sample_qhat_next = self.brain_target.predict(sample_next_states)
# set all Q values terminal states to 0
sample_qhat_next = sample_qhat_next * (
np.ones(shape=sample_dones.shape) - sample_dones)
# choose max action for each state
sample_qhat_next = np.max(sample_qhat_next, axis=1)
sample_qhat = self.brain_policy.predict(sample_states)
if self.replay_exp.is_prioritized():
errors = np.zeros(cur_batch_size)
for i in range(cur_batch_size):
a = tuple(sample_actions[i])
sample_qhat[i, ACTIONS_TO_IDX[a]] = sample_rewards[
i] + self.gamma * sample_qhat_next[i]
if self.replay_exp.is_prioritized():
old_value = sample_qhat[i, ACTIONS_TO_IDX[tuple(a)]]
errors[i] = abs(old_value - sample_qhat[i, ACTIONS_TO_IDX[a]])
q_target = sample_qhat
# self.brain_policy.fit(sample_states, q_target, epochs=1, verbose=0)
if sample is None:
if self.replay_exp.is_prioritized():
self.replay_exp.memorize_exp(mini_batch, {
'e': errors,
'is_update': True
})
self.brain_policy.fit(sample_states, q_target, epochs=1, verbose=0)
else:
self.replay_exp.memorize_exp(sample, {
'e': errors[0],
'is_update': False
})
def _should_do_exploration(self):
return np.random.uniform(0.0, 1.0) < self.epsilon
# 保存模型结构
json_config = model.to_json()
print(json_config)
# In[3]:
import json
# 保存json模型结构文件
with open('model.json','w') as m:
json.dump(json_config,m)
# In[14]:
import numpy as np
a = np.array(model.get_weights())
for i in a:
print(i.shape)
# In[ ]:
for i in
class GAN(object):
def __init__(self, steps=1, lr=0.00001, decay=0.001):
# Models
self.D = None
self.S = None
self.G = None
self.GE = None
self.SE = None
self.DM = None
self.AM = None
# Config
self.LR = lr
self.steps = steps
self.beta = 0.999
# Init Models
self.discriminator()
self.generator()
self.GMO = Adam(lr=self.LR, beta_1=0, beta_2=0.999)
self.DMO = Adam(lr=self.LR, beta_1=0, beta_2=0.999)
self.GE = clone_model(self.G)
self.GE.set_weights(self.G.get_weights())
self.SE = clone_model(self.S)
self.SE.set_weights(self.S.get_weights())
def discriminator(self):
if self.D:
return self.D
inp = Input(shape=[im_size, im_size, 3])
x = d_block(inp, 1 * cha) # 128
x = d_block(x, 2 * cha) # 64
x = d_block(x, 4 * cha) # 32
x = d_block(x, 6 * cha) # 16
x = d_block(x, 8 * cha) # 8
x = d_block(x, 16 * cha) # 4
x = d_block(x, 32 * cha, p=False) # 4
x = Flatten()(x)
x = Dense(1, kernel_initializer="he_uniform")(x)
self.D = Model(inputs=inp, outputs=x)
return self.D
def generator(self):
if self.G:
return self.G
# === Style Mapping ===
self.S = Sequential()
self.S.add(Dense(512, input_shape=[latent_size]))
self.S.add(LeakyReLU(0.2))
self.S.add(Dense(512))
self.S.add(LeakyReLU(0.2))
self.S.add(Dense(512))
self.S.add(LeakyReLU(0.2))
self.S.add(Dense(512))
self.S.add(LeakyReLU(0.2))
# === Generator ===
# Inputs
inp_style = []
for i in range(n_layers):
inp_style.append(Input([512]))
inp_noise = Input([im_size, im_size, 1])
# Latent
x = Lambda(lambda x: x[:, :1] * 0 + 1)(inp_style[0])
outs = []
# Actual Model
x = Dense(
4 * 4 * 4 * cha, activation="relu", kernel_initializer="random_normal"
)(x)
x = Reshape([4, 4, 4 * cha])(x)
x, r = g_block(x, inp_style[0], inp_noise, 32 * cha, u=False) # 4
outs.append(r)
x, r = g_block(x, inp_style[1], inp_noise, 16 * cha) # 8
outs.append(r)
x, r = g_block(x, inp_style[2], inp_noise, 8 * cha) # 16
outs.append(r)
x, r = g_block(x, inp_style[3], inp_noise, 6 * cha) # 32
outs.append(r)
x, r = g_block(x, inp_style[4], inp_noise, 4 * cha) # 64
outs.append(r)
x, r = g_block(x, inp_style[5], inp_noise, 2 * cha) # 128
outs.append(r)
x, r = g_block(x, inp_style[6], inp_noise, 1 * cha) # 256
outs.append(r)
x = add(outs)
x = Lambda(lambda y: y / 2 + 0.5)(
x
) # Use values centered around 0, but normalize to [0, 1], providing better initialization
self.G = Model(inputs=inp_style + [inp_noise], outputs=x)
return self.G
def GenModel(self):
# Generator Model for Evaluation
inp_style = []
style = []
for i in range(n_layers):
inp_style.append(Input([latent_size]))
style.append(self.S(inp_style[-1]))
inp_noise = Input([im_size, im_size, 1])
gf = self.G(style + [inp_noise])
self.GM = Model(inputs=inp_style + [inp_noise], outputs=gf)
return self.GM
def GenModelA(self):
# Parameter Averaged Generator Model
inp_style = []
style = []
for i in range(n_layers):
inp_style.append(Input([latent_size]))
style.append(self.SE(inp_style[-1]))
inp_noise = Input([im_size, im_size, 1])
gf = self.GE(style + [inp_noise])
self.GMA = Model(inputs=inp_style + [inp_noise], outputs=gf)
return self.GMA
def EMA(self):
# Parameter Averaging
for i in range(len(self.G.layers)):
up_weight = self.G.layers[i].get_weights()
old_weight = self.GE.layers[i].get_weights()
new_weight = []
for j in range(len(up_weight)):
new_weight.append(
old_weight[j] * self.beta + (1 - self.beta) * up_weight[j]
)
self.GE.layers[i].set_weights(new_weight)
for i in range(len(self.S.layers)):
up_weight = self.S.layers[i].get_weights()
old_weight = self.SE.layers[i].get_weights()
new_weight = []
for j in range(len(up_weight)):
new_weight.append(
old_weight[j] * self.beta + (1 - self.beta) * up_weight[j]
)
self.SE.layers[i].set_weights(new_weight)
def MAinit(self):
# Reset Parameter Averaging
self.GE.set_weights(self.G.get_weights())
self.SE.set_weights(self.S.get_weights())
class MultiTaskModel(Sequential):
def __init__(self,
image_shape,
num_labels,
num_inputs=4,
trainableVariables=None,
attention=None,
two_stage=False,
pix2pix=False):
#num_inputs refers to input channels(edge,texture etc.)
#image_shape is the shape of 1 image for reconstruction
#TODO - kwargs support for segnet initializations
super(MultiTaskModel, self).__init__()
self.num_inputs = num_inputs
self.image_shape = image_shape
self.segnets = []
self.attention = attention
self.pix2pix = pix2pix
self.two_stage = two_stage
if self.attention == "self":
self.attention_gates_rec = []
self.attention_gates_pred = []
if trainableVariables is None:
self.trainableVariables = [
] #Not to be confused with trainable_variables, which is read-only
else:
self.trainableVariables = trainableVariables
for i in range(num_inputs):
#TODO make better attention layers.
self.segnets.append(SegNet())
if self.attention == "self":
self.attention_gates_rec.append(
SelfAttention([
Conv2D(filters=128,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal'),
Conv2D(filters=512,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal',
activation="sigmoid")
]))
self.attention_gates_pred.append(
SelfAttention([
Conv2D(filters=128,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal'),
Conv2D(filters=512,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal',
activation="sigmoid")
]))
elif self.attention == "multi":
self.attention_gates_rec = CompleteAttention(
layers=[
Conv2D(filters=128,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal',
activation="relu"),
Conv2D(filters=128,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal'),
Conv2D(filters=512 * self.num_inputs,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal',
activation="sigmoid")
],
num_streams=self.num_inputs)
self.attention_gates_pred = CompleteAttention(
layers=[
Conv2D(filters=128,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal',
activation="relu"),
Conv2D(filters=128,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal'),
Conv2D(filters=512 * self.num_inputs,
kernel_size=3,
padding="same",
kernel_initializer='glorot_normal',
activation="sigmoid")
],
num_streams=self.num_inputs)
print("Image_Shape", image_shape)
#TODO fix the reconstruction net to have some conv layers
self.reconstruct_image = Sequential(
[ #Conv2D(filters=512, kernel_size=2,padding="same",activation="relu"),Conv2D(filters=256, kernel_size=2,padding="valid"),
Flatten(),
BatchNormalization(axis=-1),
Dense(1024),
Dense(image_shape[0] * image_shape[1] * image_shape[2],
activation='sigmoid')
])
#Change activation to relu
#Uncomment the two lines below to enable classification
self.predict_label = Sequential([
Conv2D(filters=128,
kernel_size=2,
padding="same",
activation="relu"),
Conv2D(filters=128, kernel_size=2, padding="valid"),
Flatten(), #Dense(1000),BatchNormalization(axis=-1),
Dense(num_labels, activation='softmax')
]) #The loss function uses softmax, final preds as well
if self.pix2pix:
self.discriminator = Sequential()
disc_layers = [
ReshapeAndConcat(),
Conv2D(128, 3, padding="valid"),
MaxPool2D(pool_size=(2, 2)),
LeakyReLU(),
BatchNormalization(axis=-1),
Conv2D(128, 3, padding="valid"),
MaxPool2D(pool_size=(2, 2)),
LeakyReLU(),
Flatten(),
BatchNormalization(axis=-1),
Dense(32, activation='relu'),
BatchNormalization(axis=-1),
Dense(1,
activation='sigmoid',
kernel_initializer="glorot_normal")
]
for l in disc_layers:
self.discriminator.add(l)
def examine_disc(self, X):
for l in self.discriminator.layers[:-1]:
X = l.call(X)
return (X)
def regularizer_naive(self, means, meansum, indx):
return tf.math.abs(means[indx] / meansum)
def regularizer_ratio(self, meansr, meansp, meansumr, meansump):
sum = tf.math.abs(meansr[0] / meansumr - meansp[0] / meansump)
for i in range(1, self.num_inputs):
sum += tf.math.abs(meansr[i] / meansumr - meansp[i] / meansump)
return sum
def setTrainableVariables(self, trainableVariables=None):
if trainableVariables is not None:
self.trainableVariables = trainableVariables
return
for i in range(self.num_inputs):
print("On segnet", i)
self.trainableVariables += self.segnets[i].trainable_variables
if self.attention == "self":
for i in range(self.num_inputs):
self.trainableVariables += self.attention_gates_rec[
i].trainable_variables
self.trainableVariables += self.attention_gates_pred[
i].trainable_variables
elif self.attention == "multi":
self.trainableVariables += self.attention_gates_rec.trainable_variables
self.trainableVariables += self.attention_gates_pred.trainable_variables
self.trainableVariables += self.reconstruct_image.trainable_variables
self.trainableVariables += self.predict_label.trainable_variables
if self.pix2pix:
self.disc_train_vars = []
for l in self.discriminator.layers:
self.disc_train_vars += l.trainable_variables
# @tf.function
def build(self, X):
batch, h, w, c = X[0].shape
assert len(X) == self.num_inputs
result = []
encoded_reps, rec = self.segnets[0].call(X[0])
if self.attention == "self":
encoded_reps_rec = self.attention_gates_rec[0].call(encoded_reps)
encoded_reps_pred = self.attention_gates_pred[0].call(encoded_reps)
# encoded_reps_rec = tf.expand_dims(encoded_reps_rec,1)
# encoded_reps_pred = tf.expand_dims(encoded_reps_pred,1)
else:
# encoded_reps = tf.expand_dims(encoded_reps,1)
pass
result.append(rec)
for i in range(self.num_inputs - 1):
enc, rec = self.segnets[i + 1].call(X[i + 1])
if self.attention == "self":
encoded_attended_rec = self.attention_gates_rec[i + 1].call(
encoded_reps)
encoded_attended_pred = self.attention_gates_pred[i + 1].call(
encoded_reps)
# encoded_attended_rec = tf.expand_dims(encoded_attended_rec,1)
# encoded_attended_pred = tf.expand_dims(encoded_attended_pred,1)
print("-----------\n", encoded_reps_rec.shape,
encoded_attended_rec.shape)
encoded_reps_rec = tf.concat(
[encoded_reps_rec, encoded_attended_rec], axis=-1)
encoded_reps_pred = tf.concat(
[encoded_reps_pred, encoded_attended_pred], axis=-1)
else:
# enc = tf.expand_dims(enc,1)
encoded_reps = tf.concat([encoded_reps, enc], axis=-1)
result.append(rec)
if self.attention == "multi":
encoded_reps_rec, _, _ = self.attention_gates_rec(encoded_reps)
encoded_reps_pred, _, _ = self.attention_gates_pred(encoded_reps)
if self.attention is not None:
result.append(
tf.reshape(self.reconstruct_image(encoded_reps_rec),
(batch, h, w, c))) #
result.append(
self.predict_label(encoded_reps_pred)) #Appending final labels
if self.pix2pix:
result.append(encoded_reps_rec) #Needed for pix2pix
else:
result.append(
tf.reshape(self.reconstruct_image(encoded_reps),
(batch, h, w, c))) #
result.append(
self.predict_label(encoded_reps)) #Appending final labels
if self.pix2pix:
result.append(encoded_reps) #Needed for pix2pix
if self.pix2pix:
self.discriminator.call((result[-1], result[self.num_inputs]))
log = open("log_pix2pix.txt", "w")
# log.write("Rec {}\n".format(self.discriminator((result[-1],result[self.num_inputs]))))
# log.write("weights {}\n".format(self.discriminator.layers[-1].trainable_variables))
log.write("\n")
log.close()
@tf.function
def call(self, X, classification=False):
#X is a LIST of the dimension [batch*h*w*c]*num_inputs
#TODO check if this gives us correct appending upon flatten
#TODO refactor to make everything a tensor
batch, h, w, c = X[0].shape
assert len(X) == self.num_inputs
result = []
encoded_reps, rec = self.segnets[0].call(X[0])
if self.attention == "self":
encoded_reps_rec = self.attention_gates_rec[0].call(encoded_reps)
encoded_reps_pred = self.attention_gates_pred[0].call(encoded_reps)
# encoded_reps_rec = tf.expand_dims(encoded_reps_rec,1)
# encoded_reps_pred = tf.expand_dims(encoded_reps_pred,1)
else:
# encoded_reps = tf.expand_dims(encoded_reps,1)
pass
result.append(rec)
for i in range(self.num_inputs - 1):
enc, rec = self.segnets[i + 1].call(X[i + 1])
if self.attention == "self":
encoded_attended_rec = self.attention_gates_rec[i + 1].call(
encoded_reps)
encoded_attended_pred = self.attention_gates_pred[i + 1].call(
encoded_reps)
# encoded_attended_rec = tf.expand_dims(encoded_attended_rec,1)
# encoded_attended_pred = tf.expand_dims(encoded_attended_pred,1)
encoded_reps_rec = tf.concat(
[encoded_reps_rec, encoded_attended_rec], axis=-1)
encoded_reps_pred = tf.concat(
[encoded_reps_pred, encoded_attended_pred], axis=-1)
else:
# enc = tf.expand_dims(enc,1)
encoded_reps = tf.concat([encoded_reps, enc], axis=-1)
result.append(rec)
if self.attention == "multi":
encoded_reps_rec, meansr, meansumr = self.attention_gates_rec(
encoded_reps)
encoded_reps_pred, meansp, meansump = self.attention_gates_pred(
encoded_reps)
if self.attention is not None:
result.append(
tf.reshape(self.reconstruct_image(encoded_reps_rec),
(batch, h, w, c))) #
result.append(
self.predict_label(encoded_reps_pred)) #Appending final labels
if self.pix2pix:
result.append(encoded_reps_rec) #Needed for pix2pix
if self.attention == "multi":
result.append(meansr)
result.append(meansumr)
result.append(meansp)
result.append(meansump)
else:
result.append(
tf.reshape(self.reconstruct_image(encoded_reps),
(batch, h, w, c))) #
result.append(
self.predict_label(encoded_reps)) #Appending final labels
if self.pix2pix:
result.append(encoded_reps) #Needed for pix2pix
return result
def loss_reconstruction(self, X, Y, beta=0.0):
# print(X.shape,Y.shape)
#Pixel-wise l2 loss
# return tf.math.reduce_sum(tf.math.reduce_sum(tf.math.reduce_sum((X-Y)**2,
# axis=-1),axis=-1),axis=-1,keepdims=True) #see if keepdims is required
return (1 - beta) * tf.math.reduce_sum((X - Y)**2) / (
X.shape[1] * X.shape[2] * X.shape[3]) + beta * tf.math.reduce_sum(
tf.math.abs(X - Y)) / (X.shape[1] * X.shape[2] * X.shape[3])
def loss_classification(self, X, labels):
return (-1 *
tf.reduce_mean(labels * (tf.math.log(X + 1e-10)) +
(1 - labels) * (tf.math.log(1 - X + 1e-10))))
def generator_loss(self,
disc_generated_output,
gen_output,
target,
LAMBDA=0.1):
gan_loss = self.loss_classification(
disc_generated_output, tf.ones_like(disc_generated_output))
# mean absolute error
l1_loss = tf.reduce_mean(tf.abs(target - gen_output))
total_gen_loss = gan_loss + (LAMBDA * l1_loss)
return total_gen_loss
def discriminator_loss(self, disc_real_output, disc_generated_output):
real_loss = self.loss_classification(disc_real_output,
tf.ones_like(disc_real_output))
generated_loss = self.loss_classification(
disc_generated_output, tf.zeros_like(disc_generated_output))
# log = open("log_pix2pix.txt","a")
# log.write("Rec {}\n".format(self.examine_disc((result[-1],result[self.num_inputs]))))
# log.write("Actual {}\n".format(self.examine_disc((result[-1],Y_image))))
# log.write("Rec {}\n".format(self.discriminator((result[-1],result[self.num_inputs]))))
# log.write("Actual {}\n".format(self.discriminator((result[-1],Y_image))))
# log.write("loss {}\n".format(generated_loss))
# log.write("weights {}\n".format(self.discriminator.layers[-1].trainable_variables))
# log.close()
total_disc_loss = real_loss + generated_loss
return total_disc_loss
#TODO make this tf.function
def train_on_batch(self,
X,
Y_image,
Y_labels,
optimizer,
classification=False):
# Y needs to be a list of [img,labels]
with tf.GradientTape(persistent=True) as tape:
result = self.call(X)
losses = []
loss = 0
loss_disc = 0
if self.two_stage:
if classification == False:
loss = self.loss_reconstruction(X[0], result[0])
losses.append(loss)
for i in range(self.num_inputs - 1):
loss += self.loss_reconstruction(
X[i + 1], result[i + 1])
losses.append(
self.loss_reconstruction(X[i + 1], result[i + 1]))
#TODO FIX THIS, since result[-1] is not the required thing anymore
if self.pix2pix:
disc_real_output = self.discriminator.call(
(result[-1], Y_image))
disc_generated_output = self.discriminator.call(
(result[-1], result[self.num_inputs]))
loss += self.generator_loss(disc_generated_output,
result[self.num_inputs],
Y_image)
else:
loss += self.loss_reconstruction(
result[self.num_inputs], Y_image)
losses.append(
self.loss_reconstruction(result[self.num_inputs],
Y_image))
else:
#Uncomment the two lines below to enable classification
loss += self.loss_classification(
result[self.num_inputs + 1], Y_labels)
losses.append(
self.loss_classification(result[self.num_inputs + 1],
Y_labels))
if self.attention == 'multi':
loss += self.regularizer_ratio(
result[-2], result[-4], result[-1], result[-3]
) #result[-1],result[-3]) #TODO Have this tunable
else:
loss = self.loss_reconstruction(X[0], result[0])
losses.append(loss)
for i in range(self.num_inputs - 1):
loss += self.loss_reconstruction(X[i + 1], result[i + 1])
losses.append(
self.loss_reconstruction(X[i + 1], result[i + 1]))
if self.pix2pix:
disc_real_output = self.discriminator.call(
(result[-1], Y_image))
disc_generated_output = self.discriminator.call(
(result[-1], result[self.num_inputs]))
loss += self.generator_loss(disc_generated_output,
result[self.num_inputs],
Y_image)
loss_disc += self.discriminator_loss(
disc_real_output, disc_generated_output)
losses.append(loss_disc)
losses.append(
self.generator_loss(disc_generated_output,
result[self.num_inputs], Y_image))
else:
loss += self.loss_reconstruction(result[self.num_inputs],
Y_image)
losses.append(
self.loss_reconstruction(result[self.num_inputs],
Y_image))
#Uncomment the two lines below to enable classification
loss += self.loss_classification(result[self.num_inputs + 1],
Y_labels)
losses.append(
self.loss_classification(result[self.num_inputs + 1],
Y_labels))
if self.attention == 'multi':
loss += self.regularizer_ratio(
result[-2], result[-4], result[-1],
result[-3]) #TODO Have this tunable
# loss += self.regularizer_ratio(result[-4],result[-3],indx=0)
if self.two_stage == True and classification == True:
train_vars = self.predict_label.trainable_variables
if self.attention == "self":
for att in self.attention_gates_pred:
train_vars += att.trainable_variables
elif self.attention == "multi":
train_vars += self.attention_gates_pred.trainable_variables
grads = tape.gradient(loss, train_vars)
grads_and_vars = zip(grads, train_vars)
optimizer.apply_gradients(grads_and_vars)
else:
grads = tape.gradient(loss, self.trainableVariables)
grads_and_vars = zip(grads, self.trainableVariables)
optimizer.apply_gradients(grads_and_vars)
if self.pix2pix:
grad_disc = tape.gradient(loss_disc, self.disc_train_vars)
grads_and_vars_disc = zip(grad_disc, self.disc_train_vars)
optimizer.apply_gradients(grads_and_vars_disc)
del tape
return loss, losses
def validate_batch(self, X, Y_image, Y_labels):
# Returns predictions, losses on batch
result = self.call(X)
losses = []
loss = self.loss_reconstruction(X[0], result[0])
losses.append(loss)
for i in range(self.num_inputs - 1):
loss += self.loss_reconstruction(X[i + 1], result[i + 1])
losses.append(self.loss_reconstruction(X[i + 1], result[i + 1]))
loss += self.loss_reconstruction(result[self.num_inputs], Y_image)
# print("Loss: ",loss)
losses.append(
self.loss_reconstruction(result[self.num_inputs], Y_image))
loss += self.loss_classification(result[self.num_inputs + 1], Y_labels)
losses.append(
self.loss_classification(result[self.num_inputs + 1], Y_labels))
# print(result[-1].shape,Y_labels.shape,tf.math.argmax(result[-1],axis=1).numpy()==np.argmax(Y_labels,axis=1))
if self.pix2pix:
log = open("log_pix2pix.txt", "a")
log.write("Rec {}\n".format(
self.examine_disc((result[-1], result[self.num_inputs]))))
log.write("Actual {}\n".format(
self.examine_disc((result[-1], Y_image))))
log.write("Rec {}\n".format(
self.discriminator((result[-1], result[self.num_inputs]))))
log.write("Actual {}\n".format(
self.discriminator((result[-1], Y_image))))
log.write("weights {}\n".format(
self.discriminator.layers[-1].trainable_variables))
log.close()
return (tf.math.argmax(result[-2], axis=1).numpy() == np.argmax(
Y_labels, axis=1)).sum(), losses
else:
return (tf.math.argmax(result[-1], axis=1).numpy() == np.argmax(
Y_labels, axis=1)).sum(), losses
# return losses
def getAttentionMap(self, X):
# Saves attention map for X
if self.attention == "self":
attention_maps_rec = []
attention_maps_pred = []
batch, h, w, c = X[0].shape
# print("X.shape",h,w,c)
assert len(X) == self.num_inputs
result = []
encoded_reps, rec = self.segnets[0].call(X[0])
attention = self.attention_gates_rec[0].get_attention_map(
encoded_reps).numpy()
attention_maps_rec.append(attention)
for i in range(self.num_inputs - 1):
enc, rec = self.segnets[i + 1].call(X[i + 1])
attention = self.attention_gates_rec[i + 1].get_attention_map(
encoded_reps).numpy()
attention_maps_rec.append(
attention) #Appending the reconstructed result to return
encoded_reps, rec = self.segnets[0].call(X[0])
attention = self.attention_gates_pred[0].get_attention_map(
encoded_reps).numpy()
attention_maps_pred.append(attention)
for i in range(self.num_inputs - 1):
enc, rec = self.segnets[i + 1].call(X[i + 1])
attention = self.attention_gates_pred[i + 1].get_attention_map(
encoded_reps).numpy()
attention_maps_pred.append(
attention) #Appending the reconstructed result to return
else:
encoded_reps, _ = self.segnets[0].call(X[0])
for i in range(self.num_inputs - 1):
enc, _ = self.segnets[i + 1].call(X[i + 1])
encoded_reps = tf.concat([encoded_reps, enc], axis=-1)
attention_maps_rec = self.attention_gates_rec(
encoded_reps)[0].numpy()
attention_maps_pred = self.attention_gates_pred(
encoded_reps)[0].numpy()
result = tf.math.argmax(self.predict_label(
tf.concat(attention_maps_pred, axis=-1)),
axis=1)
return np.array(attention_maps_rec), np.array(
attention_maps_pred), result.numpy()
def save(self, modelDir):
for i in range(len(self.segnets)):
self.segnets[i].save("{}/Segnet-{}".format(modelDir, i))
if self.attention == 'self':
for i in range(len(self.segnets)):
pickle.dump(
self.attention_gates_pred[i].get_weights(),
open("{}/Attention-Pred-{}".format(modelDir, i), "wb"))
pickle.dump(
self.attention_gates_rec[i].get_weights(),
open("{}/Attention-Rec-{}".format(modelDir, i), "wb"))
elif self.attention == 'multi':
pickle.dump(self.attention_gates_pred.get_weights(),
open("{}/Attention-Pred".format(modelDir, i), "wb"))
pickle.dump(self.attention_gates_rec.get_weights(),
open("{}/Attention-Rec".format(modelDir, i), "wb"))
pickle.dump(self.reconstruct_image.get_weights(),
open("{}/Reconstruction-Model".format(modelDir), "wb"))
pickle.dump(self.predict_label.get_weights(),
open("{}/Prediction-Model".format(modelDir), "wb"))
def load_model(self, modelDir, attention, two_stage, pix2pix):
for i in range(len(self.segnets)):
self.segnets[i].load_model("{}/Segnet-{}".format(modelDir, i))
rec_train_vars = pickle.load(
open("{}/Reconstruction-Model".format(modelDir), "rb"))
pred_train_vars = pickle.load(
open("{}/Prediction-Model".format(modelDir), "rb"))
# for l in self.reconstruct_image.layers:
# weights = rec_train_vars
self.reconstruct_image.set_weights(rec_train_vars)
# for l in self.predict_label.layers:
# weights = pred_train_vars
self.predict_label.set_weights(pred_train_vars)
self.TrainableVarsSet = False
self.pix2pix = pix2pix
self.attention = attention
self.two_stage = two_stage
if self.attention == "multi":
pred_gates = pickle.load(
open("{}/Attention-Pred".format(modelDir), "rb"))
rec_gates = pickle.load(
open("{}/Attention-Rec".format(modelDir), "rb"))
self.attention_gates_pred.set_weights(pred_gates)
self.attention_gates_rec.set_weights(rec_gates)
elif self.attention == 'self':
raise NotImplementedError
