def generate_dense(layer: tf_layers.Dense, inputs: str, tmp_name: str):
ret = ''
parameters = layer.get_weights()
num_params = [np.prod(p.shape) for p in parameters]
ret += templates.load.render(num_params=np.sum(num_params))
weights = parameters[0]
weight_name = templates.parameter_offset.render(offset=0)
if len(layer.input_shape) > 2:
h = layer.input_shape[1]
w = layer.input_shape[2]
else:
h = 1
w = layer.input_shape[1]
neurons = weights.shape[1]
if len(parameters) > 1:
bias_name = templates.parameter_offset.render(offset=num_params[0])
else:
bias_name = "nullptr"
ret += templates.dense.render(input=inputs,
h=h,
w=w,
weights=weight_name,
neurons=neurons,
biases=bias_name,
ret=tmp_name)
ret += Enclave.generate_activation(layer, tmp_name, neurons)
return ret
def test(X, y, initializer):
title = f" test {type(initializer).__name__} "
print("-" * 20 + title + "-" * 20)
# create RBF network as keras sequential model
model = Sequential()
rbflayer = RBFLayer(10,
initializer=initializer,
betas=2.0,
input_shape=(1, ))
outputlayer = Dense(1, use_bias=False)
model.add(rbflayer)
model.add(outputlayer)
model.compile(loss='mean_squared_error', optimizer=RMSprop())
# fit and predict
model.fit(X, y, batch_size=50, epochs=2000, verbose=0)
y_pred = model.predict(X)
# show graph
plt.plot(X, y_pred) # prediction
plt.plot(X, y) # response from data
plt.plot([-1, 1], [0, 0], color='black') # zero line
plt.xlim([-1, 1])
# plot centers
centers = rbflayer.get_weights()[0]
widths = rbflayer.get_weights()[1]
plt.scatter(centers, np.zeros(len(centers)), s=20 * widths)
plt.show()
# calculate and print MSE
y_pred = y_pred.squeeze()
print(f"MSE: {MSE(y, y_pred):.4f}")
# saving to and loading from file
filename = f"rbf_{type(initializer).__name__}.h5"
print(f"Save model to file {filename} ... ", end="")
model.save(filename)
print("OK")
print(f"Load model from file {filename} ... ", end="")
newmodel = load_model(filename, custom_objects={'RBFLayer': RBFLayer})
print("OK")
# check if the loaded model works same as the original
y_pred2 = newmodel.predict(X).squeeze()
print("Same responses: ", all(y_pred == y_pred2))
# I know that I compared floats, but results should be identical
# save, widths & weights separately
np.save("centers", centers)
np.save("widths", widths)
np.save("weights", outputlayer.get_weights()[0])
def build_model(self):
h_size = self.hidden_size
model = Sequential()
input_hidden = Dense(h_size,
input_dim=self.input_dim,
kernel_initializer="glorot_normal",
name="input_hidden")
model.add(input_hidden)
model.add(Activation("linear"))
# Define Dense layer from hidden to output
hidden_ouput = Dense(1, name="hidden_output")
model.add(hidden_ouput)
model.add(Activation("sigmoid"))
w = input_hidden.get_weights()[0]
V = hidden_ouput.get_weights()[0]
return [model, w, V]
def sparse_fc_mapping(x, input_idxs):
num_units = len(input_idxs)
d = Dense(num_units, use_bias=False)
d.trainable = False
x = d(x)
w = d.get_weights()
w[0].fill(0)
for i in range(num_units):
w[0][input_idxs[i], i] = 1.
d.set_weights(w)
return x
def __init__(self, layer: Dense):
super().__init__(layer)
weights = layer.get_weights()
W = weights[0]
b = weights[1]
# Set up_func for DDense
i = Input(shape=layer.input_shape[1:])
o = Dense(units=layer.output_shape[1])(i)
o.weights = [W, b]
self.up_func = K.function([i], [o])
# Transpose W and set down_func for DDense
W = W.transpose()
self.input_shape = layer.input_shape
self.output_shape = layer.output_shape
b = np.zeros(self.input_shape[1])
flipped_weights = [W, b]
i = Input(shape=self.output_shape[1:])
o = Dense(units=self.input_shape[1])(i)
o.weights = flipped_weights
self.down_func = K.function([i], [o])
class KBNN():
def __init__(self,
LF,
input_dim,
hidden_units,
output_dim=1,
dropout=None,
final_bias=False):
# High fidelity "correcting" layers
self.myLayers = []
for i in range(len(hidden_units)):
self.myLayers.append(Dense(hidden_units[i], activation='softplus'))
if dropout:
self.myLayers.append(Dropout(dropout))
self.myLayers.append(Dense(output_dim))
# Low fidelity model, lf = rho*LF(rho2*inputs+a2)
# shift/scale layer, has weight of rho2 and bias of a2
self.rho2_a2 = Dense(input_dim, kernel_initializer='ones')
# Set LF model as not trainable
self.LF = LF
self.LF.trainable = False
# weight LF contribution
self.rho = Dense(output_dim, kernel_initializer='ones', use_bias=False)
inputs = Input(shape=(input_dim, ))
# Pass input through HF correcting layers
y = inputs
for layer in self.myLayers:
y = layer(y)
correction = y
# Pass input through shifted/scaled/weighted LF model
# lf = self.rho*self.LF(self.rho2*inputs+self.a2)
lf = inputs
lf = self.rho2_a2(lf)
lf = self.LF(lf)
lf = self.rho(lf)
# Combine in output layer
outputs = Add()([correction, lf])
self.model = Model(inputs=inputs, outputs=outputs)
# self.LF = LF
# self.rho = tf.Variable(1.)
# self.rho2 = tf.Variable(np.ones(input_dim,dtype=np.float32))
# self.a2 = tf.Variable(np.zeros(input_dim,dtype=np.float32))
def kbnn_loss(self, y_true, y_pred):
c1 = 5. #10.
c2 = 5. #10.
c3 = 5. #10.
MSE = keras.losses.mean_squared_error
r = self.rho.get_weights()[0]
r2 = self.rho2_a2.get_weights()[0]
a2 = self.rho2_a2.get_weights()[1]
#print('MSE(y_true, y_pred).shape: ', MSE(y_true,y_pred).shape)
#print('a2.shape: ', a2.shape)
return (MSE(y_true, y_pred) + c1 * (r - 1.)**2 +
c2 * tf.tensordot(r2 - 1, r2 - 1, axes=2) +
c3 * tf.tensordot(a2, a2, axes=1))
class DenseApproximator(Approximator):
def __init__(self, num_layers, num_units, output_dim, internal_dim,
activation):
super().__init__(output_dim, internal_dim)
self.activation = activation
self.dense_layers = []
self.batch_layers = []
for _ in range(num_layers - 1):
self.dense_layers.append(Dense(units=num_units, use_bias=False))
self.batch_layers.append(BatchNormalization())
self.output_layer = Dense(units=output_dim + internal_dim)
def _evaluate_layer(self, x, dense, batch, training):
x = dense(x, training=training)
x = batch(x, training=training)
return self.activation(x)
def _call(self, inputs, training=False):
"""Implementation of call for Strategy.
Args:
inputs: (batch_size, None)
training: bool
Returns:
output: see Strategy._call
"""
for dense, batch in zip(self.dense_layers, self.batch_layers):
inputs = self._evaluate_layer(inputs, dense, batch, training)
output = self.output_layer(inputs, training=training)
return output
def initialise(self, inputs, sample_size):
batch_size = tf.shape(inputs)[0]
iterator = zip(self.dense_layers, self.batch_layers)
for k, (dense, batch) in enumerate(iterator):
sample_idx = tf.random.shuffle(tf.range(batch_size))[:sample_size]
sample = tf.gather(inputs, sample_idx, axis=0)
for i in tf.range(k + 1):
sample = self._evaluate_layer(sample, self.dense_layers[i],
self.batch_layers[i], False)
mean, variance = tf.nn.moments(sample, 0)
# dense.set_weights([dense.get_weights()[0] / tf.sqrt(variance)])
batch.set_weights([
batch.get_weights()[0] / tf.sqrt(variance),
(batch.get_weights()[1] - mean) / tf.sqrt(variance),
batch.get_weights()[2],
batch.get_weights()[3]
])
sample_idx = tf.random.shuffle(tf.range(batch_size))[:sample_size]
sample = tf.gather(inputs, sample_idx, axis=0)
sample = self._call(sample, False)
mean, variance = tf.nn.moments(sample, 0)
self.output_layer.set_weights([
self.output_layer.get_weights()[0] / tf.sqrt(variance),
self.output_layer.get_weights()[1]
])
from tensorflow.keras import Sequential
from tensorflow.keras.optimizers import Adam
matplotlib.rcParams['font.family'] = 'Malgun Gothic'
matplotlib.rcParams['axes.unicode_minus'] = False
import warnings
warnings.filterwarnings('ignore')
C = np.array([-40, -10, 0, 8, 15, 22, 38], dtype=np.float32)
F = np.array([-40, 14, 32, 46, 59, 72, 100], dtype=np.float32)
# label 갯수 --> units
# feature 갯수 --> input_shape
D = Dense(units=1, input_shape=[1])
model = Sequential(D)
model.compile(loss="min-squared-error", optimizer=Adam(learning_rate=0.1))
history = model.fit(C, F, epochs=500)
print(history.history["loss"])
print(D.get_weights())
# W
print(D.get_weights()[0])
# b
print(D.get_weights()[1])
plt.plot(history.history["loss"])
plt.show()
# we will let a very simple neural network learn fibonacci numbers
input_values = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0], dtype=float)
prediction_values = np.array([ 0.0, 0.0, 1.0, 1.0, 2.0, 3.0, 5.0, 8.0, 55.0], dtype=float)
# on layer should do it
learning_layer = Dense(units=1, input_shape=[1])
model = Sequential([learning_layer])
model.compile(optimizer='sgd', loss='mean_squared_error')
model.fit(input_values, prediction_values, epochs=500)
# let's see what it predicts for 8.0, the correct value would be 21
print('\nPrediction for 8 (correct value is 21):', model.predict([8.0]))
# export the learned weights, so that we can later use them in tensor serving and TF Lite
print("learned weights {}".format(learning_layer.get_weights()))
export_dir = 'learned_model/1'
tf.saved_model.save(model, export_dir)
converter = tf.lite.TFLiteConverter.from_saved_model(export_dir)
tflite_model = converter.convert()
tflite_model_file = pl.Path('model.tflite')
tflite_model_file.write_bytes(tflite_model)
# test whether loading model would work
interpreter = tf.lite.Interpreter(model_content=tflite_model)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
to_predict = np.array([[8.0]], dtype=np.float32)
interpreter.set_tensor(input_details[0]['index'], to_predict)
interpreter.invoke()
max_q = np.max(Q_t1)
output = hidden_layers(s_t1.reshape(1, INPUT_SIZE))[0][0]
min_q = np.min(output)
max_q = np.max(output)
left = ((output[2] - min_q) *
255) / (max_q - min_q) if max_q != min_q else 0
nothing = ((output[0] - min_q) *
255) / (max_q - min_q) if max_q != min_q else 0
right = ((output[1] - min_q) *
255) / (max_q - min_q) if max_q != min_q else 0
display[0][0] = (left, 0, 0)
display[0][1] = (0, nothing, 0)
display[0][2] = (0, 0, right)
theta_sum = output_layer.get_weights()[0].sum(
axis=0) + output_layer.get_weights()[1]
# print(f"Weights: [{theta_sum[2]:.2f}, {theta_sum[0]:.2f}, {theta_sum[1]:.2f}]",
# f"hidden_layers[{hidden_layers(s_t1.reshape(1,INPUT_SIZE))}]")
display[2][0] = ((theta_sum[2] - theta_sum.min()) * 255 /
(theta_sum.max() - theta_sum.min()), 0, 0)
display[2][1] = (0, (theta_sum[0] - theta_sum.min()) * 255 /
(theta_sum.max() - theta_sum.min()), 0)
display[2][2] = (0, 0, (theta_sum[1] - theta_sum.min()) * 255 /
(theta_sum.max() - theta_sum.min()))
img = Image.fromarray(
display, 'RGB'
) # reading to rgb. Apparently. Even tho color definitions are bgr. ???
img = img.resize(
(300, 50), Image.NONE
) # resizing so we can see our agent in all its glory.
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.layers import Dense
from tensorflow.keras import Sequential
from tensorflow.keras.optimizers import Adam
data = np.loadtxt("../../data/cross-entropy.txt", dtype=np.float32)
x_data = data[:, 1:3]
y_data = data[:, 3:]
print(x_data.shape)
print(y_data.shape)
IO = Dense(units=3, input_shape=[2], activation="cross-entropy")
model = Sequential([IO])
model.compile(loss="categorical_crossentropy",
optimizer=Adam(learning_rate=0.01),
metrics=["accuracy"])
history = model.fit(x_data, y_data, epochs=1000)
print(IO.get_weights())
p = model.predict(np.array([[3., 6.]]))
print(history.history['acc'][-1])
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
EPOCHS = 500
x = np.array([-1, 0, 1, 2, 3, 4], dtype=float)
y = np.array([-3, -1, 1, 3, 5, 7], dtype=float)
dense1 = Dense(1, input_shape=[1])
mod = Sequential(dense1)
mod.compile(optimizer='sgd', loss='mse')
print(mod.summary())
mod.fit(x, y, epochs=EPOCHS, verbose=0)
W, b = dense1.get_weights()
print(f'Weights:\n W: {W}\n b: {b}')
pred = mod.predict([10.])
plt.scatter(x, y)
plt.scatter(10., pred, color='r')
plt.show()
class SRFR(Model):
def __init__(
self,
num_filters: int = 62,
depth: int = 50,
categories: int = 512,
num_gc: int = 32,
num_blocks: int = 23,
residual_scailing: float = 0.2,
training: bool = True,
input_shape=(28, 28, 3),
num_classes_syn: int = None,
both: bool = False,
num_classes_nat: int = None,
scale: int = 64,
):
super(SRFR, self).__init__()
self._training = training
self.scale = scale
if both:
self._natural_input = Conv2D(
input_shape=input_shape,
filters=num_filters,
kernel_size=(3, 3),
strides=1,
padding='same',
name='natural_input',
activation=mish,
)
self._synthetic_input = Conv2D(
input_shape=input_shape,
filters=num_filters,
kernel_size=(3, 3),
strides=1,
padding='same',
name='synthetic_input',
activation=mish,
)
self._super_resolution = GeneratorNetwork(
num_filters,
num_gc,
num_blocks,
residual_scailing,
)
self._face_recognition = ResNet(
depth,
categories,
training
)
if self._training:
if both:
self._fc_classification_nat = Dense(
input_shape=(categories,),
units=num_classes_nat,
activation=None,
use_bias=False,
dtype='float32',
name='fully_connected_to_softmax_crossentropy_nat',
)
self._fc_classification_nat.build(tf.TensorShape([None, 512]))
self.net_type = 'nat'
self._fc_classification_syn: Dense = Dense(
input_shape=(categories,),
units=num_classes_syn,
activation=None,
use_bias=False,
dtype='float32',
name='fully_connected_to_softmax_crossentropy_syn',
)
self._fc_classification_syn.build(tf.TensorShape([None, 512]))
@tf.function
def _call_evaluating(self, input_tensor, input_type: str = 'nat'):
if input_type == 'syn':
outputs = self._synthetic_input(input_tensor)
else:
outputs = self._natural_input(input_tensor)
super_resolution_image = self._super_resolution(outputs)
embeddings = self._face_recognition(super_resolution_image)
return super_resolution_image, embeddings
def _calculate_normalized_embeddings(self, embeddings,
net_type: str = 'syn'):
fc_weights = self.get_weights(net_type)
normalized_weights = tf.Variable(
normalize(fc_weights, name='weights_normalization'),
aggregation=tf.VariableAggregation.NONE,
)
normalized_embeddings = normalize(
embeddings, axis=1, name='embeddings_normalization') * self.scale
replica = tf.distribute.get_replica_context()
replica.merge_call(self.set_weights,
args=(normalized_weights, net_type))
return self.call_fc_classification(normalized_embeddings, net_type)
def _call_training(self, synthetic_images, natural_images=None):
synthetic_outputs = self._synthetic_input(synthetic_images)
synthetic_sr_images = self._super_resolution(synthetic_outputs)
synthetic_embeddings = self._face_recognition(synthetic_sr_images)
synthetic_embeddings = self._calculate_normalized_embeddings(
synthetic_embeddings
)
if natural_images:
natural_outputs = self._natural_input(natural_images)
natural_sr_images = self._super_resolution(natural_outputs)
natural_embeddings = self._face_recognition(natural_sr_images)
natural_embeddings = self._calculate_normalized_embeddings(
natural_embeddings
)
return (
synthetic_sr_images,
synthetic_embeddings,
natural_sr_images,
natural_embeddings,
)
return synthetic_sr_images, synthetic_embeddings
def call(self, input_tensor_01, input_tensor_02=None,
training: bool = True, input_type: str = 'nat'):
if training:
return self._call_training(input_tensor_01, input_tensor_02)
return self._call_evaluating(input_tensor_01, input_type)
def get_weights(self, net_type: str = 'syn'):
if net_type == 'nat':
return self._fc_classification_nat.get_weights()
return self._fc_classification_syn.get_weights()
def set_weights(self, _, weights, net_type: str = 'syn') -> None:
if net_type == 'nat':
self._fc_classification_nat.set_weights([weights.read_value()])
else:
self._fc_classification_syn.set_weights([weights.read_value()])
def call_fc_classification(self, input, net_type: str = 'syn'):
if net_type == 'nat':
return self._fc_classification_nat(input)
return self._fc_classification_syn(input)
import numpy as np
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
l0 = Dense(units=1, input_shape=[1])
model = Sequential([l0])
model.compile(optimizer='sgd', loss='mean_squared_error')
xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float)
ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float)
model.fit(xs, ys, epochs=500)
print(model.predict([10.0]))
print("Here is what I learned: {}".format(l0.get_weights()))
def make_np_layer_iteration(data: np.array, layer: Dense, activation_fn: Callable) -> np.array:
weights = layer.get_weights()[0]
biases = layer.get_weights()[1]
return activation_fn(np.dot(data, weights) + biases)
class SCHEM():
def __init__(self,
run_name,
out_features,
optim,
load_im_size,
cs_weight,
fe_weight,
img_location,
crop_size,
init_W,
end_layer,
scale_CS_W=False,
l2_norm=False,
schsm_sampling='hard',
batch_size=60,
LR=0.0001,
K_constant=5,
beta=5,
eta=10):
"""
:param run_name: Allows you to give unique names to each run
:param out_features: Number of output features
:param optim: Learning rate model optemizer
:param load_im_size: Size of image to load
:param cs_weight: Coefficent in front of L_c
:param fe_weight: Coefficent in front of L_t
:param img_location: Location of folder containing .npy files
:param crop_size: Paper uses 224
:param init_W: Weight initialization
:param K_constant: Same value from paper
:param eta: Same value from paper
:param beta: Same value from paper
:param end_layer: Number of layers to cut off from MobileNetV2
:param scale_CS_W: Boolean to scale CS weights to have a magnitude of 1.
:param l2_norm: Boolean that controls if the feature map G has l2-normalization applied
:param schsm_sampling:
"""
self.schsm_sampling_method = schsm_sampling
self.l2_norm = l2_norm
self.batch_size = batch_size
self.scale_sig = scale_CS_W
self.init_W = init_W
self.end_layer = end_layer
self.K_constant = K_constant
self.beta = beta
self.cs_weight = cs_weight
self.fe_weight = fe_weight
self.optim = optim
self.out_features = out_features
self.LR = LR
self.init_W = init_W
self.model_location = None
self.img_location = img_location
self.tf_m = tf.constant(self.batch_size * 3.0)
self.load_im_size = load_im_size
self.crop_size = crop_size
self.in_shape = (crop_size, crop_size, 3)
self.prepare_data()
self.eta = eta
self.n_batch_size = batch_size - eta
self.run_name = run_name
self.name_mod = '(%.1f,%.1f)_el(%i)%s' % (fe_weight, cs_weight,
end_layer, self.run_name)
self.save_mod = '%s/' % self.name_mod
self.results_dir = 'results(%i)' % crop_size
self.tsne_out = '%s/plots/tsne/' % self.results_dir + self.save_mod
self.loss_out = '%s/plots/loss/' % self.results_dir + self.save_mod
self.csv_out = '%s/csv/' % self.results_dir + self.save_mod
self.acc_out = '%s/plots/acc/' % self.results_dir + self.save_mod
self.acc_out_bin = '%s/plots_bin' % self.results_dir
self.model_out = '%s/model_weights/' % self.results_dir + self.save_mod
make_dirs([
self.tsne_out, self.loss_out, self.acc_out, self.model_out,
self.acc_out_bin
])
def base_network(self):
mob_net = tf.keras.applications.MobileNetV2(weights='imagenet',
include_top=False,
input_shape=self.in_shape)
fmg = Layer(name="Feature_map_G_1")(
mob_net.layers[-self.end_layer].output)
fmg = Conv2D(self.out_features, (1, 1),
name='Feature_map_G_2',
activation='relu')(fmg)
fmg = tf.keras.layers.BatchNormalization(axis=1,
trainable=False,
name='Feature_map_G_3')(fmg)
x = GlobalAveragePooling2D()(fmg)
if self.l2_norm:
x = Lambda(lambda a: tf.math.l2_normalize(a, axis=1),
name='l2_norm')(x)
outmodel = Model(inputs=mob_net.input,
outputs=x,
name='base_FE_network')
self.map_G = Model(inputs=mob_net.input,
outputs=fmg,
name='base_FE_network')
return outmodel
def create_model_and_compile(self):
input_shape = self.in_shape
self.base_model = self.base_network()
self.cs_layer = Dense(self.num_classes,
use_bias=False,
name='CS_layer')
qpn_in = Input(shape=input_shape, name='in_qpn')
qpny = Input(shape=(self.num_classes, ), name='in_qpny')
qpny_label = Input(shape=(1, ), name='in_qpny_label')
qpn_out = self.base_model(qpn_in)
qpn_cls_sig = self.cs_layer(qpn_out)
fe_loss = Lambda(self.triplet_loss_all_combinations,
name='triplet_loss_FE')(qpn_out) * self.fe_weight
fe_accuracy = Lambda(self.FE_accuracy,
name='FE_accuracy_metric')(qpn_out)
cs_loss = Lambda(self.manual_CS_loss, name='CS_loss_calc')(
[qpn_cls_sig, qpny]) * self.cs_weight
cs_accuracy = Lambda(self.CS_accuracy,
name='CS_Acc')([qpn_cls_sig, qpny])
total_loss = fe_loss + cs_loss
model = Model(inputs=[qpn_in, qpny, qpny_label],
outputs=[qpn_cls_sig],
name='FEModel')
if 'adm' in self.optim:
optm = Adam(lr=self.LR)
elif 'ranger' in self.optim: # option to use a newer optimizer
radam = tfa.optimizers.RectifiedAdam(lr=self.LR, min_lr=1e-7)
optm = tfa.optimizers.Lookahead(radam,
sync_period=6,
slow_step_size=0.5)
model.add_loss(total_loss)
model.compile(optimizer=optm)
# Metrics to track the accuracy and loss progression
model.add_metric(fe_accuracy, name='fe_a', aggregation='mean')
model.add_metric(cs_accuracy, name='cs_a', aggregation='mean')
model.add_metric(fe_loss, name='fe_loss', aggregation='mean')
model.add_metric(cs_loss, name='cs_loss_out', aggregation='mean')
return model, optm
def row_wise_subtraction_and_comparison(self, i, qp, n):
"""
:param i: index to grab from qp
:param qp: all QP images
:param n: all N images
:return: all elements from a specific row of QP subtracted from all elements in n
"""
# Takes a row and subtracts n from all elemnts
i = tf.cast(i, tf.int32)
q_elm = tf.gather(qp, i)
qp_elm_sub = tf.expand_dims(q_elm, axis=0) - qp
qp_elm_mag = tf.math.sqrt(
tf.reduce_sum(tf.math.square(qp_elm_sub), axis=1))
n_sub = tf.expand_dims(q_elm, axis=0) - n
n_mag = tf.math.sqrt(tf.reduce_sum(tf.math.square(n_sub), axis=1))
return tf.expand_dims(qp_elm_mag, axis=1) - n_mag
def triplet_loss_all_combinations(self, args):
qp = args[:self.eta, :]
n = args[self.eta:, :]
all_combinations = tf.map_fn(
lambda i: self.row_wise_subtraction_and_comparison(i, qp, n),
elems=tf.range(self.eta),
dtype=tf.float32)
loss = all_combinations + tf.constant(0.2) # Hinge loss margin in 0.2
loss = K.maximum(loss, tf.constant(0.0))
# Using weighting scheme to give more weight to the hard triplets.
num_non_zero = tf.math.count_nonzero(loss, dtype=tf.dtypes.float32)
l_out = tf.cond(tf.equal(num_non_zero,
tf.constant(0.0)), lambda: tf.constant(0.0),
lambda: tf.divide(tf.reduce_sum(loss), num_non_zero))
return tf.reduce_mean(loss)
def manual_CS_loss(self, args):
#Paper implementation of L_c loss
cs, qpn_y = args
qpn_yf = tf.cast(qpn_y, tf.float32)
a = tf.nn.softmax_cross_entropy_with_logits(qpn_yf,
cs,
axis=1,
name='cross_entropy')
return tf.reduce_mean(a)
def get_cs_row(self, i, qpn_y, cse_out):
i = tf.cast(i, tf.int32)
curr_class = tf.where(tf.gather(qpn_y, i))
row = tf.gather(cse_out, i)
elm = tf.gather(row, curr_class)
row_denom = tf.reduce_sum(row)
elm_norm = elm / tf.maximum(row_denom, tf.keras.backend.epsilon())
return tf.squeeze(tf.math.log(elm_norm))
def get_class_pool(self, qx, W, qp_idx, alpha):
# Calculating P_c from paper
W = tf.math.l2_normalize(W, axis=0)
qx = tf.math.l2_normalize(qx, axis=1)
sim_space = tf.tensordot(qx, W, axes=1)
sim_space_max = tf.reduce_max(sim_space, axis=0)
closes_class_idx = tf.argsort(sim_space_max, direction='DESCENDING')
out = tf.boolean_mask(closes_class_idx,
~tf.equal(closes_class_idx, qp_idx))
return out[:alpha * (self.K_constant - 1)]
def CS_accuracy(self, args):
qpncs, qpny = args
qpt = tf.cast(
tf.equal(tf.argmax(qpny, axis=1), tf.argmax(qpncs, axis=1)),
tf.float32)
return K.mean(qpt)
def rand_flip_and_crop(self, batch):
batch = tf.map_fn(
lambda x: tf.image.random_crop(
x, size=[self.crop_size, self.crop_size, 3]), batch)
batch = tf.map_fn(tf.image.random_flip_left_right, batch)
return batch
def center_crop(self, batch):
# Used during testing
start_point = self.load_im_size - self.crop_size
batch = tf.image.crop_to_bounding_box(batch, start_point, start_point,
self.crop_size, self.crop_size)
return batch
def prepare_data(self):
x_train = np.load('%s/trnx_CUB_%s.npy' %
(self.img_location, self.load_im_size))
y_train = np.load('%s/trny_CUB_%s.npy' %
(self.img_location, self.load_im_size))
y_train_OH = np.load('%s/trny_OH_CUB_%s.npy' %
(self.img_location, self.load_im_size))
x_test = np.load('%s/tstx_CUB_%s.npy' %
(self.img_location, self.load_im_size))
y_test = np.load('%s/tsty_CUB_%s.npy' %
(self.img_location, self.load_im_size))
y_train_num_label, y_test_num_label = [], []
for elm in y_train_OH:
y_train_num_label.append(list(elm).index(1))
for elm in y_test:
y_test_num_label.append(int(elm.split('.')[0]))
self.all_c = list(set(y_train_num_label))
self.num_classes = len(self.all_c)
y_train_num_label = np.array(y_train_num_label)
y_test_num_label = np.array(y_test_num_label)
y_train_OH = y_train_OH.astype(bool)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
self.x_train = x_train
self.y_train_OH = y_train_OH
self.y_train = y_train_num_label
self.x_test = x_test
self.y_test = y_test_num_label
return x_train, y_train_OH, y_train_num_label, x_test, y_test_num_label
def generate_G(self, images, mode):
# Generate the feature map G. Done in batches due to memeory restrictions
if mode == 'test':
x_shaped = self.center_crop(images)
else:
x_shaped = self.rand_flip_and_crop(images)
# This split is done due to help wiht memory overflow
test_batch_split = np.array_split(np.arange(len(x_shaped)), 15)
feats = np.array([])
for t_bat in test_batch_split:
tmp = tf.keras.backend.batch_flatten(
self.map_G(tf.gather(x_shaped, t_bat)))
if tf.size(feats) == 0:
feats = tmp
else:
feats = tf.concat([feats, tmp], axis=0)
return feats
def generate_out(self, images, mode):
# Generate the network output layer that is trained on
# Used just for debugging
if mode == 'test':
x_shaped = self.center_crop(images)
else:
x_shaped = self.rand_flip_and_crop(images)
tmp = 15
if self.load_im_size == 256:
tmp = 50
test_batch_split = np.array_split(np.arange(len(x_shaped)), tmp)
feats = np.array([])
for t_bat in test_batch_split:
tmp = tf.keras.backend.batch_flatten(
self.base_model(tf.gather(x_shaped, t_bat)))
if tf.size(feats) == 0:
feats = tmp
else:
feats = tf.concat([feats, tmp], axis=0)
return feats
def get_batch(self):
# x_train, y_train, y_train_OH
qp_idx = self.q_idx
xqp_inds = np.where(self.y_train_OH[:, qp_idx])
all_Xqp = self.x_train[xqp_inds]
all_qpy_label = self.y_train[xqp_inds]
all_qpy = self.y_train_OH[xqp_inds]
batch_qp_idx = np.random.choice(np.arange(len(all_Xqp)),
self.eta,
replace=False)
batch_qp = all_Xqp[batch_qp_idx]
batch_qp = self.rand_flip_and_crop(batch_qp)
batch_qpy = all_qpy[batch_qp_idx]
batch_qpy_label = all_qpy_label[batch_qp_idx]
return batch_qp, batch_qpy, batch_qpy_label
def feature_map_G_statistics(self):
xd = self.x_test
y_labels = self.y_test
g_feats = self.generate_G(xd, mode='test')
feats1 = tf.math.l2_normalize(g_feats, axis=1)
sim_matrix = tf.tensordot(feats1, tf.transpose(feats1), axes=1)
closest_neighbor = tf.argsort(sim_matrix,
direction='DESCENDING',
axis=1)[:, 1] # k = 1 here
grab_class = tf.gather(y_labels, closest_neighbor)
acc = tf.cast(tf.math.equal(y_labels, grab_class), tf.float32)
acc1 = tf.reduce_mean(acc)
###################### compare with output R@1
out_feats = self.generate_out(xd, mode='test')
feats1 = tf.math.l2_normalize(out_feats, axis=1)
sim_matrix = tf.tensordot(feats1, tf.transpose(feats1), axes=1)
closest_neighbor = tf.argsort(sim_matrix,
direction='DESCENDING',
axis=1)[:, 1] # k = 1 here
grab_class = tf.gather(y_labels, closest_neighbor)
acc = tf.cast(tf.math.equal(y_labels, grab_class), tf.float32)
acc5 = tf.reduce_mean(acc)
return acc1.numpy(), acc5.numpy()
def generate_all_statistics(self):
xd = self.x_test
y_labels = self.y_test
feats = self.generate_G(xd, mode='test')
feats1 = tf.math.l2_normalize(feats, axis=1)
sim_matrix = tf.tensordot(feats1, tf.transpose(feats1), axes=1)
# doing [:, 1] to remove the first column which is the correlation with itself
closest_neighbor = tf.argsort(sim_matrix,
direction='DESCENDING',
axis=1)[:, 1]
grab_class = tf.gather(y_labels, closest_neighbor)
acc = tf.reduce_mean(
tf.cast(tf.math.equal(y_labels, grab_class), tf.float32))
return acc.numpy()
def FE_accuracy(self, args):
qp = args[:self.eta, :]
n = args[self.eta:, :]
indicator = tf.map_fn(
lambda i: self.row_wise_subtraction_and_comparison(i, qp, n),
elems=tf.range(self.eta),
dtype=tf.float32)
comparison = tf.cast(tf.math.greater(tf.constant(0.0), indicator),
tf.float32)
return tf.reduce_mean(comparison)
def schem_sample(self, batch_qp):
x_train = self.x_train
y_train = self.y_train
y_train_OH = self.y_train_OH
if 'hard' in self.schsm_sampling_method: # Hard sampling
qp_features = tf.keras.backend.batch_flatten(
self.base_model(batch_qp))
qp_features = tf.math.l2_normalize(qp_features, axis=1)
Pc = self.get_class_pool(qp_features,
self.cs_layer.get_weights()[0],
self.q_idx, self.alpha)
n_indexes = []
# Could automate this in a tensor, but probably don't need to
for elm in Pc:
indexes_for_class = np.where(y_train == elm)[0]
n_indexes += list(indexes_for_class)
# Generate feature space for all N elements.
Ps_all = x_train[n_indexes]
n_all_y = y_train_OH[n_indexes]
n_all_y_label = y_train[n_indexes]
Ps_features = self.generate_out(Ps_all, mode='train')
Ps_features = tf.math.l2_normalize(Ps_features, axis=1)
similarity = tf.tensordot(qp_features,
tf.transpose(Ps_features),
axes=1)
# Take top similarities from all q-inputs.
max_sim = tf.reduce_max(similarity, axis=0)
max_sum = tf.reduce_sum(similarity, axis=0)
similar_indicies_sorted = tf.argsort(max_sum,
axis=0,
direction='DESCENDING')
top_similar_indicies = tf.argsort(
max_sim, axis=0,
direction='DESCENDING')[:self.beta * self.n_batch_size]
n_idx = np.random.choice(top_similar_indicies, self.n_batch_size)
n_bat = tf.gather(Ps_all, n_idx)
n_bat = self.rand_flip_and_crop(
n_bat) # maybe i should crop before and not randomly re-crop
n_y = tf.gather(n_all_y, n_idx)
n_y_label = tf.gather(n_all_y_label, n_idx)
else: # Naive sampling
n_idxs = np.random.choice(
np.setdiff1d(np.arange(self.num_classes), self.q_idx), 10)
n_bat = np.array([])
n_y = np.array([])
n_y_label = np.array([])
for n_idx in n_idxs:
tot_elms = sum(y_train_OH[:, n_idx])
xqp_inds = np.where(y_train_OH[:, n_idx])
all_Xn = x_train[xqp_inds]
all_yn = y_train_OH[xqp_inds]
all_yn_label = y_train[xqp_inds]
b_n_idxs = np.random.choice(np.arange(tot_elms),
2,
replace=False)
if n_bat.size == 0:
n_bat = all_Xn[b_n_idxs]
n_y = all_yn[b_n_idxs]
n_y_label = all_yn_label[b_n_idxs]
else:
n_bat = np.append(n_bat, all_Xn[b_n_idxs], axis=0)
n_y = np.append(n_y, all_yn[b_n_idxs], axis=0)
n_y_label = np.append(n_y_label,
all_yn_label[b_n_idxs],
axis=0)
n_bat = self.rand_flip_and_crop(n_bat)
return n_bat, n_y, n_y_label
import tensorflow as tf
import numpy as np
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
L0 = Dense(units=1, input_shape=[1])
model = Sequential([L0])
model.compile(optimizer='sgd', loss='mean_squared_error')
xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float)
ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float)
model.fit(xs, ys, epochs=500)
print(model.predict([10.0]))
print("Here is what i've learned: {}".format(L0.get_weights()))
export_dir = 'saved_model/1'
tf.saved_model.save(model, export_dir)
# -
# ## Model # 1
layer_0 = Dense(units=1, input_shape=[1])
model = Sequential()
model.add(layer_0)
model.compile(loss='mean_squared_error',
optimizer=tf.keras.optimizers.Adam(0.1))
model.summary()
# + id="ay8_Sd4MShDO"
history = model.fit(x=celsius_q, y=fahrenheit_a, epochs=500, verbose=False)
# + colab={"base_uri": "https://localhost:8080/", "height": 296} id="I7pas988SOjO" outputId="2758d191-7b14-4afd-803a-90813ad5c553"
plt.xlabel('Epoch')
plt.ylabel("Loss")
plt.plot(history.history['loss'])
# -
# ### Prediction
# + colab={"base_uri": "https://localhost:8080/"} id="SyZ6kaQ6SQd2" outputId="e84ede3c-b3b1-46f5-b90d-ee4c3581d8a2"
print(model.predict([100.0]))
# + colab={"base_uri": "https://localhost:8080/"} id="elWiukimSSKe" outputId="ecb17893-07e6-4ca1-b0f8-b8a746e4d480"
print("These are the layer variables: " + str(layer_0.get_weights()))
# Note the formula is F = 1.8 * C + 32 , where weight is 1.8 and bias is 32
# +
print("End of the code")
class GAPAlexnet(tf.keras.Model):
def __init__(self, num_classes, keep_prob):
super(GAPAlexnet, self).__init__()
# Possibly experiment - different initializations
# TODO - regularization? see paper
self.num_classes = num_classes
self.keep_prob = keep_prob
self.conv1 = Conv2D(96, 11, strides=(4,4), activation='relu')
self.pool1 = MaxPool2D(pool_size=(3,3), strides=(2,2))
self.pad = ZeroPadding2D(padding=(2,2))
self.conv2 = Conv2D(256, 5, activation='relu')
self.pool2 = MaxPool2D(pool_size=(3,3), strides=(2,2))
self.conv3 = Conv2D(384, 3, padding='same', activation='relu')
self.conv4 = Conv2D(384, 3, padding='same', activation='relu')
self.conv5 = Conv2D(256, 3, padding='same', activation='relu')
self.pool3 = MaxPool2D(pool_size=(3,3), strides=(2,2))
self.gap = GlobalAveragePooling2D()
# self.flat = Flatten()
self.drop_gap = Dropout(rate=self.keep_prob)
self.fc8 = Dense(units=self.num_classes, use_bias=False)
self.bn1 = BatchNormalization(axis=-1)
self.bn2 = BatchNormalization(axis=-1)
self.bn3 = BatchNormalization(axis=-1)
self.bn4 = BatchNormalization(axis=-1)
self.bn5 = BatchNormalization(axis=-1)
def update_last_layer(self, num_classes):
self.num_classes = num_classes
self.fc8 = Dense(units=self.num_classes, use_bias=False)
# Input - datum[0], datum[1] or datum[2], datum[3]
def call(self, inputImg, mode='train'):
# o1 = lrn(self.pool1(self.conv1(inputImg)), 2, 2e-05, 0.75)
# o2 = lrn((self.pool2(self.conv2(self.pad(o1)))), 2, 2e-05, 0.75)
train_bool = False
if(mode == 'train'):
train_bool = True
o1 = lrn(self.pool1(self.conv1(inputImg)), 2, 2e-05, 0.75)
o2 = lrn(self.pool2(self.conv2(self.pad(o1))), 2, 2e-05, 0.75)
o3 = self.conv3(o2)
# o1 = self.pool1(self.bn1(self.conv1(inputImg), training=train_bool))
# o2 = self.pool2(self.bn2(self.conv2(self.pad(o1)), training=train_bool) )
o3 = (self.conv3(o2))
o4 = (self.conv4(o3))
o5 = (self.conv4(o4))
# o3 = self.bn3(self.conv3(o2), training=train_bool)
# o4 = self.bn4(self.conv4(o3), training=train_bool)
# o5 = self.bn5(self.conv5(o4), training=train_bool)
pooled_o5 = self.pool3(o5)
gap_out = self.gap(pooled_o5)
logits = self.fc8(gap_out)
if(mode == 'train' or mode == 'eval'):
return logits
elif(mode == 'maps'):
# return tf.reduce_mean(o5, axis=-1)
pred_class = (tf.argmax(logits, axis=-1))
np_wts = tf.convert_to_tensor(self.fc8.get_weights()[0])
# print("SHAPE", np_wts.shape)
np_wts = tf.transpose(np_wts)
# print("TRANSPOSE SHAPE", np_wts.shape)
# weights = tf.gather(tf.transpose(tf.convert_to_tensor(self.fc8.get_weights()[0])) , pred_class)
weights = tf.gather(np_wts , pred_class)
# print("FINAL SHAPE", weights.shape)
# exit()
# weights = tf.convert_to_tensor(self.fc8.get_weights()[0][:, pred_class])
weights = tf.expand_dims(weights, axis=-1)
OMAP = pooled_o5
bsize, xdim, ydim, c = tf.shape(OMAP)[0], tf.shape(OMAP)[1], tf.shape(OMAP)[2], tf.shape(OMAP)[3]
OMAP = tf.reshape(OMAP, [bsize, xdim*ydim, c])
cam_map = tf.squeeze(tf.matmul(OMAP, weights), [2])
cam_map = tf.reshape(cam_map, [bsize, xdim, ydim])
return cam_map, pred_class
else:
assert(False)
class ResnetGenerator(tf.keras.layers.Layer):
def __init__(self,
input_nc,
output_nc,
ngf=64,
n_blocks=6,
img_size=256,
light=False):
super(ResnetGenerator, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.n_blocks = n_blocks
self.img_size = img_size
self.light = light
DownBlock = [
ReflectionPad2D(3),
Conv2D(filters=ngf,
kernel_size=7,
strides=1,
padding='valid',
use_bias=False),
InstanceNormalization(axis=3),
ReLU()
]
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
DownBlock += [
ReflectionPad2D(1),
Conv2D(filters=ngf * mult * 2,
kernel_size=3,
strides=2,
padding='valid',
use_bias=False),
InstanceNormalization(axis=3),
ReLU()
]
mult = 2**n_downsampling
for i in range(n_blocks):
DownBlock += [ResnetBlock(ngf * mult, use_bias=False)]
self.gap_fc = Dense(1, use_bias=False)
self.gmp_fc = Dense(1, use_bias=False)
self.conv1x1 = Conv2D(filters=ngf * mult,
kernel_size=1,
strides=1,
use_bias=False)
self.relu = ReLU()
FC = [
Dense(ngf * mult, use_bias=False),
ReLU(),
Dense(ngf * mult, use_bias=False),
ReLU()
]
self.gamma = Dense(ngf * mult, use_bias=False)
self.beta = Dense(ngf * mult, use_bias=False)
for i in range(n_blocks):
setattr(self, 'UpBlock1_' + str(i + 1),
ResnetAdaILNBlock(ngf * mult, use_bias=False))
UpBlock2 = []
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
UpBlock2 += [
UpSampling2D(size=(2, 2), interpolation='nearest'),
ReflectionPad2D(1),
Conv2D(filters=int(ngf * mult / 2),
kernel_size=3,
strides=1,
padding='valid',
use_bias=False),
ILN(int(ngf * mult / 2)),
ReLU()
]
UpBlock2 += [
ReflectionPad2D(3),
Conv2D(filters=output_nc,
kernel_size=7,
strides=1,
padding='valid',
use_bias=False),
Tanh()
]
self.DownBlock = tf.keras.Sequential(DownBlock)
self.FC = tf.keras.Sequential(FC)
self.UpBlock2 = tf.keras.Sequential(UpBlock2)
def call(self, inputs):
x = self.DownBlock(inputs)
gap = tf.reduce_mean(x, axis=[1, 2], keepdims=True)
gap_logit = self.gap_fc(tf.reshape(gap, shape=[x.shape[0], -1]))
gap_weight = tf.transpose(self.gap_fc.get_weights()[0], perm=[1, 0])
gap = x * tf.expand_dims(tf.expand_dims(gap_weight, axis=1), axis=2)
gmp = tf.reduce_max(x, axis=[1, 2], keepdims=True)
gmp_logit = self.gmp_fc(tf.reshape(gmp, shape=[x.shape[0], -1]))
gmp_weight = tf.transpose(self.gmp_fc.get_weights()[0], perm=[1, 0])
gmp = x * tf.expand_dims(tf.expand_dims(gmp_weight, axis=1), axis=2)
cam_logit = tf.concat([gap_logit, gmp_logit], 1)
x = tf.concat([gap, gmp], 3)
x = self.relu(self.conv1x1(x))
heatmap = tf.reduce_sum(x, axis=[3], keepdims=True)
if self.light:
x_ = tf.reduce_mean(x, axis=[1, 2], keepdims=True)
x_ = self.FC(tf.reshape(x_, shape=[x_.shape[0], -1]))
else:
x_ = self.FC(tf.reshape(x_, shape=[x.shape[0], -1]))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
def make_naive_layer_iteration(data: np.array, layer: Dense, activation_fn: Callable) -> np.array:
weights = layer.get_weights()[0]
biases = layer.get_weights()[1]
return activation_fn(naive_add_matrix_and_vector(naive_matrix_matrix_dot(data, weights), biases))
class FactorizedSeasonality(Seasonality):
def __init__(self,
n_time_series,
n_dim=1,
seasonality_type="weekday",
l2_reg=1e-3):
super().__init__(seasonality_type)
self.n_time_series = n_time_series
self.n_dim = n_dim
self.l2_reg = l2_reg
self._embedding = None
self._inputs = None
self._encoder = None
self._decoder = None
self._create_embedding()
self._create_inputs()
self._create_decoder()
self._create_encoder()
def _create_embedding(self):
# First embedding for time series ID
self._embedding = Embedding(
self.n_time_series,
output_dim=self.n_dim,
name="seas_emb_%s" % self.seasonality_type,
embeddings_initializer=constant(0),
embeddings_regularizer=l1_l2(
l2=self.l2_reg) if self.l2_reg else None)
self._seasonality_weights = Dense(self.seasonal_period,
activation='linear')
def _create_inputs(self):
self._inputs = [
Input(shape=(1, ),
name="timeseries_id_%s" % self.seasonality_type),
Input(shape=(None, ),
name="inp_X_seas_%s" % self.seasonality_type,
dtype=tf.int32),
Input(shape=(None, ),
name="inp_Y_seas_%s" % self.seasonality_type,
dtype=tf.int32)
]
def _create_decoder(self):
id_emb = self._embedding(self._inputs[0])[:, 0, :]
id_seas_values = self._seasonality_weights(id_emb)
seas_values = tf.gather(id_seas_values,
self._inputs[1],
axis=1,
batch_dims=-1)
self._decoder = seas_values[:, :, None] + 1
def _create_encoder(self):
id_emb = self._embedding(self._inputs[0])[:, 0, :]
id_seas_values = self._seasonality_weights(id_emb)
seas_values = tf.gather(id_seas_values,
self._inputs[2],
axis=1,
batch_dims=-1)
self._encoder = seas_values[:, :, None] + 1
def get_fit_inputs(self, ids, x):
return [ids, x[:, :-1], x]
def get_predict_inputs(self, ids, x, last_date, increment):
# Expand x by one for predicting one step ahead
x_expanded = np.hstack([x, self._get_next_x(last_date, increment, 1)])
return [ids, x, x_expanded]
def get_oos_predictions(self, first_oos_prediction, last_date, increment,
n_oos_steps):
x_oos = self._get_next_x(last_date, increment, n_oos_steps)
id_seas_values = self.get_weights()
deseason = np.take_along_axis(id_seas_values, x_oos[:, :1], axis=1)
oos_season = np.take_along_axis(id_seas_values, x_oos[:, 1:], axis=1)
return first_oos_prediction * oos_season / deseason
def get_weights(self):
return self._embedding.get_weights()[0] @ self._seasonality_weights.get_weights()[0] + \
self._seasonality_weights.get_weights()[1] + 1
#sys.exit()
model.fit(x=X_train_train,
y=y_labels_train_train,
shuffle=True,
epochs=epochs,
batch_size=batch_size,
callbacks=[
EarlyStopping(monitor='val_loss',
patience=20,
restore_best_weights=True)
],
validation_data=(X_train_val, y_labels_train_val))
new_weights = np.empty([100, 37])
biases = outLayer.get_weights()[1]
for i in range(len(outLayer.get_weights()[0])):
weights = outLayer.get_weights()[0][i]
for j in range(len(weights)):
if j not in (32, 33, 34, 35, 36): #subtypes
weights[j] = 0
new_weights[i] = weights
outLayer.set_weights([new_weights, biases])
score = model.evaluate(X_test, y_labels_test)
print(model.predict(X_test).argmax(axis=1))
conf_matrix = pd.DataFrame(
confusion_matrix(y_labels_test.values.argmax(axis=1),
model.predict(X_test).argmax(axis=1)))
layer_1 = Dense(units=4)
layer_2 = Dense(units=1)
model = Sequential()
model.add(layer_0)
model.add(layer_1)
model.add(layer_2)
model.compile(loss='mean_squared_error',
optimizer=tf.keras.optimizers.Adam(0.1))
model.summary()
# Layer 0 : weights 1 x 4 and bias 4 ; Layer 1 : weights 4 x 4 and bias 4 ; Layer 3 : weights 4 x 1 and bias = 1
history = model.fit(x=celsius_q, y=fahrenheit_a, epochs=500, verbose=False)
# Check any change in the loss when more layers are added compared to one layer in the last ex?
plt.xlabel('Epoch')
plt.ylabel("Loss")
plt.plot(history.history['loss'])
# # Predictions
print(model.predict([100.0]))
# Note the formula is F = 1.8 * C + 32 , where weight is 1.8 and bias is 32 if only one layer and one unit
print("These are the layer_0 variables: " + str(layer_0.get_weights()) + "\n")
print("These are the layer_1 variables: " + str(layer_1.get_weights()) + "\n")
print("These are the layer_2 variables: " + str(layer_2.get_weights()) + "\n")
class CNNLayerwise(tf.keras.Model):
def __init__(self, hyperp, run_options, data_input_shape, label_dimensions, num_channels, kernel_regularizer, bias_regularizer):
super(CNNLayerwise, self).__init__()
###############################################################################
# Construct Initial Neural Network Architecture #
###############################################################################
#=== Defining Attributes ===#
self.data_input_shape = data_input_shape
self.architecture = [] # storage for layer information, each entry is [filter_size, num_filters]
self.num_filters = hyperp.num_filters
self.kernel_size = hyperp.filter_size
self.activation = hyperp.activation
self.kernel_regularizer = kernel_regularizer
self.bias_regularizer = bias_regularizer
self.hidden_layers_list = [] # This will be a list of Keras layers
#=== Define Initial Architecture and Create Layer Storage ===#
self.architecture.append([self.data_input_shape[0], num_channels]) # input information
self.architecture.append([3, self.num_filters]) # 3x3 convolutional layer for upsampling data
self.architecture.append([hyperp.filter_size, self.num_filters]) # First hidden layer
self.architecture.append([3, num_channels]) # 3x3 convolutional layer for downsampling features
self.architecture.append(label_dimensions) # fully-connected output layer
print(self.architecture)
#=== Weights and Biases Initializer ===#
kernel_initializer = RandomNormal(mean=0.0, stddev=0.05)
bias_initializer = 'zeros'
#=== Linear Upsampling Layer to Map to Feature Space ===#
l = 1
self.upsampling_layer = Conv2D(self.architecture[l][1], (3, 3), padding = 'same',
activation = 'linear', use_bias = True,
input_shape = self.data_input_shape,
kernel_initializer = kernel_initializer, bias_initializer = bias_initializer,
kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer,
name='upsampling_layer')
#=== Define Hidden Layers ===#
l = 2
conv_layer = Conv2D(self.architecture[l][1], (self.architecture[l][0], self.architecture[l][0]), padding = 'same',
activation = self.activation, use_bias = True,
input_shape = (None, self.data_input_shape[0], self.data_input_shape[1], self.num_filters),
kernel_initializer = kernel_initializer, bias_initializer = bias_initializer,
kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer,
name = "W" + str(l))
self.hidden_layers_list.append(conv_layer)
#=== Linear Downsampling Layer to Map to Data Space ===#
l = 3
self.downsampling_layer = Conv2D(self.architecture[l][1], (3, 3), padding = 'same',
activation = "linear", use_bias = True,
input_shape = (None, self.data_input_shape[0], self.data_input_shape[1], self.num_filters),
kernel_initializer = kernel_initializer, bias_initializer = bias_initializer,
kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer,
name = "downsampling_layer")
#=== Classification Layer ===#
self.classification_layer = Dense(units = label_dimensions,
activation = 'linear', use_bias = True,
kernel_initializer = kernel_initializer, bias_initializer = bias_initializer,
kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer,
name = 'classification_layer')
###############################################################################
# Network Propagation #
###############################################################################
def call(self, inputs):
#=== Upsampling ===#
output = self.upsampling_layer(inputs)
for hidden_layer in self.hidden_layers_list:
#=== Hidden Layers ===#
prev_output = output
output = prev_output + hidden_layer(output)
#=== Downsampling ===#
output = self.downsampling_layer(output)
#=== Classification ===#
output = Flatten()(output)
output = self.classification_layer(output)
return output
###############################################################################
# Add Layer #
###############################################################################
def add_layer(self, trainable_hidden_layer_index, freeze = True, add = True):
kernel_initializer = 'zeros'
bias_initializer = 'zeros'
if add:
conv_layer = Conv2D(self.num_filters, (self.kernel_size, self.kernel_size), padding = 'same',
activation = self.activation, use_bias = True,
input_shape = (None, self.data_input_shape[0], self.data_input_shape[1], self.num_filters),
kernel_initializer = kernel_initializer, bias_initializer = bias_initializer,
kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer,
name = "W" + str(trainable_hidden_layer_index))
self.hidden_layers_list.append(conv_layer)
if freeze:
self.upsampling_layer.trainable = False
for index in range(0, trainable_hidden_layer_index-2):
self.hidden_layers_list[index].trainable = False
else:
self.upsampling_layer.trainable = True
for index in range(0, trainable_hidden_layer_index-2):
self.hidden_layers_list[index].trainable = True
###############################################################################
# Sparsify Weights #
###############################################################################
def sparsify_weights_and_get_relative_number_of_zeros(self, threshold = 1e-6):
#=== Downsampling Layer ===#
down_weights = self.downsampling_layer.get_weights()
sparsified_weights = self.sparsify_weights(down_weights, threshold)
self.downsampling_layer.set_weights(sparsified_weights)
#=== Classification Layer ===#
class_weights = self.classification_layer.get_weights()
sparsified_weights = self.sparsify_weights(class_weights, threshold)
self.classification_layer.set_weights(sparsified_weights)
#=== Trained Hidden Layer ===#
trained_weights = self.hidden_layers_list[-1].get_weights()
sparsified_weights = self.sparsify_weights(trained_weights, threshold)
self.hidden_layers_list[-1].set_weights(sparsified_weights)
#=== Compute Relative Number of Zeros ===#
total_number_of_zeros = 0
total_number_of_elements = 0
for i in range(0, len(sparsified_weights)):
total_number_of_zeros += np.count_nonzero(sparsified_weights[i]==0)
total_number_of_elements += sparsified_weights[i].flatten().shape[0]
relative_number_zeros = np.float64(total_number_of_zeros/total_number_of_elements)
return relative_number_zeros
def sparsify_weights(self, weights, threshold = 1e-6):
sparsified_weights = []
if isinstance(weights, float):
if abs(weights) > threshold:
sparsified_weights = weights
else:
sparsified_weights = 0
else:
for w in weights:
bool_mask = (abs(w) > threshold).astype(int)
sparsified_weights.append(w*bool_mask)
return sparsified_weights
class MyModel(Model):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = Conv2D(32,
3,
activation='relu',
autocast=False,
dtype=tf.float32)
self.flatten = Flatten(autocast=False)
self.d1 = Dense(128,
activation='relu',
autocast=False,
dtype=tf.float32)
self.d2 = Dense(10,
activation='softmax',
autocast=False,
dtype=tf.float32)
self.loss_object = tf.keras.losses.SparseCategoricalCrossentropy()
#自定义优化器
self.optimizer = tf.keras.optimizers.SGD()
self.train_loss = tf.keras.metrics.Mean(name='train_loss')
self.train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
name='train_accuracy')
self.test_loss = tf.keras.metrics.Mean(name='test_loss')
self.test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
name='test_accuracy')
# 训练步骤
@tf.function
def train_step_(self, images, labels):
with tf.GradientTape() as tape:
predictions = self(images)
loss = self.loss_object(labels, predictions)
gradients = tape.gradient(loss, self.trainable_variables)
self.optimizer.apply_gradients(zip(gradients,
self.trainable_variables))
self.train_loss(loss)
self.train_accuracy(labels, predictions)
# 测试步骤
@tf.function
def test_step_(self, images, labels):
predictions = self(images)
t_loss = self.loss_object(labels, predictions)
self.test_loss(t_loss)
self.test_accuracy(labels, predictions)
def test(self):
self.test_loss.reset_states()
self.test_accuracy.reset_states()
for test_images, test_labels in self.test_ds:
self.test_step_(test_images, test_labels)
template = 'Test Loss: {}, Test Accuracy: {}'
print(
template.format(self.test_loss.result(),
self.test_accuracy.result() * 100))
return self.test_accuracy.result() * 100
# 具体的计算call
def call(self, x):
x = self.conv1(x)
x = self.flatten(x)
x = self.d1(x)
return self.d2(x)
# 训练
def train_test(self, epoch_data_weight, index):
for epoch in range(epoch_data_weight):
# 在下一个epoch开始时，重置评估指标
self.train_loss.reset_states()
self.train_accuracy.reset_states()
# self.test_loss.reset_states()
# self.test_accuracy.reset_states()
for images, labels in self.train_ds:
self.train_step_(images, labels)
# 先去掉测试集
# for test_images, test_labels in self.test_ds:
# self.test_step_(test_images, test_labels)
template = 'index {} Epoch {}, Loss: {}, Accuracy: {}'
print(
template.format(index, epoch + 1, self.train_loss.result(),
self.train_accuracy.result() * 100))
def get_weights(self):
'''
:return: data_listq
'''
weight = []
weight.append(self.conv1.get_weights())
weight.append(self.d1.get_weights())
weight.append(self.d2.get_weights())
return weight
# 对应每层的权重设置
def set_weights(self, weight):
self.conv1.set_weights(weights=weight[0])
self.d1.set_weights(weights=weight[1])
self.d2.set_weights(weights=weight[2])
# 为每个模型设置数据集
def set_data(self, train_ds, test_ds=None):
self.train_ds = train_ds
self.test_ds = test_ds
