def generate_dense(layer: tf_layers.Dense, inputs: str, tmp_name: str): ret = '' parameters = layer.get_weights() num_params = [np.prod(p.shape) for p in parameters] ret += templates.load.render(num_params=np.sum(num_params)) weights = parameters[0] weight_name = templates.parameter_offset.render(offset=0) if len(layer.input_shape) > 2: h = layer.input_shape[1] w = layer.input_shape[2] else: h = 1 w = layer.input_shape[1] neurons = weights.shape[1] if len(parameters) > 1: bias_name = templates.parameter_offset.render(offset=num_params[0]) else: bias_name = "nullptr" ret += templates.dense.render(input=inputs, h=h, w=w, weights=weight_name, neurons=neurons, biases=bias_name, ret=tmp_name) ret += Enclave.generate_activation(layer, tmp_name, neurons) return ret
def test(X, y, initializer): title = f" test {type(initializer).__name__} " print("-" * 20 + title + "-" * 20) # create RBF network as keras sequential model model = Sequential() rbflayer = RBFLayer(10, initializer=initializer, betas=2.0, input_shape=(1, )) outputlayer = Dense(1, use_bias=False) model.add(rbflayer) model.add(outputlayer) model.compile(loss='mean_squared_error', optimizer=RMSprop()) # fit and predict model.fit(X, y, batch_size=50, epochs=2000, verbose=0) y_pred = model.predict(X) # show graph plt.plot(X, y_pred) # prediction plt.plot(X, y) # response from data plt.plot([-1, 1], [0, 0], color='black') # zero line plt.xlim([-1, 1]) # plot centers centers = rbflayer.get_weights()[0] widths = rbflayer.get_weights()[1] plt.scatter(centers, np.zeros(len(centers)), s=20 * widths) plt.show() # calculate and print MSE y_pred = y_pred.squeeze() print(f"MSE: {MSE(y, y_pred):.4f}") # saving to and loading from file filename = f"rbf_{type(initializer).__name__}.h5" print(f"Save model to file {filename} ... ", end="") model.save(filename) print("OK") print(f"Load model from file {filename} ... ", end="") newmodel = load_model(filename, custom_objects={'RBFLayer': RBFLayer}) print("OK") # check if the loaded model works same as the original y_pred2 = newmodel.predict(X).squeeze() print("Same responses: ", all(y_pred == y_pred2)) # I know that I compared floats, but results should be identical # save, widths & weights separately np.save("centers", centers) np.save("widths", widths) np.save("weights", outputlayer.get_weights()[0])
def build_model(self): h_size = self.hidden_size model = Sequential() input_hidden = Dense(h_size, input_dim=self.input_dim, kernel_initializer="glorot_normal", name="input_hidden") model.add(input_hidden) model.add(Activation("linear")) # Define Dense layer from hidden to output hidden_ouput = Dense(1, name="hidden_output") model.add(hidden_ouput) model.add(Activation("sigmoid")) w = input_hidden.get_weights()[0] V = hidden_ouput.get_weights()[0] return [model, w, V]
def sparse_fc_mapping(x, input_idxs): num_units = len(input_idxs) d = Dense(num_units, use_bias=False) d.trainable = False x = d(x) w = d.get_weights() w[0].fill(0) for i in range(num_units): w[0][input_idxs[i], i] = 1. d.set_weights(w) return x
def __init__(self, layer: Dense): super().__init__(layer) weights = layer.get_weights() W = weights[0] b = weights[1] # Set up_func for DDense i = Input(shape=layer.input_shape[1:]) o = Dense(units=layer.output_shape[1])(i) o.weights = [W, b] self.up_func = K.function([i], [o]) # Transpose W and set down_func for DDense W = W.transpose() self.input_shape = layer.input_shape self.output_shape = layer.output_shape b = np.zeros(self.input_shape[1]) flipped_weights = [W, b] i = Input(shape=self.output_shape[1:]) o = Dense(units=self.input_shape[1])(i) o.weights = flipped_weights self.down_func = K.function([i], [o])
class KBNN(): def __init__(self, LF, input_dim, hidden_units, output_dim=1, dropout=None, final_bias=False): # High fidelity "correcting" layers self.myLayers = [] for i in range(len(hidden_units)): self.myLayers.append(Dense(hidden_units[i], activation='softplus')) if dropout: self.myLayers.append(Dropout(dropout)) self.myLayers.append(Dense(output_dim)) # Low fidelity model, lf = rho*LF(rho2*inputs+a2) # shift/scale layer, has weight of rho2 and bias of a2 self.rho2_a2 = Dense(input_dim, kernel_initializer='ones') # Set LF model as not trainable self.LF = LF self.LF.trainable = False # weight LF contribution self.rho = Dense(output_dim, kernel_initializer='ones', use_bias=False) inputs = Input(shape=(input_dim, )) # Pass input through HF correcting layers y = inputs for layer in self.myLayers: y = layer(y) correction = y # Pass input through shifted/scaled/weighted LF model # lf = self.rho*self.LF(self.rho2*inputs+self.a2) lf = inputs lf = self.rho2_a2(lf) lf = self.LF(lf) lf = self.rho(lf) # Combine in output layer outputs = Add()([correction, lf]) self.model = Model(inputs=inputs, outputs=outputs) # self.LF = LF # self.rho = tf.Variable(1.) # self.rho2 = tf.Variable(np.ones(input_dim,dtype=np.float32)) # self.a2 = tf.Variable(np.zeros(input_dim,dtype=np.float32)) def kbnn_loss(self, y_true, y_pred): c1 = 5. #10. c2 = 5. #10. c3 = 5. #10. MSE = keras.losses.mean_squared_error r = self.rho.get_weights()[0] r2 = self.rho2_a2.get_weights()[0] a2 = self.rho2_a2.get_weights()[1] #print('MSE(y_true, y_pred).shape: ', MSE(y_true,y_pred).shape) #print('a2.shape: ', a2.shape) return (MSE(y_true, y_pred) + c1 * (r - 1.)**2 + c2 * tf.tensordot(r2 - 1, r2 - 1, axes=2) + c3 * tf.tensordot(a2, a2, axes=1))
class DenseApproximator(Approximator): def __init__(self, num_layers, num_units, output_dim, internal_dim, activation): super().__init__(output_dim, internal_dim) self.activation = activation self.dense_layers = [] self.batch_layers = [] for _ in range(num_layers - 1): self.dense_layers.append(Dense(units=num_units, use_bias=False)) self.batch_layers.append(BatchNormalization()) self.output_layer = Dense(units=output_dim + internal_dim) def _evaluate_layer(self, x, dense, batch, training): x = dense(x, training=training) x = batch(x, training=training) return self.activation(x) def _call(self, inputs, training=False): """Implementation of call for Strategy. Args: inputs: (batch_size, None) training: bool Returns: output: see Strategy._call """ for dense, batch in zip(self.dense_layers, self.batch_layers): inputs = self._evaluate_layer(inputs, dense, batch, training) output = self.output_layer(inputs, training=training) return output def initialise(self, inputs, sample_size): batch_size = tf.shape(inputs)[0] iterator = zip(self.dense_layers, self.batch_layers) for k, (dense, batch) in enumerate(iterator): sample_idx = tf.random.shuffle(tf.range(batch_size))[:sample_size] sample = tf.gather(inputs, sample_idx, axis=0) for i in tf.range(k + 1): sample = self._evaluate_layer(sample, self.dense_layers[i], self.batch_layers[i], False) mean, variance = tf.nn.moments(sample, 0) # dense.set_weights([dense.get_weights()[0] / tf.sqrt(variance)]) batch.set_weights([ batch.get_weights()[0] / tf.sqrt(variance), (batch.get_weights()[1] - mean) / tf.sqrt(variance), batch.get_weights()[2], batch.get_weights()[3] ]) sample_idx = tf.random.shuffle(tf.range(batch_size))[:sample_size] sample = tf.gather(inputs, sample_idx, axis=0) sample = self._call(sample, False) mean, variance = tf.nn.moments(sample, 0) self.output_layer.set_weights([ self.output_layer.get_weights()[0] / tf.sqrt(variance), self.output_layer.get_weights()[1] ])
from tensorflow.keras import Sequential from tensorflow.keras.optimizers import Adam matplotlib.rcParams['font.family'] = 'Malgun Gothic' matplotlib.rcParams['axes.unicode_minus'] = False import warnings warnings.filterwarnings('ignore') C = np.array([-40, -10, 0, 8, 15, 22, 38], dtype=np.float32) F = np.array([-40, 14, 32, 46, 59, 72, 100], dtype=np.float32) # label 갯수 --> units # feature 갯수 --> input_shape D = Dense(units=1, input_shape=[1]) model = Sequential(D) model.compile(loss="min-squared-error", optimizer=Adam(learning_rate=0.1)) history = model.fit(C, F, epochs=500) print(history.history["loss"]) print(D.get_weights()) # W print(D.get_weights()[0]) # b print(D.get_weights()[1]) plt.plot(history.history["loss"]) plt.show()
# we will let a very simple neural network learn fibonacci numbers input_values = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0], dtype=float) prediction_values = np.array([ 0.0, 0.0, 1.0, 1.0, 2.0, 3.0, 5.0, 8.0, 55.0], dtype=float) # on layer should do it learning_layer = Dense(units=1, input_shape=[1]) model = Sequential([learning_layer]) model.compile(optimizer='sgd', loss='mean_squared_error') model.fit(input_values, prediction_values, epochs=500) # let's see what it predicts for 8.0, the correct value would be 21 print('\nPrediction for 8 (correct value is 21):', model.predict([8.0])) # export the learned weights, so that we can later use them in tensor serving and TF Lite print("learned weights {}".format(learning_layer.get_weights())) export_dir = 'learned_model/1' tf.saved_model.save(model, export_dir) converter = tf.lite.TFLiteConverter.from_saved_model(export_dir) tflite_model = converter.convert() tflite_model_file = pl.Path('model.tflite') tflite_model_file.write_bytes(tflite_model) # test whether loading model would work interpreter = tf.lite.Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() to_predict = np.array([[8.0]], dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], to_predict) interpreter.invoke()
max_q = np.max(Q_t1) output = hidden_layers(s_t1.reshape(1, INPUT_SIZE))[0][0] min_q = np.min(output) max_q = np.max(output) left = ((output[2] - min_q) * 255) / (max_q - min_q) if max_q != min_q else 0 nothing = ((output[0] - min_q) * 255) / (max_q - min_q) if max_q != min_q else 0 right = ((output[1] - min_q) * 255) / (max_q - min_q) if max_q != min_q else 0 display[0][0] = (left, 0, 0) display[0][1] = (0, nothing, 0) display[0][2] = (0, 0, right) theta_sum = output_layer.get_weights()[0].sum( axis=0) + output_layer.get_weights()[1] # print(f"Weights: [{theta_sum[2]:.2f}, {theta_sum[0]:.2f}, {theta_sum[1]:.2f}]", # f"hidden_layers[{hidden_layers(s_t1.reshape(1,INPUT_SIZE))}]") display[2][0] = ((theta_sum[2] - theta_sum.min()) * 255 / (theta_sum.max() - theta_sum.min()), 0, 0) display[2][1] = (0, (theta_sum[0] - theta_sum.min()) * 255 / (theta_sum.max() - theta_sum.min()), 0) display[2][2] = (0, 0, (theta_sum[1] - theta_sum.min()) * 255 / (theta_sum.max() - theta_sum.min())) img = Image.fromarray( display, 'RGB' ) # reading to rgb. Apparently. Even tho color definitions are bgr. ??? img = img.resize( (300, 50), Image.NONE ) # resizing so we can see our agent in all its glory.
import tensorflow as tf import numpy as np import matplotlib.pyplot as plt from tensorflow.keras.layers import Dense from tensorflow.keras import Sequential from tensorflow.keras.optimizers import Adam data = np.loadtxt("../../data/cross-entropy.txt", dtype=np.float32) x_data = data[:, 1:3] y_data = data[:, 3:] print(x_data.shape) print(y_data.shape) IO = Dense(units=3, input_shape=[2], activation="cross-entropy") model = Sequential([IO]) model.compile(loss="categorical_crossentropy", optimizer=Adam(learning_rate=0.01), metrics=["accuracy"]) history = model.fit(x_data, y_data, epochs=1000) print(IO.get_weights()) p = model.predict(np.array([[3., 6.]])) print(history.history['acc'][-1])
import matplotlib.pyplot as plt import numpy as np import tensorflow as tf from tensorflow.keras import Sequential from tensorflow.keras.layers import Dense EPOCHS = 500 x = np.array([-1, 0, 1, 2, 3, 4], dtype=float) y = np.array([-3, -1, 1, 3, 5, 7], dtype=float) dense1 = Dense(1, input_shape=[1]) mod = Sequential(dense1) mod.compile(optimizer='sgd', loss='mse') print(mod.summary()) mod.fit(x, y, epochs=EPOCHS, verbose=0) W, b = dense1.get_weights() print(f'Weights:\n W: {W}\n b: {b}') pred = mod.predict([10.]) plt.scatter(x, y) plt.scatter(10., pred, color='r') plt.show()
class SRFR(Model): def __init__( self, num_filters: int = 62, depth: int = 50, categories: int = 512, num_gc: int = 32, num_blocks: int = 23, residual_scailing: float = 0.2, training: bool = True, input_shape=(28, 28, 3), num_classes_syn: int = None, both: bool = False, num_classes_nat: int = None, scale: int = 64, ): super(SRFR, self).__init__() self._training = training self.scale = scale if both: self._natural_input = Conv2D( input_shape=input_shape, filters=num_filters, kernel_size=(3, 3), strides=1, padding='same', name='natural_input', activation=mish, ) self._synthetic_input = Conv2D( input_shape=input_shape, filters=num_filters, kernel_size=(3, 3), strides=1, padding='same', name='synthetic_input', activation=mish, ) self._super_resolution = GeneratorNetwork( num_filters, num_gc, num_blocks, residual_scailing, ) self._face_recognition = ResNet( depth, categories, training ) if self._training: if both: self._fc_classification_nat = Dense( input_shape=(categories,), units=num_classes_nat, activation=None, use_bias=False, dtype='float32', name='fully_connected_to_softmax_crossentropy_nat', ) self._fc_classification_nat.build(tf.TensorShape([None, 512])) self.net_type = 'nat' self._fc_classification_syn: Dense = Dense( input_shape=(categories,), units=num_classes_syn, activation=None, use_bias=False, dtype='float32', name='fully_connected_to_softmax_crossentropy_syn', ) self._fc_classification_syn.build(tf.TensorShape([None, 512])) @tf.function def _call_evaluating(self, input_tensor, input_type: str = 'nat'): if input_type == 'syn': outputs = self._synthetic_input(input_tensor) else: outputs = self._natural_input(input_tensor) super_resolution_image = self._super_resolution(outputs) embeddings = self._face_recognition(super_resolution_image) return super_resolution_image, embeddings def _calculate_normalized_embeddings(self, embeddings, net_type: str = 'syn'): fc_weights = self.get_weights(net_type) normalized_weights = tf.Variable( normalize(fc_weights, name='weights_normalization'), aggregation=tf.VariableAggregation.NONE, ) normalized_embeddings = normalize( embeddings, axis=1, name='embeddings_normalization') * self.scale replica = tf.distribute.get_replica_context() replica.merge_call(self.set_weights, args=(normalized_weights, net_type)) return self.call_fc_classification(normalized_embeddings, net_type) def _call_training(self, synthetic_images, natural_images=None): synthetic_outputs = self._synthetic_input(synthetic_images) synthetic_sr_images = self._super_resolution(synthetic_outputs) synthetic_embeddings = self._face_recognition(synthetic_sr_images) synthetic_embeddings = self._calculate_normalized_embeddings( synthetic_embeddings ) if natural_images: natural_outputs = self._natural_input(natural_images) natural_sr_images = self._super_resolution(natural_outputs) natural_embeddings = self._face_recognition(natural_sr_images) natural_embeddings = self._calculate_normalized_embeddings( natural_embeddings ) return ( synthetic_sr_images, synthetic_embeddings, natural_sr_images, natural_embeddings, ) return synthetic_sr_images, synthetic_embeddings def call(self, input_tensor_01, input_tensor_02=None, training: bool = True, input_type: str = 'nat'): if training: return self._call_training(input_tensor_01, input_tensor_02) return self._call_evaluating(input_tensor_01, input_type) def get_weights(self, net_type: str = 'syn'): if net_type == 'nat': return self._fc_classification_nat.get_weights() return self._fc_classification_syn.get_weights() def set_weights(self, _, weights, net_type: str = 'syn') -> None: if net_type == 'nat': self._fc_classification_nat.set_weights([weights.read_value()]) else: self._fc_classification_syn.set_weights([weights.read_value()]) def call_fc_classification(self, input, net_type: str = 'syn'): if net_type == 'nat': return self._fc_classification_nat(input) return self._fc_classification_syn(input)
import numpy as np from tensorflow.keras import Sequential from tensorflow.keras.layers import Dense l0 = Dense(units=1, input_shape=[1]) model = Sequential([l0]) model.compile(optimizer='sgd', loss='mean_squared_error') xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float) ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float) model.fit(xs, ys, epochs=500) print(model.predict([10.0])) print("Here is what I learned: {}".format(l0.get_weights()))
def make_np_layer_iteration(data: np.array, layer: Dense, activation_fn: Callable) -> np.array: weights = layer.get_weights()[0] biases = layer.get_weights()[1] return activation_fn(np.dot(data, weights) + biases)
class SCHEM(): def __init__(self, run_name, out_features, optim, load_im_size, cs_weight, fe_weight, img_location, crop_size, init_W, end_layer, scale_CS_W=False, l2_norm=False, schsm_sampling='hard', batch_size=60, LR=0.0001, K_constant=5, beta=5, eta=10): """ :param run_name: Allows you to give unique names to each run :param out_features: Number of output features :param optim: Learning rate model optemizer :param load_im_size: Size of image to load :param cs_weight: Coefficent in front of L_c :param fe_weight: Coefficent in front of L_t :param img_location: Location of folder containing .npy files :param crop_size: Paper uses 224 :param init_W: Weight initialization :param K_constant: Same value from paper :param eta: Same value from paper :param beta: Same value from paper :param end_layer: Number of layers to cut off from MobileNetV2 :param scale_CS_W: Boolean to scale CS weights to have a magnitude of 1. :param l2_norm: Boolean that controls if the feature map G has l2-normalization applied :param schsm_sampling: """ self.schsm_sampling_method = schsm_sampling self.l2_norm = l2_norm self.batch_size = batch_size self.scale_sig = scale_CS_W self.init_W = init_W self.end_layer = end_layer self.K_constant = K_constant self.beta = beta self.cs_weight = cs_weight self.fe_weight = fe_weight self.optim = optim self.out_features = out_features self.LR = LR self.init_W = init_W self.model_location = None self.img_location = img_location self.tf_m = tf.constant(self.batch_size * 3.0) self.load_im_size = load_im_size self.crop_size = crop_size self.in_shape = (crop_size, crop_size, 3) self.prepare_data() self.eta = eta self.n_batch_size = batch_size - eta self.run_name = run_name self.name_mod = '(%.1f,%.1f)_el(%i)%s' % (fe_weight, cs_weight, end_layer, self.run_name) self.save_mod = '%s/' % self.name_mod self.results_dir = 'results(%i)' % crop_size self.tsne_out = '%s/plots/tsne/' % self.results_dir + self.save_mod self.loss_out = '%s/plots/loss/' % self.results_dir + self.save_mod self.csv_out = '%s/csv/' % self.results_dir + self.save_mod self.acc_out = '%s/plots/acc/' % self.results_dir + self.save_mod self.acc_out_bin = '%s/plots_bin' % self.results_dir self.model_out = '%s/model_weights/' % self.results_dir + self.save_mod make_dirs([ self.tsne_out, self.loss_out, self.acc_out, self.model_out, self.acc_out_bin ]) def base_network(self): mob_net = tf.keras.applications.MobileNetV2(weights='imagenet', include_top=False, input_shape=self.in_shape) fmg = Layer(name="Feature_map_G_1")( mob_net.layers[-self.end_layer].output) fmg = Conv2D(self.out_features, (1, 1), name='Feature_map_G_2', activation='relu')(fmg) fmg = tf.keras.layers.BatchNormalization(axis=1, trainable=False, name='Feature_map_G_3')(fmg) x = GlobalAveragePooling2D()(fmg) if self.l2_norm: x = Lambda(lambda a: tf.math.l2_normalize(a, axis=1), name='l2_norm')(x) outmodel = Model(inputs=mob_net.input, outputs=x, name='base_FE_network') self.map_G = Model(inputs=mob_net.input, outputs=fmg, name='base_FE_network') return outmodel def create_model_and_compile(self): input_shape = self.in_shape self.base_model = self.base_network() self.cs_layer = Dense(self.num_classes, use_bias=False, name='CS_layer') qpn_in = Input(shape=input_shape, name='in_qpn') qpny = Input(shape=(self.num_classes, ), name='in_qpny') qpny_label = Input(shape=(1, ), name='in_qpny_label') qpn_out = self.base_model(qpn_in) qpn_cls_sig = self.cs_layer(qpn_out) fe_loss = Lambda(self.triplet_loss_all_combinations, name='triplet_loss_FE')(qpn_out) * self.fe_weight fe_accuracy = Lambda(self.FE_accuracy, name='FE_accuracy_metric')(qpn_out) cs_loss = Lambda(self.manual_CS_loss, name='CS_loss_calc')( [qpn_cls_sig, qpny]) * self.cs_weight cs_accuracy = Lambda(self.CS_accuracy, name='CS_Acc')([qpn_cls_sig, qpny]) total_loss = fe_loss + cs_loss model = Model(inputs=[qpn_in, qpny, qpny_label], outputs=[qpn_cls_sig], name='FEModel') if 'adm' in self.optim: optm = Adam(lr=self.LR) elif 'ranger' in self.optim: # option to use a newer optimizer radam = tfa.optimizers.RectifiedAdam(lr=self.LR, min_lr=1e-7) optm = tfa.optimizers.Lookahead(radam, sync_period=6, slow_step_size=0.5) model.add_loss(total_loss) model.compile(optimizer=optm) # Metrics to track the accuracy and loss progression model.add_metric(fe_accuracy, name='fe_a', aggregation='mean') model.add_metric(cs_accuracy, name='cs_a', aggregation='mean') model.add_metric(fe_loss, name='fe_loss', aggregation='mean') model.add_metric(cs_loss, name='cs_loss_out', aggregation='mean') return model, optm def row_wise_subtraction_and_comparison(self, i, qp, n): """ :param i: index to grab from qp :param qp: all QP images :param n: all N images :return: all elements from a specific row of QP subtracted from all elements in n """ # Takes a row and subtracts n from all elemnts i = tf.cast(i, tf.int32) q_elm = tf.gather(qp, i) qp_elm_sub = tf.expand_dims(q_elm, axis=0) - qp qp_elm_mag = tf.math.sqrt( tf.reduce_sum(tf.math.square(qp_elm_sub), axis=1)) n_sub = tf.expand_dims(q_elm, axis=0) - n n_mag = tf.math.sqrt(tf.reduce_sum(tf.math.square(n_sub), axis=1)) return tf.expand_dims(qp_elm_mag, axis=1) - n_mag def triplet_loss_all_combinations(self, args): qp = args[:self.eta, :] n = args[self.eta:, :] all_combinations = tf.map_fn( lambda i: self.row_wise_subtraction_and_comparison(i, qp, n), elems=tf.range(self.eta), dtype=tf.float32) loss = all_combinations + tf.constant(0.2) # Hinge loss margin in 0.2 loss = K.maximum(loss, tf.constant(0.0)) # Using weighting scheme to give more weight to the hard triplets. num_non_zero = tf.math.count_nonzero(loss, dtype=tf.dtypes.float32) l_out = tf.cond(tf.equal(num_non_zero, tf.constant(0.0)), lambda: tf.constant(0.0), lambda: tf.divide(tf.reduce_sum(loss), num_non_zero)) return tf.reduce_mean(loss) def manual_CS_loss(self, args): #Paper implementation of L_c loss cs, qpn_y = args qpn_yf = tf.cast(qpn_y, tf.float32) a = tf.nn.softmax_cross_entropy_with_logits(qpn_yf, cs, axis=1, name='cross_entropy') return tf.reduce_mean(a) def get_cs_row(self, i, qpn_y, cse_out): i = tf.cast(i, tf.int32) curr_class = tf.where(tf.gather(qpn_y, i)) row = tf.gather(cse_out, i) elm = tf.gather(row, curr_class) row_denom = tf.reduce_sum(row) elm_norm = elm / tf.maximum(row_denom, tf.keras.backend.epsilon()) return tf.squeeze(tf.math.log(elm_norm)) def get_class_pool(self, qx, W, qp_idx, alpha): # Calculating P_c from paper W = tf.math.l2_normalize(W, axis=0) qx = tf.math.l2_normalize(qx, axis=1) sim_space = tf.tensordot(qx, W, axes=1) sim_space_max = tf.reduce_max(sim_space, axis=0) closes_class_idx = tf.argsort(sim_space_max, direction='DESCENDING') out = tf.boolean_mask(closes_class_idx, ~tf.equal(closes_class_idx, qp_idx)) return out[:alpha * (self.K_constant - 1)] def CS_accuracy(self, args): qpncs, qpny = args qpt = tf.cast( tf.equal(tf.argmax(qpny, axis=1), tf.argmax(qpncs, axis=1)), tf.float32) return K.mean(qpt) def rand_flip_and_crop(self, batch): batch = tf.map_fn( lambda x: tf.image.random_crop( x, size=[self.crop_size, self.crop_size, 3]), batch) batch = tf.map_fn(tf.image.random_flip_left_right, batch) return batch def center_crop(self, batch): # Used during testing start_point = self.load_im_size - self.crop_size batch = tf.image.crop_to_bounding_box(batch, start_point, start_point, self.crop_size, self.crop_size) return batch def prepare_data(self): x_train = np.load('%s/trnx_CUB_%s.npy' % (self.img_location, self.load_im_size)) y_train = np.load('%s/trny_CUB_%s.npy' % (self.img_location, self.load_im_size)) y_train_OH = np.load('%s/trny_OH_CUB_%s.npy' % (self.img_location, self.load_im_size)) x_test = np.load('%s/tstx_CUB_%s.npy' % (self.img_location, self.load_im_size)) y_test = np.load('%s/tsty_CUB_%s.npy' % (self.img_location, self.load_im_size)) y_train_num_label, y_test_num_label = [], [] for elm in y_train_OH: y_train_num_label.append(list(elm).index(1)) for elm in y_test: y_test_num_label.append(int(elm.split('.')[0])) self.all_c = list(set(y_train_num_label)) self.num_classes = len(self.all_c) y_train_num_label = np.array(y_train_num_label) y_test_num_label = np.array(y_test_num_label) y_train_OH = y_train_OH.astype(bool) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 self.x_train = x_train self.y_train_OH = y_train_OH self.y_train = y_train_num_label self.x_test = x_test self.y_test = y_test_num_label return x_train, y_train_OH, y_train_num_label, x_test, y_test_num_label def generate_G(self, images, mode): # Generate the feature map G. Done in batches due to memeory restrictions if mode == 'test': x_shaped = self.center_crop(images) else: x_shaped = self.rand_flip_and_crop(images) # This split is done due to help wiht memory overflow test_batch_split = np.array_split(np.arange(len(x_shaped)), 15) feats = np.array([]) for t_bat in test_batch_split: tmp = tf.keras.backend.batch_flatten( self.map_G(tf.gather(x_shaped, t_bat))) if tf.size(feats) == 0: feats = tmp else: feats = tf.concat([feats, tmp], axis=0) return feats def generate_out(self, images, mode): # Generate the network output layer that is trained on # Used just for debugging if mode == 'test': x_shaped = self.center_crop(images) else: x_shaped = self.rand_flip_and_crop(images) tmp = 15 if self.load_im_size == 256: tmp = 50 test_batch_split = np.array_split(np.arange(len(x_shaped)), tmp) feats = np.array([]) for t_bat in test_batch_split: tmp = tf.keras.backend.batch_flatten( self.base_model(tf.gather(x_shaped, t_bat))) if tf.size(feats) == 0: feats = tmp else: feats = tf.concat([feats, tmp], axis=0) return feats def get_batch(self): # x_train, y_train, y_train_OH qp_idx = self.q_idx xqp_inds = np.where(self.y_train_OH[:, qp_idx]) all_Xqp = self.x_train[xqp_inds] all_qpy_label = self.y_train[xqp_inds] all_qpy = self.y_train_OH[xqp_inds] batch_qp_idx = np.random.choice(np.arange(len(all_Xqp)), self.eta, replace=False) batch_qp = all_Xqp[batch_qp_idx] batch_qp = self.rand_flip_and_crop(batch_qp) batch_qpy = all_qpy[batch_qp_idx] batch_qpy_label = all_qpy_label[batch_qp_idx] return batch_qp, batch_qpy, batch_qpy_label def feature_map_G_statistics(self): xd = self.x_test y_labels = self.y_test g_feats = self.generate_G(xd, mode='test') feats1 = tf.math.l2_normalize(g_feats, axis=1) sim_matrix = tf.tensordot(feats1, tf.transpose(feats1), axes=1) closest_neighbor = tf.argsort(sim_matrix, direction='DESCENDING', axis=1)[:, 1] # k = 1 here grab_class = tf.gather(y_labels, closest_neighbor) acc = tf.cast(tf.math.equal(y_labels, grab_class), tf.float32) acc1 = tf.reduce_mean(acc) ###################### compare with output R@1 out_feats = self.generate_out(xd, mode='test') feats1 = tf.math.l2_normalize(out_feats, axis=1) sim_matrix = tf.tensordot(feats1, tf.transpose(feats1), axes=1) closest_neighbor = tf.argsort(sim_matrix, direction='DESCENDING', axis=1)[:, 1] # k = 1 here grab_class = tf.gather(y_labels, closest_neighbor) acc = tf.cast(tf.math.equal(y_labels, grab_class), tf.float32) acc5 = tf.reduce_mean(acc) return acc1.numpy(), acc5.numpy() def generate_all_statistics(self): xd = self.x_test y_labels = self.y_test feats = self.generate_G(xd, mode='test') feats1 = tf.math.l2_normalize(feats, axis=1) sim_matrix = tf.tensordot(feats1, tf.transpose(feats1), axes=1) # doing [:, 1] to remove the first column which is the correlation with itself closest_neighbor = tf.argsort(sim_matrix, direction='DESCENDING', axis=1)[:, 1] grab_class = tf.gather(y_labels, closest_neighbor) acc = tf.reduce_mean( tf.cast(tf.math.equal(y_labels, grab_class), tf.float32)) return acc.numpy() def FE_accuracy(self, args): qp = args[:self.eta, :] n = args[self.eta:, :] indicator = tf.map_fn( lambda i: self.row_wise_subtraction_and_comparison(i, qp, n), elems=tf.range(self.eta), dtype=tf.float32) comparison = tf.cast(tf.math.greater(tf.constant(0.0), indicator), tf.float32) return tf.reduce_mean(comparison) def schem_sample(self, batch_qp): x_train = self.x_train y_train = self.y_train y_train_OH = self.y_train_OH if 'hard' in self.schsm_sampling_method: # Hard sampling qp_features = tf.keras.backend.batch_flatten( self.base_model(batch_qp)) qp_features = tf.math.l2_normalize(qp_features, axis=1) Pc = self.get_class_pool(qp_features, self.cs_layer.get_weights()[0], self.q_idx, self.alpha) n_indexes = [] # Could automate this in a tensor, but probably don't need to for elm in Pc: indexes_for_class = np.where(y_train == elm)[0] n_indexes += list(indexes_for_class) # Generate feature space for all N elements. Ps_all = x_train[n_indexes] n_all_y = y_train_OH[n_indexes] n_all_y_label = y_train[n_indexes] Ps_features = self.generate_out(Ps_all, mode='train') Ps_features = tf.math.l2_normalize(Ps_features, axis=1) similarity = tf.tensordot(qp_features, tf.transpose(Ps_features), axes=1) # Take top similarities from all q-inputs. max_sim = tf.reduce_max(similarity, axis=0) max_sum = tf.reduce_sum(similarity, axis=0) similar_indicies_sorted = tf.argsort(max_sum, axis=0, direction='DESCENDING') top_similar_indicies = tf.argsort( max_sim, axis=0, direction='DESCENDING')[:self.beta * self.n_batch_size] n_idx = np.random.choice(top_similar_indicies, self.n_batch_size) n_bat = tf.gather(Ps_all, n_idx) n_bat = self.rand_flip_and_crop( n_bat) # maybe i should crop before and not randomly re-crop n_y = tf.gather(n_all_y, n_idx) n_y_label = tf.gather(n_all_y_label, n_idx) else: # Naive sampling n_idxs = np.random.choice( np.setdiff1d(np.arange(self.num_classes), self.q_idx), 10) n_bat = np.array([]) n_y = np.array([]) n_y_label = np.array([]) for n_idx in n_idxs: tot_elms = sum(y_train_OH[:, n_idx]) xqp_inds = np.where(y_train_OH[:, n_idx]) all_Xn = x_train[xqp_inds] all_yn = y_train_OH[xqp_inds] all_yn_label = y_train[xqp_inds] b_n_idxs = np.random.choice(np.arange(tot_elms), 2, replace=False) if n_bat.size == 0: n_bat = all_Xn[b_n_idxs] n_y = all_yn[b_n_idxs] n_y_label = all_yn_label[b_n_idxs] else: n_bat = np.append(n_bat, all_Xn[b_n_idxs], axis=0) n_y = np.append(n_y, all_yn[b_n_idxs], axis=0) n_y_label = np.append(n_y_label, all_yn_label[b_n_idxs], axis=0) n_bat = self.rand_flip_and_crop(n_bat) return n_bat, n_y, n_y_label
import tensorflow as tf import numpy as np from tensorflow.keras import Sequential from tensorflow.keras.layers import Dense L0 = Dense(units=1, input_shape=[1]) model = Sequential([L0]) model.compile(optimizer='sgd', loss='mean_squared_error') xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float) ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float) model.fit(xs, ys, epochs=500) print(model.predict([10.0])) print("Here is what i've learned: {}".format(L0.get_weights())) export_dir = 'saved_model/1' tf.saved_model.save(model, export_dir)
# - # ## Model # 1 layer_0 = Dense(units=1, input_shape=[1]) model = Sequential() model.add(layer_0) model.compile(loss='mean_squared_error', optimizer=tf.keras.optimizers.Adam(0.1)) model.summary() # + id="ay8_Sd4MShDO" history = model.fit(x=celsius_q, y=fahrenheit_a, epochs=500, verbose=False) # + colab={"base_uri": "https://localhost:8080/", "height": 296} id="I7pas988SOjO" outputId="2758d191-7b14-4afd-803a-90813ad5c553" plt.xlabel('Epoch') plt.ylabel("Loss") plt.plot(history.history['loss']) # - # ### Prediction # + colab={"base_uri": "https://localhost:8080/"} id="SyZ6kaQ6SQd2" outputId="e84ede3c-b3b1-46f5-b90d-ee4c3581d8a2" print(model.predict([100.0])) # + colab={"base_uri": "https://localhost:8080/"} id="elWiukimSSKe" outputId="ecb17893-07e6-4ca1-b0f8-b8a746e4d480" print("These are the layer variables: " + str(layer_0.get_weights())) # Note the formula is F = 1.8 * C + 32 , where weight is 1.8 and bias is 32 # + print("End of the code")
class GAPAlexnet(tf.keras.Model): def __init__(self, num_classes, keep_prob): super(GAPAlexnet, self).__init__() # Possibly experiment - different initializations # TODO - regularization? see paper self.num_classes = num_classes self.keep_prob = keep_prob self.conv1 = Conv2D(96, 11, strides=(4,4), activation='relu') self.pool1 = MaxPool2D(pool_size=(3,3), strides=(2,2)) self.pad = ZeroPadding2D(padding=(2,2)) self.conv2 = Conv2D(256, 5, activation='relu') self.pool2 = MaxPool2D(pool_size=(3,3), strides=(2,2)) self.conv3 = Conv2D(384, 3, padding='same', activation='relu') self.conv4 = Conv2D(384, 3, padding='same', activation='relu') self.conv5 = Conv2D(256, 3, padding='same', activation='relu') self.pool3 = MaxPool2D(pool_size=(3,3), strides=(2,2)) self.gap = GlobalAveragePooling2D() # self.flat = Flatten() self.drop_gap = Dropout(rate=self.keep_prob) self.fc8 = Dense(units=self.num_classes, use_bias=False) self.bn1 = BatchNormalization(axis=-1) self.bn2 = BatchNormalization(axis=-1) self.bn3 = BatchNormalization(axis=-1) self.bn4 = BatchNormalization(axis=-1) self.bn5 = BatchNormalization(axis=-1) def update_last_layer(self, num_classes): self.num_classes = num_classes self.fc8 = Dense(units=self.num_classes, use_bias=False) # Input - datum[0], datum[1] or datum[2], datum[3] def call(self, inputImg, mode='train'): # o1 = lrn(self.pool1(self.conv1(inputImg)), 2, 2e-05, 0.75) # o2 = lrn((self.pool2(self.conv2(self.pad(o1)))), 2, 2e-05, 0.75) train_bool = False if(mode == 'train'): train_bool = True o1 = lrn(self.pool1(self.conv1(inputImg)), 2, 2e-05, 0.75) o2 = lrn(self.pool2(self.conv2(self.pad(o1))), 2, 2e-05, 0.75) o3 = self.conv3(o2) # o1 = self.pool1(self.bn1(self.conv1(inputImg), training=train_bool)) # o2 = self.pool2(self.bn2(self.conv2(self.pad(o1)), training=train_bool) ) o3 = (self.conv3(o2)) o4 = (self.conv4(o3)) o5 = (self.conv4(o4)) # o3 = self.bn3(self.conv3(o2), training=train_bool) # o4 = self.bn4(self.conv4(o3), training=train_bool) # o5 = self.bn5(self.conv5(o4), training=train_bool) pooled_o5 = self.pool3(o5) gap_out = self.gap(pooled_o5) logits = self.fc8(gap_out) if(mode == 'train' or mode == 'eval'): return logits elif(mode == 'maps'): # return tf.reduce_mean(o5, axis=-1) pred_class = (tf.argmax(logits, axis=-1)) np_wts = tf.convert_to_tensor(self.fc8.get_weights()[0]) # print("SHAPE", np_wts.shape) np_wts = tf.transpose(np_wts) # print("TRANSPOSE SHAPE", np_wts.shape) # weights = tf.gather(tf.transpose(tf.convert_to_tensor(self.fc8.get_weights()[0])) , pred_class) weights = tf.gather(np_wts , pred_class) # print("FINAL SHAPE", weights.shape) # exit() # weights = tf.convert_to_tensor(self.fc8.get_weights()[0][:, pred_class]) weights = tf.expand_dims(weights, axis=-1) OMAP = pooled_o5 bsize, xdim, ydim, c = tf.shape(OMAP)[0], tf.shape(OMAP)[1], tf.shape(OMAP)[2], tf.shape(OMAP)[3] OMAP = tf.reshape(OMAP, [bsize, xdim*ydim, c]) cam_map = tf.squeeze(tf.matmul(OMAP, weights), [2]) cam_map = tf.reshape(cam_map, [bsize, xdim, ydim]) return cam_map, pred_class else: assert(False)
class ResnetGenerator(tf.keras.layers.Layer): def __init__(self, input_nc, output_nc, ngf=64, n_blocks=6, img_size=256, light=False): super(ResnetGenerator, self).__init__() self.input_nc = input_nc self.output_nc = output_nc self.ngf = ngf self.n_blocks = n_blocks self.img_size = img_size self.light = light DownBlock = [ ReflectionPad2D(3), Conv2D(filters=ngf, kernel_size=7, strides=1, padding='valid', use_bias=False), InstanceNormalization(axis=3), ReLU() ] n_downsampling = 2 for i in range(n_downsampling): mult = 2**i DownBlock += [ ReflectionPad2D(1), Conv2D(filters=ngf * mult * 2, kernel_size=3, strides=2, padding='valid', use_bias=False), InstanceNormalization(axis=3), ReLU() ] mult = 2**n_downsampling for i in range(n_blocks): DownBlock += [ResnetBlock(ngf * mult, use_bias=False)] self.gap_fc = Dense(1, use_bias=False) self.gmp_fc = Dense(1, use_bias=False) self.conv1x1 = Conv2D(filters=ngf * mult, kernel_size=1, strides=1, use_bias=False) self.relu = ReLU() FC = [ Dense(ngf * mult, use_bias=False), ReLU(), Dense(ngf * mult, use_bias=False), ReLU() ] self.gamma = Dense(ngf * mult, use_bias=False) self.beta = Dense(ngf * mult, use_bias=False) for i in range(n_blocks): setattr(self, 'UpBlock1_' + str(i + 1), ResnetAdaILNBlock(ngf * mult, use_bias=False)) UpBlock2 = [] for i in range(n_downsampling): mult = 2**(n_downsampling - i) UpBlock2 += [ UpSampling2D(size=(2, 2), interpolation='nearest'), ReflectionPad2D(1), Conv2D(filters=int(ngf * mult / 2), kernel_size=3, strides=1, padding='valid', use_bias=False), ILN(int(ngf * mult / 2)), ReLU() ] UpBlock2 += [ ReflectionPad2D(3), Conv2D(filters=output_nc, kernel_size=7, strides=1, padding='valid', use_bias=False), Tanh() ] self.DownBlock = tf.keras.Sequential(DownBlock) self.FC = tf.keras.Sequential(FC) self.UpBlock2 = tf.keras.Sequential(UpBlock2) def call(self, inputs): x = self.DownBlock(inputs) gap = tf.reduce_mean(x, axis=[1, 2], keepdims=True) gap_logit = self.gap_fc(tf.reshape(gap, shape=[x.shape[0], -1])) gap_weight = tf.transpose(self.gap_fc.get_weights()[0], perm=[1, 0]) gap = x * tf.expand_dims(tf.expand_dims(gap_weight, axis=1), axis=2) gmp = tf.reduce_max(x, axis=[1, 2], keepdims=True) gmp_logit = self.gmp_fc(tf.reshape(gmp, shape=[x.shape[0], -1])) gmp_weight = tf.transpose(self.gmp_fc.get_weights()[0], perm=[1, 0]) gmp = x * tf.expand_dims(tf.expand_dims(gmp_weight, axis=1), axis=2) cam_logit = tf.concat([gap_logit, gmp_logit], 1) x = tf.concat([gap, gmp], 3) x = self.relu(self.conv1x1(x)) heatmap = tf.reduce_sum(x, axis=[3], keepdims=True) if self.light: x_ = tf.reduce_mean(x, axis=[1, 2], keepdims=True) x_ = self.FC(tf.reshape(x_, shape=[x_.shape[0], -1])) else: x_ = self.FC(tf.reshape(x_, shape=[x.shape[0], -1])) gamma, beta = self.gamma(x_), self.beta(x_) for i in range(self.n_blocks): x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta) out = self.UpBlock2(x) return out, cam_logit, heatmap
def make_naive_layer_iteration(data: np.array, layer: Dense, activation_fn: Callable) -> np.array: weights = layer.get_weights()[0] biases = layer.get_weights()[1] return activation_fn(naive_add_matrix_and_vector(naive_matrix_matrix_dot(data, weights), biases))
class FactorizedSeasonality(Seasonality): def __init__(self, n_time_series, n_dim=1, seasonality_type="weekday", l2_reg=1e-3): super().__init__(seasonality_type) self.n_time_series = n_time_series self.n_dim = n_dim self.l2_reg = l2_reg self._embedding = None self._inputs = None self._encoder = None self._decoder = None self._create_embedding() self._create_inputs() self._create_decoder() self._create_encoder() def _create_embedding(self): # First embedding for time series ID self._embedding = Embedding( self.n_time_series, output_dim=self.n_dim, name="seas_emb_%s" % self.seasonality_type, embeddings_initializer=constant(0), embeddings_regularizer=l1_l2( l2=self.l2_reg) if self.l2_reg else None) self._seasonality_weights = Dense(self.seasonal_period, activation='linear') def _create_inputs(self): self._inputs = [ Input(shape=(1, ), name="timeseries_id_%s" % self.seasonality_type), Input(shape=(None, ), name="inp_X_seas_%s" % self.seasonality_type, dtype=tf.int32), Input(shape=(None, ), name="inp_Y_seas_%s" % self.seasonality_type, dtype=tf.int32) ] def _create_decoder(self): id_emb = self._embedding(self._inputs[0])[:, 0, :] id_seas_values = self._seasonality_weights(id_emb) seas_values = tf.gather(id_seas_values, self._inputs[1], axis=1, batch_dims=-1) self._decoder = seas_values[:, :, None] + 1 def _create_encoder(self): id_emb = self._embedding(self._inputs[0])[:, 0, :] id_seas_values = self._seasonality_weights(id_emb) seas_values = tf.gather(id_seas_values, self._inputs[2], axis=1, batch_dims=-1) self._encoder = seas_values[:, :, None] + 1 def get_fit_inputs(self, ids, x): return [ids, x[:, :-1], x] def get_predict_inputs(self, ids, x, last_date, increment): # Expand x by one for predicting one step ahead x_expanded = np.hstack([x, self._get_next_x(last_date, increment, 1)]) return [ids, x, x_expanded] def get_oos_predictions(self, first_oos_prediction, last_date, increment, n_oos_steps): x_oos = self._get_next_x(last_date, increment, n_oos_steps) id_seas_values = self.get_weights() deseason = np.take_along_axis(id_seas_values, x_oos[:, :1], axis=1) oos_season = np.take_along_axis(id_seas_values, x_oos[:, 1:], axis=1) return first_oos_prediction * oos_season / deseason def get_weights(self): return self._embedding.get_weights()[0] @ self._seasonality_weights.get_weights()[0] + \ self._seasonality_weights.get_weights()[1] + 1
#sys.exit() model.fit(x=X_train_train, y=y_labels_train_train, shuffle=True, epochs=epochs, batch_size=batch_size, callbacks=[ EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True) ], validation_data=(X_train_val, y_labels_train_val)) new_weights = np.empty([100, 37]) biases = outLayer.get_weights()[1] for i in range(len(outLayer.get_weights()[0])): weights = outLayer.get_weights()[0][i] for j in range(len(weights)): if j not in (32, 33, 34, 35, 36): #subtypes weights[j] = 0 new_weights[i] = weights outLayer.set_weights([new_weights, biases]) score = model.evaluate(X_test, y_labels_test) print(model.predict(X_test).argmax(axis=1)) conf_matrix = pd.DataFrame( confusion_matrix(y_labels_test.values.argmax(axis=1), model.predict(X_test).argmax(axis=1)))
layer_1 = Dense(units=4) layer_2 = Dense(units=1) model = Sequential() model.add(layer_0) model.add(layer_1) model.add(layer_2) model.compile(loss='mean_squared_error', optimizer=tf.keras.optimizers.Adam(0.1)) model.summary() # Layer 0 : weights 1 x 4 and bias 4 ; Layer 1 : weights 4 x 4 and bias 4 ; Layer 3 : weights 4 x 1 and bias = 1 history = model.fit(x=celsius_q, y=fahrenheit_a, epochs=500, verbose=False) # Check any change in the loss when more layers are added compared to one layer in the last ex? plt.xlabel('Epoch') plt.ylabel("Loss") plt.plot(history.history['loss']) # # Predictions print(model.predict([100.0])) # Note the formula is F = 1.8 * C + 32 , where weight is 1.8 and bias is 32 if only one layer and one unit print("These are the layer_0 variables: " + str(layer_0.get_weights()) + "\n") print("These are the layer_1 variables: " + str(layer_1.get_weights()) + "\n") print("These are the layer_2 variables: " + str(layer_2.get_weights()) + "\n")
class CNNLayerwise(tf.keras.Model): def __init__(self, hyperp, run_options, data_input_shape, label_dimensions, num_channels, kernel_regularizer, bias_regularizer): super(CNNLayerwise, self).__init__() ############################################################################### # Construct Initial Neural Network Architecture # ############################################################################### #=== Defining Attributes ===# self.data_input_shape = data_input_shape self.architecture = [] # storage for layer information, each entry is [filter_size, num_filters] self.num_filters = hyperp.num_filters self.kernel_size = hyperp.filter_size self.activation = hyperp.activation self.kernel_regularizer = kernel_regularizer self.bias_regularizer = bias_regularizer self.hidden_layers_list = [] # This will be a list of Keras layers #=== Define Initial Architecture and Create Layer Storage ===# self.architecture.append([self.data_input_shape[0], num_channels]) # input information self.architecture.append([3, self.num_filters]) # 3x3 convolutional layer for upsampling data self.architecture.append([hyperp.filter_size, self.num_filters]) # First hidden layer self.architecture.append([3, num_channels]) # 3x3 convolutional layer for downsampling features self.architecture.append(label_dimensions) # fully-connected output layer print(self.architecture) #=== Weights and Biases Initializer ===# kernel_initializer = RandomNormal(mean=0.0, stddev=0.05) bias_initializer = 'zeros' #=== Linear Upsampling Layer to Map to Feature Space ===# l = 1 self.upsampling_layer = Conv2D(self.architecture[l][1], (3, 3), padding = 'same', activation = 'linear', use_bias = True, input_shape = self.data_input_shape, kernel_initializer = kernel_initializer, bias_initializer = bias_initializer, kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer, name='upsampling_layer') #=== Define Hidden Layers ===# l = 2 conv_layer = Conv2D(self.architecture[l][1], (self.architecture[l][0], self.architecture[l][0]), padding = 'same', activation = self.activation, use_bias = True, input_shape = (None, self.data_input_shape[0], self.data_input_shape[1], self.num_filters), kernel_initializer = kernel_initializer, bias_initializer = bias_initializer, kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer, name = "W" + str(l)) self.hidden_layers_list.append(conv_layer) #=== Linear Downsampling Layer to Map to Data Space ===# l = 3 self.downsampling_layer = Conv2D(self.architecture[l][1], (3, 3), padding = 'same', activation = "linear", use_bias = True, input_shape = (None, self.data_input_shape[0], self.data_input_shape[1], self.num_filters), kernel_initializer = kernel_initializer, bias_initializer = bias_initializer, kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer, name = "downsampling_layer") #=== Classification Layer ===# self.classification_layer = Dense(units = label_dimensions, activation = 'linear', use_bias = True, kernel_initializer = kernel_initializer, bias_initializer = bias_initializer, kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer, name = 'classification_layer') ############################################################################### # Network Propagation # ############################################################################### def call(self, inputs): #=== Upsampling ===# output = self.upsampling_layer(inputs) for hidden_layer in self.hidden_layers_list: #=== Hidden Layers ===# prev_output = output output = prev_output + hidden_layer(output) #=== Downsampling ===# output = self.downsampling_layer(output) #=== Classification ===# output = Flatten()(output) output = self.classification_layer(output) return output ############################################################################### # Add Layer # ############################################################################### def add_layer(self, trainable_hidden_layer_index, freeze = True, add = True): kernel_initializer = 'zeros' bias_initializer = 'zeros' if add: conv_layer = Conv2D(self.num_filters, (self.kernel_size, self.kernel_size), padding = 'same', activation = self.activation, use_bias = True, input_shape = (None, self.data_input_shape[0], self.data_input_shape[1], self.num_filters), kernel_initializer = kernel_initializer, bias_initializer = bias_initializer, kernel_regularizer = self.kernel_regularizer, bias_regularizer = self.bias_regularizer, name = "W" + str(trainable_hidden_layer_index)) self.hidden_layers_list.append(conv_layer) if freeze: self.upsampling_layer.trainable = False for index in range(0, trainable_hidden_layer_index-2): self.hidden_layers_list[index].trainable = False else: self.upsampling_layer.trainable = True for index in range(0, trainable_hidden_layer_index-2): self.hidden_layers_list[index].trainable = True ############################################################################### # Sparsify Weights # ############################################################################### def sparsify_weights_and_get_relative_number_of_zeros(self, threshold = 1e-6): #=== Downsampling Layer ===# down_weights = self.downsampling_layer.get_weights() sparsified_weights = self.sparsify_weights(down_weights, threshold) self.downsampling_layer.set_weights(sparsified_weights) #=== Classification Layer ===# class_weights = self.classification_layer.get_weights() sparsified_weights = self.sparsify_weights(class_weights, threshold) self.classification_layer.set_weights(sparsified_weights) #=== Trained Hidden Layer ===# trained_weights = self.hidden_layers_list[-1].get_weights() sparsified_weights = self.sparsify_weights(trained_weights, threshold) self.hidden_layers_list[-1].set_weights(sparsified_weights) #=== Compute Relative Number of Zeros ===# total_number_of_zeros = 0 total_number_of_elements = 0 for i in range(0, len(sparsified_weights)): total_number_of_zeros += np.count_nonzero(sparsified_weights[i]==0) total_number_of_elements += sparsified_weights[i].flatten().shape[0] relative_number_zeros = np.float64(total_number_of_zeros/total_number_of_elements) return relative_number_zeros def sparsify_weights(self, weights, threshold = 1e-6): sparsified_weights = [] if isinstance(weights, float): if abs(weights) > threshold: sparsified_weights = weights else: sparsified_weights = 0 else: for w in weights: bool_mask = (abs(w) > threshold).astype(int) sparsified_weights.append(w*bool_mask) return sparsified_weights
class MyModel(Model): def __init__(self): super(MyModel, self).__init__() self.conv1 = Conv2D(32, 3, activation='relu', autocast=False, dtype=tf.float32) self.flatten = Flatten(autocast=False) self.d1 = Dense(128, activation='relu', autocast=False, dtype=tf.float32) self.d2 = Dense(10, activation='softmax', autocast=False, dtype=tf.float32) self.loss_object = tf.keras.losses.SparseCategoricalCrossentropy() #自定义优化器 self.optimizer = tf.keras.optimizers.SGD() self.train_loss = tf.keras.metrics.Mean(name='train_loss') self.train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( name='train_accuracy') self.test_loss = tf.keras.metrics.Mean(name='test_loss') self.test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( name='test_accuracy') # 训练步骤 @tf.function def train_step_(self, images, labels): with tf.GradientTape() as tape: predictions = self(images) loss = self.loss_object(labels, predictions) gradients = tape.gradient(loss, self.trainable_variables) self.optimizer.apply_gradients(zip(gradients, self.trainable_variables)) self.train_loss(loss) self.train_accuracy(labels, predictions) # 测试步骤 @tf.function def test_step_(self, images, labels): predictions = self(images) t_loss = self.loss_object(labels, predictions) self.test_loss(t_loss) self.test_accuracy(labels, predictions) def test(self): self.test_loss.reset_states() self.test_accuracy.reset_states() for test_images, test_labels in self.test_ds: self.test_step_(test_images, test_labels) template = 'Test Loss: {}, Test Accuracy: {}' print( template.format(self.test_loss.result(), self.test_accuracy.result() * 100)) return self.test_accuracy.result() * 100 # 具体的计算call def call(self, x): x = self.conv1(x) x = self.flatten(x) x = self.d1(x) return self.d2(x) # 训练 def train_test(self, epoch_data_weight, index): for epoch in range(epoch_data_weight): # 在下一个epoch开始时,重置评估指标 self.train_loss.reset_states() self.train_accuracy.reset_states() # self.test_loss.reset_states() # self.test_accuracy.reset_states() for images, labels in self.train_ds: self.train_step_(images, labels) # 先去掉测试集 # for test_images, test_labels in self.test_ds: # self.test_step_(test_images, test_labels) template = 'index {} Epoch {}, Loss: {}, Accuracy: {}' print( template.format(index, epoch + 1, self.train_loss.result(), self.train_accuracy.result() * 100)) def get_weights(self): ''' :return: data_listq ''' weight = [] weight.append(self.conv1.get_weights()) weight.append(self.d1.get_weights()) weight.append(self.d2.get_weights()) return weight # 对应每层的权重设置 def set_weights(self, weight): self.conv1.set_weights(weights=weight[0]) self.d1.set_weights(weights=weight[1]) self.d2.set_weights(weights=weight[2]) # 为每个模型设置数据集 def set_data(self, train_ds, test_ds=None): self.train_ds = train_ds self.test_ds = test_ds