def on_epoch_end(self, epoch, logs=None): self.losses.append(logs.get('loss')) self.num_epochs = epoch self.losses.append(logs.get('loss')) if epoch % 6 != 0: return # return intermediate_layer_model = Model( inputs=self.model.input, outputs=self.model.get_layer('abundances').output) abundances = intermediate_layer_model.predict(self.input) endmembers = self.model.layers[len(self.model.layers) - 1].get_weights()[0] plotHist(self.losses, 33) if self.plotGT: dict = order_endmembers(endmembers, self.endmembersGT) # if self.is_GT_for_A: # plotAbundances(self.num_endmembers, self.size_data, abundances, self.abundancesGT, dict, self.use_ASC) # else: # plotAbundances(self.num_endmembers, self.size_data, abundances, None, None, self.use_ASC, is_GT=False) plotEndmembersAndGT(endmembers, self.endmembersGT, dict) # plotAbundancesSimple(self.num_endmembers, self.size_data, abundances, dict, use_ASC=1, figure_nr=10) else: # plotAbundances(self.num_endmembers, self.size_data, abundances, None, None, self.use_ASC) plotEndmembers(self.num_endmembers, endmembers) print(K.eval(self.model._collected_trainable_weights[-3])) return
def on_epoch_end(self, epoch, logs=None): self.losses.append(logs.get('loss')) self.num_epochs = epoch self.losses.append(logs.get('loss')) if epoch % self.plot_every_n != 0: return return if self.plotS: intermediate_layer_model = Model( inputs=self.model.input, outputs=self.model.get_layer('abundances').output) abundances = intermediate_layer_model.predict(self.input) if self.size is None: self.size = (int(np.sqrt(abundances.shape[0])), int(np.sqrt(abundances.shape[0]))) endmembers = self.model.layers[len(self.model.layers) - 1].get_weights()[0] # plotHist(self.losses, 33) self.plotGT = True if self.plotGT: dict = order_endmembers(endmembers, self.endmembersGT) # if self.is_GT_for_A: # plotAbundances(self.num_endmembers, self.size_data, abundances, self.abundancesGT, dict, self.use_ASC) # else: # plotAbundances(self.num_endmembers, self.size_data, abundances, None, None, self.use_ASC, is_GT=False) plotEndmembersAndGT(self.endmembersGT, endmembers, dict) if self.plotS: plotAbundancesSimple(self.num_endmembers, (self.size[0], self.size[1]), abundances, dict, use_ASC=1, figure_nr=10) else: # plotAbundances(self.num_endmembers, self.size_data, abundances, None, None, self.use_ASC) plotEndmembers(self.num_endmembers, endmembers) plotAbundancesSimple(self.num_endmembers, (self.size[0], self.size[1]), abundances, dict=None, use_ASC=1, figure_nr=10) return
class VGG19Features(object): def __init__( self, session, feature_layers=None, feature_weights=None, gram_weights=None, default_gram=0.1, original_scale=False, eager=False, ): if eager: pass else: K.set_session(session) self.base_model = VGG19(include_top=False, weights="imagenet") if feature_layers is None: feature_layers = [ "input_1", "block1_conv2", "block2_conv2", "block3_conv2", "block4_conv2", "block5_conv2", ] self.layer_names = [l.name for l in self.base_model.layers] for k in feature_layers: if not k in self.layer_names: raise KeyError("Invalid layer {}. Available layers: {}".format( k, self.layer_names)) self.feature_layers = feature_layers features = [ self.base_model.get_layer(k).output for k in feature_layers ] self.model = Model(inputs=self.base_model.input, outputs=features) if feature_weights is None: feature_weights = len(feature_layers) * [1.0] if gram_weights is None: gram_weights = len(feature_layers) * [default_gram] elif isinstance(gram_weights, (int, float)): gram_weights = len(feature_layers) * [gram_weights] self.feature_weights = feature_weights self.gram_weights = gram_weights assert len(self.feature_weights) == len(features) self.use_gram = np.max(self.gram_weights) > 0.0 self.original_scale = original_scale self.variables = self.base_model.weights def extract_features(self, x): """x should be rgb in [-1,1].""" x = self.preprocess_input(x) features = self.model.predict(x) return features def make_features_op(self, x): """y should be rgb tensors in [-1, 1]. This function is just used for testing""" if self.original_scale: x = tf.image.resize_bilinear(x, [256, 256]) bs = tf.shape(x)[0] # x = tf.random_crop(xy, [bs, 224, 224, 3]) x = x[:, 16:239, 16:239, :] x = self.preprocess_input(x) x_features = self.model(x) return x_features def grams(self, fs): gs = list() for f in fs: bs, h, w, c = f.shape.as_list() bs = -1 if bs is None else bs f = tf.reshape(f, [bs, h * w, c]) ft = tf.transpose(f, [0, 2, 1]) g = tf.matmul(ft, f) g = g / (4.0 * h * w) gs.append(g) return gs def make_loss_op(self, x, y): """x, y should be rgb tensors in [-1,1]. Uses l1 and spatial average.""" if self.original_scale: xy = tf.concat([x, y], axis=0) xy = tf.image.resize_bilinear(xy, [256, 256]) bs = tf.shape(xy)[0] xy = tf.random_crop(xy, [bs, 224, 224, 3]) x, y = tf.split(xy, 2, 0) x = self.preprocess_input(x) x_features = self.model(x) y = self.preprocess_input(y) y_features = self.model(y) x_grams = self.grams(x_features) y_grams = self.grams(y_features) losses = [ tf.reduce_mean(tf.abs(xf - yf)) for xf, yf in zip(x_features, y_features) ] gram_losses = [ tf.reduce_mean(tf.abs(xg - yg)) for xg, yg in zip(x_grams, y_grams) ] for i in range(len(losses)): losses[i] = self.feature_weights[i] * losses[i] gram_losses[i] = self.gram_weights[i] * gram_losses[i] loss = tf.add_n(losses) if self.use_gram: loss = loss + tf.add_n(gram_losses) self.losses = losses self.gram_losses = gram_losses return loss def make_nll_op(self, x, y, log_variances, gram_log_variances=None, calibrate=True): """x, y should be rgb tensors in [-1,1]. This version treats every layer independently.""" use_gram = gram_log_variances is not None if self.original_scale: xy = tf.concat([x, y], axis=0) xy = tf.image.resize_bilinear(xy, [256, 256]) bs = tf.shape(xy)[0] xy = tf.random_crop(xy, [bs, 224, 224, 3]) x, y = tf.split(xy, 2, 0) x = self.preprocess_input(x) x_features = self.model(x) y = self.preprocess_input(y) y_features = self.model(y) if use_gram: x_grams = self.grams(x_features) y_grams = self.grams(y_features) if len(log_variances) == 1: log_variances = len(x_features) * [log_variances[0]] feature_ops = [ _ll_loss(xf, yf, logvar, calibrate=calibrate) for xf, yf, logvar in zip(x_features, y_features, log_variances) ] losses = [f[0] for f in feature_ops] self.losses = losses calibrations = [f[1] for f in feature_ops] self.calibrations = calibrations if use_gram: gram_ops = [ _ll_loss(xg, yg, glogvar) for xg, yg, glogvar in zip( x_grams, y_grams, gram_log_variances) ] gram_losses = [g[0] for g in gram_ops] self.gram_losses = gram_losses gram_calibrations = [g[1] for g in gram_ops] self.gram_calibrations = gram_calibrations loss = tf.add_n(losses) if use_gram: loss = loss + tf.add_n(gram_losses) return loss def make_l1_nll_op(self, x, y, log_variance): """x, y should be rgb tensors in [-1,1]. Uses make_loss_op to compute version compatible with previous experiments.""" rec_loss = 1e-3 * self.make_loss_op(x, y) dim = np.prod(x.shape.as_list()[1:]) log_gamma = log_variance gamma = tf.exp(log_gamma) log2pi = np.log(2.0 * np.pi) likelihood = 0.5 * dim * (rec_loss / gamma + log_gamma + log2pi) return likelihood def make_style_op(self, x, y): __feature_weights = self.feature_weights __gram_weights = self.gram_weights self.feature_weights = [0.01 for _ in __feature_weights] self.gram_weights = [1.0 for _ in __gram_weights] loss = self.make_loss_op(x, y) self.feature_weights = __feature_weights self.gram_weights = __gram_weights return loss def preprocess_input(self, x): """Preprocesses a tensor encoding a batch of images. 1. Transform range [-1, 1] to [0, 255.0] 2. center around imagenet mean RGB values Parameters ---------- x : tf.Tenser input tensor, 4D in [-1,1] Returns ------- Preprocessed tensor : tf.Tensor """ # from [-1, 1] to [0, 255.0] x = (x + 1.0) / 2.0 * 255.0 # 'RGB'->'BGR' x = x[:, :, :, ::-1] # Zero-center by mean pixel x = x - np.array([103.939, 116.779, 123.68]).reshape((1, 1, 1, 3)) return x
class VGG19Features(object): def __init__(self, session, feature_layers=None, feature_weights=None, gram_weights=None): K.set_session(session) self.base_model = VGG19(include_top=False, weights='imagenet') if feature_layers is None: feature_layers = [ "input_1", "block1_conv2", "block2_conv2", "block3_conv2", "block4_conv2", "block5_conv2" ] self.layer_names = [l.val_set_name for l in self.base_model.layers] for k in feature_layers: if not k in self.layer_names: raise KeyError("Invalid layer {}. Available layers: {}".format( k, self.layer_names)) features = [ self.base_model.get_layer(k).output for k in feature_layers ] self.model = Model(inputs=self.base_model.input, outputs=features) if feature_weights is None: feature_weights = len(feature_layers) * [1.0] if gram_weights is None: gram_weights = len(feature_layers) * [0.1] self.feature_weights = feature_weights self.gram_weights = gram_weights assert len(self.feature_weights) == len(features) self.use_gram = np.max(self.gram_weights) > 0.0 self.variables = self.base_model.weights def extract_features(self, x): """x should be rgb in [-1,1].""" x = preprocess_input(x) features = self.model.predict(x) return features def make_feature_ops(self, x): """x should be rgb tensor in [-1,1].""" x = preprocess_input(x) features = self.model(x) return features def grams(self, fs): gs = list() for f in fs: bs, h, w, c = f.shape.as_list() f = tf.reshape(f, [bs, h * w, c]) ft = tf.transpose(f, [0, 2, 1]) g = tf.matmul(ft, f) g = g / (c * h * w) gs.append(g) return gs def make_loss_op(self, x, y): """x, y should be rgb tensors in [-1,1].""" x = preprocess_input(x) x_features = self.model(x) self.x_features = x_features y = preprocess_input(y) y_features = self.model(y) self.y_features = y_features x_grams = self.grams(x_features) self.x_grams = x_grams y_grams = self.grams(y_features) self.y_grams = y_grams losses = [ tf.reduce_mean(tf.abs(xf - yf)) for xf, yf in zip(x_features, y_features) ] gram_losses = [ tf.reduce_mean(tf.abs(xg - yg)) for xg, yg in zip(x_grams, y_grams) ] for i in range(len(losses)): losses[i] = self.feature_weights[i] * losses[i] gram_losses[i] = self.gram_weights[i] * gram_losses[i] loss = tf.add_n(losses) if self.use_gram: loss = loss + tf.add_n(gram_losses) self.losses = losses self.gram_losses = gram_losses return loss
def get_activations(model, layer_name, input): # layer_name = 'abundances' intermediate_layer_model = Model( inputs=model.input, outputs=model.get_layer(layer_name).output) return intermediate_layer_model.predict(input)
class VGG19Features(object): def __init__(self, session, feature_layers = None, feature_weights = None): K.set_session(session) self.base_model = VGG19( include_top = False, weights='imagenet') if feature_layers is None: feature_layers = [ "input_1", "block1_conv2", "block2_conv2", "block3_conv2", "block4_conv2", "block5_conv2"] self.layer_names = [l.name for l in self.base_model.layers] for k in feature_layers: if not k in self.layer_names: raise KeyError( "Invalid layer {}. Available layers: {}".format( k, self.layer_names)) features = [self.base_model.get_layer(k).output for k in feature_layers] self.model = Model( inputs = self.base_model.input, outputs = features) if feature_weights is None: feature_weights = len(feature_layers) * [1.0] self.feature_weights = feature_weights assert len(self.feature_weights) == len(features) self.variables = self.base_model.weights def extract_features(self, x): """x should be rgb in [-1,1].""" x = preprocess_input(x) features = self.model.predict(x) return features def make_feature_ops(self, x): """x should be rgb tensor in [-1,1].""" x = preprocess_input(x) features = self.model(x) return features def make_loss_op(self, x, y): """x, y should be rgb tensors in [-1,1].""" x = preprocess_input(x) x_features = self.model(x) y = preprocess_input(y) y_features = self.model(y) losses = [ tf.reduce_mean(tf.abs(xf - yf)) for xf, yf in zip( x_features, y_features)] for i in range(len(losses)): losses[i] = self.feature_weights[i] * losses[i] loss = tf.add_n(losses) self.losses = losses return loss
def on_epoch_end(self, epoch, logs=None): if self.endmembersGT is None: self.plotGT = False self.losses.append(logs.get('loss')) self.num_epochs = epoch # self.losses.append(logs.get('loss')) endmembers = self.model.get_layer('endmembers').get_weights()[0] if self.plotGT: sad = compute_ASAM_rad(endmembers, self.endmembersGT, None) self.sads.append(sad) # print('SAD: ' + str(sad)) # spio.savemat('sad.mat', {'SAD': self.sads,'M':endmembers,'M_GT':self.endmembersGT}) sio.savemat('sad.mat', {'SAD': self.sads, 'LOSS': self.losses}) if self.plot_every_n == 0 or epoch % self.plot_every_n != 0: return if self.plotS: if self.num_inputs > 1: intermediate_layer_model = Model( inputs=self.model.input, outputs=[ self.model.get_layer('abundances' + str(i)).output for i in range(self.num_inputs) ]) abundances = np.mean(intermediate_layer_model.predict( [self.input.orig_data for i in range(self.num_inputs)]), axis=0) else: intermediate_layer_model = Model( inputs=self.model.input, outputs=self.model.get_layer('abundances').output) abundances = intermediate_layer_model.predict( self.input.orig_data) if self.size is None: self.size = (int(np.sqrt(abundances.shape[0])), int(np.sqrt(abundances.shape[0]))) # plotHist(self.losses, 33) # self.plotGT = False if self.plotGT: dict = order_endmembers(endmembers, self.endmembersGT) # if self.is_GT_for_A: # plotAbundances(self.num_endmembers, self.size_data, abundances, self.abundancesGT, dict, self.use_ASC) # else: # plotAbundances(self.num_endmembers, self.size_data, abundances, None, None, self.use_ASC, is_GT=False) plotEndmembersAndGT(self.endmembersGT, endmembers, dict) if self.plotS: plotAbundancesSimple(self.num_endmembers, (self.size[0], self.size[1]), abundances, dict, use_ASC=1, figure_nr=10) else: # plotAbundances(self.num_endmembers, self.size_data, abundances, None, None, self.use_ASC) plotEndmembers(self.num_endmembers, endmembers) plotAbundancesSimple(self.num_endmembers, (self.size[0], self.size[1]), abundances, None, use_ASC=1, figure_nr=10) return
def transfer_model(source_df, target_df, test_df, method_flag, fold_num): source_labels, source_data = np.split(np.array(source_df),[1],axis=1) target_labels, target_data = np.split(np.array(target_df),[1],axis=1) test_labels, test_data = np.split(np.array(test_df),[1],axis=1) # normalization #normalized_source_data = pre.normalize(source_data) #normalized_target_data = pre.normalize(target_data) #normalized_test_data = pre.normalize(test_data) normalized_source_data = source_data normalized_target_data = target_data normalized_test_data = test_data ### constuct model for source domain task ### # optimization opt = Adam() # network setting latent = models.latent(normalized_source_data.shape[1]) sll = models.source_last_layer() tll = models.target_last_layer() source_inputs = Input(shape=normalized_source_data.shape[1:]) latent_features = latent(source_inputs) source_predictors = sll(latent_features) latent.trainable = mc._SORUCE_LATENT_TRAIN source_predictors.trainable = True source_nn = Model(inputs=[source_inputs], outputs=[source_predictors]) source_nn.compile(loss=['mean_squared_error'],optimizer=opt) #source_nn.summary() # training using source domain data if method_flag != mc._SCRATCH: source_max_loop = int(normalized_source_data.shape[0]/mc._BATCH_SIZE) source_progbar = Progbar(target=mc._SOURCE_EPOCH_NUM) for epoch in range(mc._SOURCE_EPOCH_NUM): shuffle_data, shuffle_labels, _ = pre.paired_shuffle(normalized_source_data,source_labels,1) for loop in range(source_max_loop): batch_train_data = shuffle_data[loop*mc._BATCH_SIZE:(loop+1)*mc._BATCH_SIZE] batch_train_labels = shuffle_labels[loop*mc._BATCH_SIZE:(loop+1)*mc._BATCH_SIZE] batch_train_labels = np.reshape(batch_train_labels, [len(batch_train_labels)]) one_hots = np.identity(mc._SOURCE_DIM_NUM)[np.array(batch_train_labels, dtype=np.int32)] loss = source_nn.train_on_batch([batch_train_data],[one_hots]) #source_progbar.add(1, values=[("source loss",loss)]) # save #latent.save('../results/source_latent.h5') #sll.save('../results/source_last_layer.h5') # compute relation vectors if method_flag == mc._SCRATCH or method_flag == mc._CONV_TRANSFER: target_vectors = np.identity(mc._TARGET_DIM_NUM)[np.array(target_labels, dtype=np.int32)] target_vectors = np.reshape(target_vectors, [target_vectors.shape[0], target_vectors.shape[2]]) elif method_flag == mc._COUNT_ATDL: target_labels, relations = rv.compute_relation_labels(source_nn, normalized_target_data, target_labels, fold_num) target_vectors = np.identity(mc._SOURCE_DIM_NUM)[np.array(target_labels, dtype=np.int32)] target_vectors = np.reshape(target_vectors, [target_vectors.shape[0], target_vectors.shape[2]]) else: relation_vectors = rv.compute_relation_vectors(source_nn, normalized_target_data, target_labels, fold_num, method_flag) target_vectors = np.zeros((len(target_labels),mc._SOURCE_DIM_NUM), dtype=np.float32) for i in range(len(target_labels)): target_vectors[i] = relation_vectors[int(target_labels[i])] ### tuning model for target domain task ### latent.trainable = mc._TARGET_LATENT_TRAIN target_inputs = Input(shape=normalized_target_data.shape[1:]) latent_features = latent(target_inputs) if method_flag == mc._SCRATCH or method_flag == mc._CONV_TRANSFER: predictors = tll(latent_features) label_num = mc._TARGET_DIM_NUM else: predictors= sll(latent_features) label_num = mc._SOURCE_DIM_NUM target_nn = Model(inputs=[target_inputs], outputs=[predictors]) target_nn.compile(loss=['mean_squared_error'],optimizer=opt) #target_nn.summary() # training using target domain data target_max_loop = int(normalized_target_data.shape[0]/mc._BATCH_SIZE) target_progbar = Progbar(target=mc._TARGET_EPOCH_NUM) for epoch in range(mc._TARGET_EPOCH_NUM): shuffle_data, shuffle_labels, _ = \ pre.paired_shuffle(normalized_target_data, target_vectors, label_num) for loop in range(target_max_loop): batch_train_data = shuffle_data[loop*mc._BATCH_SIZE:(loop+1)*mc._BATCH_SIZE] batch_train_labels = shuffle_labels[loop*mc._BATCH_SIZE:(loop+1)*mc._BATCH_SIZE] loss = target_nn.train_on_batch([batch_train_data],[batch_train_labels]) #target_progbar.add(1, values=[("target loss",loss)]) # compute outputs of test data of target domain x = target_nn.predict([normalized_test_data]) if method_flag == mc._SCRATCH or method_flag == mc._CONV_TRANSFER: idx = np.argmax(x, axis=1) elif method_flag == mc._COUNT_ATDL: idx = np.argmax(x,axis=1) for j in range(len(test_labels)): for i in range(mc._TARGET_DIM_NUM): if test_labels[j] == i: test_labels[j] = relations[i] break else: distance, idx = Neighbors(x, relation_vectors, 1) idx = idx[:,0] backend.clear_session() return idx.T, test_labels.T