def ensemble_folds(models, input_shape): model_input = layers.Input(shape=input_shape) outputs = [model(model_input) for model in models] y = layers.Average()(outputs) model = Model(model_input, y, name='ensemble') return model
def ensemble(self, models, model_input): # collect outputs of models in a list outputs = [model(model_input) for model in models] # averaging outputs y = layers.Average()(outputs) # build model from same input and avg output model = Model(inputs=model_input, outputs=y, name='ensemble') return model
def build_model(self): """ Build the model Returns: Model : Keras model instance """ # Checks if len(self.kernel_sizes) != len(self.feature_maps): raise Exception('Please define `kernel_sizes` and `feature_maps` with the same amount.') if not self.embedding_layer and (not self.num_words or not self.embedding_dim): raise Exception('Please define `num_words` and `embedding_dim` if you not using a pre-trained embedding.') if self.use_char and (not self.char_max_length or not self.alphabet_size): raise Exception('Please define `char_max_length` and `alphabet_size` if you are using char.') # Building word-embeddings from scratch if self.embedding_layer is None: self.embedding_layer = layers.Embedding( input_dim=self.num_words, output_dim=self.embedding_dim, input_length=self.max_seq_length, weights=None, trainable=True, name="word_embedding" ) # WORD-level word_input = layers.Input(shape=(self.max_seq_length,), dtype='int32', name='word_input') x = self.embedding_layer(word_input) if self.dropout_rate: x = layers.Dropout(self.dropout_rate)(x) x = self.building_block(x, self.kernel_sizes, self.feature_maps) x = layers.Activation('relu')(x) prediction = layers.Dense(self.nb_classes, activation='softmax')(x) # CHAR-level if self.use_char: char_input = layers.Input(shape=(self.char_max_length,), dtype='int32', name='char_input') x_char = layers.Embedding( input_dim=self.alphabet_size + 1, output_dim=50, input_length=self.char_max_length, name='char_embedding' )(char_input) x_char = self.building_block(x_char, self.char_kernel_sizes, self.char_feature_maps) x_char = layers.Activation('relu')(x_char) x_char = layers.Dense(self.nb_classes, activation='softmax')(x_char) prediction = layers.Average()([prediction, x_char]) return keras.Model(inputs=[word_input, char_input], outputs=prediction, name='CNN_Word_Char') return keras.Model(inputs=word_input, outputs=prediction, name='CNN_Word')
def test_merge_average(): i1 = layers.Input(shape=(4, 5)) i2 = layers.Input(shape=(4, 5)) o = layers.average([i1, i2]) assert o._keras_shape == (None, 4, 5) model = models.Model([i1, i2], o) avg_layer = layers.Average() o2 = avg_layer([i1, i2]) assert avg_layer.output_shape == (None, 4, 5) x1 = np.random.random((2, 4, 5)) x2 = np.random.random((2, 4, 5)) out = model.predict([x1, x2]) assert out.shape == (2, 4, 5) assert_allclose(out, 0.5 * (x1 + x2), atol=1e-4)
def model(reader): assert reader.channeltypes == [ChannelType.POS_TAGS] known_in = L.Input(shape=(None, ), dtype='int32') unknown_in = L.Input(shape=(None, ), dtype='int32') embedding = L.Embedding( len(reader.channels[0].vocabulary_above_cutoff) + 2, 1) known_emb = embedding(known_in) unknown_emb = embedding(unknown_in) feature_extractor1 = L.Bidirectional(L.LSTM(100, return_sequences=True)) pool = L.GlobalAvgPool1D() features_known = pool(feature_extractor1(known_emb)) features_unknown = pool(feature_extractor1(unknown_emb)) abs_diff = L.merge(inputs=[features_known, features_unknown], mode=lambda x: abs(x[0] - x[1]), output_shape=lambda x: x[0], name='absolute_difference') # Dense network. dense1 = L.Dense(300)(abs_diff) dense2 = L.Dense(200)(dense1) dense3 = L.Dense(100)(dense2) output1 = L.Dense(2, activation='softmax')(dense1) output2 = L.Dense(2, activation='softmax')(dense2) output3 = L.Dense(2, activation='softmax')(dense3) output = L.Average()([output1, output2, output3]) model = Model(inputs=[known_in, unknown_in], outputs=output) optimizer = O.Adam(lr=0.0005) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) return model
def create_multiview_model(base_model, multiviews_num, input_shape, output_size, use_lstm): assert base_model == "voxnet" or base_model == "pointnet" if base_model == "voxnet": base_model = create_voxnet_model_homepage(input_shape, output_size) elif base_model == "pointnet": base_model = create_point_net(input_shape, output_size) input = layers.Input(shape=(multiviews_num,) + input_shape) multiview_outputs = [] for i in range(multiviews_num): multiview_input = layers.Lambda(lambda x: x[:,i])(input) multiview_output = base_model(multiview_input) multiview_outputs.append(multiview_output) output = layers.Average()(multiview_outputs) model = models.Model(input, output) return model
def build_ensemble(net_configs, pop_per_type = 5, merge_type = None): merge_layers = { "Average" : nn_layers.Average() } inputs = [] outputs = [] for net_conf in net_configs: for i in range(pop_per_type): il, ol = build_layers(json.loads(net_conf)) inputs.append(il) outputs.append(ol) train_model = Model(inputs=inputs, outputs=outputs) if len(outputs) > 1: merge_layer = merge_layers[merge_type] if merge_type != None else None merge_model = Model(inputs=inputs, outputs=merge_layer(outputs)) if merge_layer != None else None else: # There is only 1 member in the Ensemble merge_model = train_model model_list = [Model(inputs=il,outputs=ol) for il,ol in zip(inputs,outputs)] return inputs, outputs, train_model, model_list, merge_model
def get_keras_model(self): inp, bias_counts_input, bias_profile_input = self.get_inputs() rev_inp = kl.Lambda(lambda x: x[:, ::-1, ::-1])(inp) countouttaskname, profileouttaskname = self.get_names() first_conv = kl.Conv1D(self.filters, kernel_size=self.conv1_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid', activation='relu') first_conv_fwd = first_conv(inp) first_conv_rev = first_conv(rev_inp) curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1) prev_layers_fwd = first_conv_fwd prev_layers_rev = first_conv_rev for i in range(1, self.n_dil_layers + 1): dilation_rate = 2**i conv_output = kl.Conv1D(self.filters, kernel_size=self.dil_kernel_size, padding='valid', kernel_initializer=self.kernel_initializer, activation='relu', dilation_rate=dilation_rate) conv_output_fwd = conv_output(prev_layers_fwd) conv_output_rev = conv_output(prev_layers_rev) width_to_trim = dilation_rate * (self.dil_kernel_size - 1) curr_layer_size = (curr_layer_size - width_to_trim) prev_layers_fwd = self.trim_flanks_of_conv_layer( conv_layer=prev_layers_fwd, output_len=curr_layer_size, width_to_trim=width_to_trim, filters=self.filters) prev_layers_rev = self.trim_flanks_of_conv_layer_revcomp( conv_layer=prev_layers_rev, output_len=curr_layer_size, width_to_trim=width_to_trim, filters=self.filters) if (self.is_add): prev_layers_fwd = kl.add([prev_layers_fwd, conv_output_fwd]) prev_layers_rev = kl.add([prev_layers_rev, conv_output_rev]) else: prev_layers_fwd = kl.average( [prev_layers_fwd, conv_output_fwd]) prev_layers_rev = kl.average( [prev_layers_rev, conv_output_rev]) combined_conv_fwd = prev_layers_fwd combined_conv_rev = prev_layers_rev # Counts Prediction counts_dense_layer = kl.Dense( 2, kernel_initializer=self.kernel_initializer, ) gap_combined_conv_fwd = kl.GlobalAvgPool1D()(combined_conv_fwd) gap_combined_conv_rev = kl.GlobalAvgPool1D()(combined_conv_rev) main_count_out_fwd = counts_dense_layer( kl.concatenate([gap_combined_conv_fwd, bias_counts_input], axis=-1)) main_count_out_rev = counts_dense_layer( kl.concatenate([bias_counts_input, gap_combined_conv_rev], axis=-1)) rc_rev_count_out = kl.Lambda(lambda x: x[:, ::-1])(main_count_out_rev) avg_count_out = kl.Average(name=countouttaskname)( [main_count_out_fwd, rc_rev_count_out]) # Profile Prediction profile_penultimate_conv = kl.Conv1D( filters=2, kernel_size=self.outconv_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid') profile_final_conv = kl.Conv1D( 2, kernel_size=1, kernel_initializer=self.kernel_initializer, ) profile_out_prebias_fwd = profile_penultimate_conv(combined_conv_fwd) main_profile_out_fwd = profile_final_conv( kl.concatenate([profile_out_prebias_fwd, bias_profile_input], axis=-1)) profile_out_prebias_rev = profile_penultimate_conv(combined_conv_rev) rev_bias_profile_input = kl.Lambda(lambda x: x[:, ::-1, :])( bias_profile_input) main_profile_out_rev = profile_final_conv( kl.concatenate([profile_out_prebias_rev, rev_bias_profile_input], axis=-1)) rc_rev_profile_out = kl.Lambda(lambda x: x[:, ::-1, ::-1])( main_profile_out_rev) avg_profile_out = kl.Average(name=profileouttaskname)( [main_profile_out_fwd, rc_rev_profile_out]) model = keras.models.Model( inputs=[inp, bias_counts_input, bias_profile_input], outputs=[avg_count_out, avg_profile_out]) model.compile(keras.optimizers.Adam(lr=self.lr), loss=['mse', MultichannelMultinomialNLL(2)], loss_weights=[self.c_task_weight, self.p_task_weight]) return model
def train(self): U.save_GPU_mem_keras() expr = U.ExprCreaterAndResumer(modelDir, postfix="dr%s_imgOnly" % (str(dropout))) # x channel: image x_inputs = L.Input(shape=inputShape) x = x_inputs # inputs is used by the line "Model(inputs, ... )" below conv11 = L.Conv2D(16, (3, 3), strides=1, dilation_rate=2, padding='valid') x = conv11(x) x = L.Activation('relu')(x) x = L.BatchNormalization()(x) # Batch needs to be after relu, otherwise it won't train... conv12 = L.Conv2D(16, (3, 3), strides=1, dilation_rate=2, padding='valid') x = conv12(x) x = L.Activation('relu')(x) x = L.BatchNormalization()(x) conv13 = L.Conv2D(16, (3, 3), strides=1, dilation_rate=2, padding='valid') x = conv13(x) x = L.Activation('relu')(x) x_output = L.BatchNormalization()(x) # z channel: optical flow z_inputs = L.Input(shape=inputShapeOF) z = z_inputs # inputs is used by the line "Model(inputs, ... )" below conv21 = L.Conv2D(16, (3, 3), strides=1, dilation_rate=2, padding='valid') z = conv21(z) z = L.Activation('relu')(z) z = L.BatchNormalization()(z) conv22 = L.Conv2D(16, (3, 3), strides=1, dilation_rate=2, padding='valid') z = conv22(z) z = L.Activation('relu')(z) z = L.BatchNormalization()(z) conv23 = L.Conv2D(16, (3, 3), strides=1, dilation_rate=2, padding='valid') z = conv23(z) z = L.Activation('relu')(z) z_output = L.BatchNormalization()(z) joint = L.Average()([x_output, z_output]) joint = L.Flatten()(joint) joint = L.Dense(32, activation='relu')(joint) joint = L.Dropout(dropout)(joint) output = L.Dense(1, activation='sigmoid')(joint) model = Model(inputs=[x_inputs, z_inputs], outputs=output) opt = K.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0) #opt = K.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[K.metrics.binary_accuracy]) print(model.summary()) # snapshot code before training the model expr.dump_src_code_and_model_def(sys.argv[0], model) model.fit([self.trainData, self.trainDataOF], self.trainLabel, validation_data=([self.testData, self.testDataOF], self.testLabel), shuffle=True, batch_size=100, epochs=epoch, verbose=2, callbacks=[ K.callbacks.TensorBoard(log_dir=expr.dir), K.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=0.00001), U.PrintLrCallback() ]) expr.save_weight_and_training_config_state(model) score = model.evaluate([self.testData, self.testDataOF], self.testLabel, batch_size=100, verbose=0) expr.printdebug("eval score:" + str(score))
def get_or_create(self, gaze_model, name, reuse, num_actions, layer_norm, dueling): if name in self.models: assert reuse == True logger.log("QFunc model named %s is reused" % name) else: logger.log("QFunc model named %s is created" % name) assert reuse == False imgs = L.Input(shape=(84, 84, 4)) g = gaze_model(imgs) gaze_heatmaps = L.Lambda(lambda x: (x - tf.reduce_min( x, [1, 2, 3], True)) / (tf.reduce_max(x, [1, 2, 3], True) - tf. reduce_min(x, [1, 2, 3], True)))(g) g = gaze_heatmaps x = imgs x = L.Multiply(name="img_mul_gaze")([x, g]) c1 = L.Conv2D(32, (8, 8), strides=4, padding='same', activation="relu", name='mul_c1') x = c1(x) c2 = L.Conv2D(64, (4, 4), strides=2, padding='same', activation="relu", name='mul_c2') x = c2(x) c3 = L.Conv2D(64, (3, 3), strides=1, padding='same', activation="relu", name='mul_c3') x = c3(x) # ============================ channel 2 ============================ orig_x = imgs orig_x = L.Conv2D(32, (8, 8), strides=4, padding='same', activation="relu")(orig_x) orig_x = L.Conv2D(64, (4, 4), strides=2, padding='same', activation="relu")(orig_x) orig_x = L.Conv2D(64, (3, 3), strides=1, padding='same', activation="relu")(orig_x) x = L.Average()([x, orig_x]) x = L.Flatten()(x) if dueling: state_score = L.Dense(512)(x) if layer_norm: state_score = L.BatchNormalization()(state_score) state_score = L.Activation('relu')(state_score) state_score = L.Dense(1)(state_score) action_score = L.Dense(512)(x) if layer_norm: logger.log( "Warning: layer_norm is set to True, but Keras doesn't have it. Replacing with BatchNorm." ) action_score = L.BatchNormalization()(action_score) action_score = L.Activation('relu')(action_score) last_dense = L.Dense(num_actions, name="logits") action_score = last_dense(action_score) if dueling: def wrapped_tf_ops(s): action_score, state_score = s return action_score - tf.reduce_mean( action_score, 1, keep_dims=True) + state_score action_score = L.Lambda(wrapped_tf_ops)( [action_score, state_score]) model = Model(inputs=[imgs], outputs=[action_score, gaze_heatmaps]) model.interesting_layers = [ c1, c2, c3, last_dense ] # export variable interesting_layers for monitoring in train.py self.models[name] = model return self.models[name]
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data_more') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [25, 27, 30] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = digits_data2['optimal_lw'] d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 50 trials = 10 mnist_correct = [] mnist_wrong = [] digits_wrong = [] digits_correct = [] d2_wrong = [] d2_correct = [] for t in range(trials): inputs = [] outputs = [] model_list = [] for e in range(ensemble_size): model = Sequential() model.add( layers.Dense(200, input_dim=784, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(200, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(200, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(10, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("softmax")) es = clb.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True) model.compile(optimizer=opt.Adam(), loss="categorical_crossentropy", metrics=['acc']) model.fit(xtrain, ytrain, epochs=epochs, batch_size=100, validation_split=(1 / 6), callbacks=[es]) model_list.append(model) inputs.extend(model.inputs) outputs.extend(model.outputs) merge_model = Model(inputs=inputs, outputs=layers.Average()(outputs)) #mnist_preds = merge_model.predict([xtest]*ensemble_size) #mnist_mem_preds = np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,2,0) #correct, wrong = bin_entropies(mnist_preds, mnist_mem_preds ,ytest) #mnist_correct.extend(correct) #mnist_wrong.extend(wrong) sp_digits = apply_salt_pepper(digits_og) for s_d in sp_digits: s_d = utils.normalize_data(reshape_fun(s_d)) d2_preds = merge_model.predict([s_d] * ensemble_size) d2_mempreds = np.array( list(map(lambda m: m.predict(s_d), model_list))).transpose(1, 2, 0) correct, wrong = bin_entropies(d2_preds, d2_mempreds, d2_labels) d2_correct.extend(correct) d2_wrong.extend(wrong) for d in digits: digits_preds = merge_model.predict([d] * ensemble_size) mempreds = np.array(list(map(lambda m: m.predict(d), model_list))).transpose(1, 2, 0) correct, wrong = bin_entropies(digits_preds, mempreds, d_labels) digits_wrong.extend(wrong) digits_correct.extend(correct) ensemble = { #'mnist_correct' : mnist_correct, #'mnist_wrong' : mnist_wrong, 'digits_correct': digits_correct, 'digits_wrong': digits_wrong, 'lecunn_correct': d2_correct, 'lecunn_wrong': d2_wrong } return ensemble
orig_x=imgs orig_x=L.Conv2D(32, (8,8), strides=4, padding='same')(orig_x) orig_x=L.BatchNormalization()(orig_x) orig_x=L.Activation('relu')(orig_x) orig_x=L.Dropout(dropout)(orig_x) orig_x=L.Conv2D(64, (4,4), strides=2, padding='same')(orig_x) orig_x=L.BatchNormalization()(orig_x) orig_x=L.Activation('relu')(orig_x) orig_x=L.Dropout(dropout)(orig_x) orig_x=L.Conv2D(64, (3,3), strides=1, padding='same')(orig_x) orig_x=L.BatchNormalization()(orig_x) orig_x=L.Activation('relu')(orig_x) x=L.Average()([x,orig_x]) x=L.Dropout(dropout)(x) x=L.Flatten()(x) x=L.Dense(512, activation='relu')(x) x=L.Dropout(dropout)(x) logits=L.Dense(NUM_CLASSES, name="logits")(x) prob=L.Activation('softmax', name="prob")(logits) model=Model(inputs=[imgs, gaze_heatmaps], outputs=[logits, prob, g, x_intermediate]) opt=K.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0) model.compile(loss={"prob":None, "logits": MU.loss_func}, optimizer=opt,metrics={"logits": MU.acc_}) expr.dump_src_code_and_model_def(sys.argv[0], model) d=input_utils.Dataset(LABELS_FILE_TRAIN, LABELS_FILE_VAL, SHAPE)
def model_creation(input_shape1, input_shape2, model_version, optim, problem_type=0): """ op_sequence : False, by default. If True, then also uses the num_nodes parameter. num_nodes : The number that denotes that how many values to predict at the output layer. """ print("Inside model_creation") past_inp = L.Input(shape=(input_shape1)) fut_inp = L.Input(shape=(input_shape2)) if (model_version == "M1V1"): cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn1 = L.Conv1D(filters=32, kernel_size=2)(cnn1) cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp) cnn2 = L.Conv1D(filters=32, kernel_size=2)(cnn2) lstm_inp = L.Average()([cnn1, cnn2]) lstm_out = L.LSTM(32, recurrent_dropout=0.2, return_sequences=True, bias_initializer='ones')(lstm_inp) x1 = L.Average()([lstm_out, lstm_inp]) x1 = L.Flatten()(x1) elif (model_version == "M1V2"): cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn1 = L.Conv1D(filters=32, kernel_size=3)(cnn1) cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp) cnn2 = L.Conv1D(filters=32, kernel_size=3)(cnn2) x1 = L.Average()([cnn1, cnn2]) x1 = L.Flatten()(x1) elif (model_version == "M1V3"): x1 = L.LSTM(32, recurrent_dropout=0.2, return_sequences=True, bias_initializer='ones')(past_inp) #x1 = L.LSTM(32, recurrent_dropout=0.2, return_sequences=True, bias_initializer='ones')(x1) x1 = L.LSTM(32, recurrent_dropout=0.2, bias_initializer='ones')(x1) elif (model_version == "M2V1"): cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn1 = L.Conv1D(filters=32, kernel_size=2)(cnn1) lstm_out1 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(cnn1) lstm_out1 = L.Average()([cnn1, lstm_out1]) cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp) cnn2 = L.Conv1D(filters=32, kernel_size=2)(cnn2) lstm_out2 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(cnn2) lstm_out2 = L.Average()([cnn2, lstm_out2]) x1 = L.Average()([lstm_out1, lstm_out2]) x1 = L.Flatten()(x1) elif (model_version == "M2V2"): lstm_out1 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(past_inp) lstm_out2 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(fut_inp) x1 = L.Average()([lstm_out1, lstm_out2]) x1 = L.Flatten()(x1) elif (model_version == "M3V1"): # cnn_inp = L.Concatenate(axis=1)([past_inp,fut_inp]) cnn = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn = L.Conv1D(filters=32, kernel_size=2)(cnn) layer = L.Lambda(lambda cnn: K.reverse(cnn, axes=1)) cnn2 = layer(cnn) lstm1 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(cnn) lstm2 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(cnn2) lstm = L.Average()([lstm1, lstm2]) x1 = L.Average()([cnn, lstm]) x1 = L.Flatten()(x1) elif (model_version == "M3V2"): cnn_inp = L.Concatenate(axis=1)([past_inp, fut_inp]) cnn = L.Conv1D(filters=32, kernel_size=5)(cnn_inp) cnn = L.Conv1D(filters=32, kernel_size=2)(cnn) layer = L.Lambda(lambda cnn: K.reverse(cnn, axes=1)) cnn2 = layer(cnn) lstm1 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(cnn) lstm2 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(cnn2) lstm = L.Average()([lstm1, lstm2]) x1 = L.Average()([cnn, lstm]) x1 = L.Flatten()(x1) elif (model_version == "M4"): cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn1 = L.Conv1D(filters=32, kernel_size=2)(cnn1) cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp) cnn2 = L.Conv1D(filters=32, kernel_size=2)(cnn2) lstm1 = L.Bidirectional( L.LSTM(16, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones'))(past_inp) lstm2 = L.Bidirectional( L.LSTM(16, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones'))(fut_inp) out1 = L.Concatenate(axis=1)([cnn1, lstm1]) out2 = L.Concatenate(axis=1)([cnn2, lstm2]) # out1 = L.Average()([cnn1,cnn2]) # ousst2 = L.Average()([lstm1,lstm2]) # x1 = L.Concatenate(axis=1)([out1,out2]) x1 = L.Average()([out1, out2]) x1 = L.Flatten()(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) #Classification Part. if (problem_type == 1): main_out = L.Dense(2, activation='softmax')(x1) model = M.Model(inputs=[past_inp, fut_inp], outputs=[main_out], name=model_version) model.summary() model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) #Regression Part. else: x1 = L.Dense(1)(x1) main_out = L.advanced_activations.LeakyReLU(0.2)(x1) model = M.Model(inputs=[past_inp, fut_inp], outputs=[main_out], name=model_version) model.compile(optimizer=optim, loss=tf.losses.huber_loss, metrics=['mae', 'mse', rmse]) model.summary() return model
model.fit(xtrain, ytrain, epochs=epochs, batch_size=100, callbacks=[es], validation_split=(1 / 6)) members.append(model) i = 0 for ensemble_size in range(2, ensemble_size_top + 1): inputs = [] outputs = [] members_to_use = members[i:i + ensemble_size] for m in members_to_use: inputs.extend(m.inputs) outputs.extend(m.outputs) print((outputs)) ensemble = Model(inputs=inputs, outputs=layers.Average()(outputs)) ensemble.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) accuracy = ann.test_model(ensemble, [xtest] * ensemble_size, ytest, 'accuracy') i += ensemble_size t_accuracies.append(accuracy) t_accuracies = np.array(t_accuracies) utils.save_processed_data(t_accuracies, 'ensemble_sizesim-trial-%s' % (t + 1))
def experiment(reshape_mode = 'mlp'): reshape_funs = { "conv" : lambda d : d.reshape(-1,28,28,1), "mlp" : lambda d : d.reshape(-1,784) } xtrain,ytrain,xtest,ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest) notmnist = load_notMNIST() notmnist = reshape_fun(notmnist) print(notmnist.shape) ensemble_size = 20 epochs = 50 trials = 10 results = { 'A': [], 'B': [], 'C': [], 'D': [], 'E': [], 'F': [], 'G': [], 'H': [], 'I': [], 'J': [] } results = [] for t in range(trials): inputs = [] outputs = [] model_list = [] for e in range(ensemble_size): model = Sequential() model.add(layers.Dense(200,input_dim=784, kernel_initializer = inits.RandomUniform(maxval=0.5,minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add(layers.Dense(200, kernel_initializer = inits.RandomUniform(maxval=0.5,minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add(layers.Dense(200, kernel_initializer = inits.RandomUniform(maxval=0.5,minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add(layers.Dense(10, kernel_initializer = inits.RandomUniform(maxval=0.5,minval=-0.5))) model.add(layers.Activation("softmax")) es = clb.EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True) model.compile(optimizer=opt.Adam(),loss="categorical_crossentropy",metrics=['acc']) model.fit(xtrain,ytrain,epochs=epochs,batch_size=100,validation_split=(1/6),callbacks=[es]) model_list.append(model) inputs.extend(model.inputs) outputs.extend(model.outputs) merge_model = Model(inputs = inputs, outputs = layers.Average()(outputs)) preds = merge_model.predict([notmnist]*ensemble_size) mem_preds = np.array(list(map(lambda m : m.predict(notmnist), model_list))).transpose(1,2,0) print(mem_preds.shape) bits = list(map(stats.entropy,preds)) s_q = list(map(calc_pred_vars,mem_preds)) results.extend(list(zip(bits,s_q))) return results
layers.Dense(10, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("softmax")) es = clb.EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True) model.compile(optimizer=opt.Adam(), loss="categorical_crossentropy", metrics=['acc']) model.fit(xtrain, ytrain, epochs=epochs, batch_size=100, validation_split=(1 / 6), callbacks=[es]) model_list.append(model) inputs.extend(model.inputs) outputs.extend(model.outputs) merge_layer = layers.Average()( outputs) if ensemble_size > 1 else outputs ensemble = Model(inputs=inputs, outputs=merge_layer) pred = ensemble.predict([notmnist] * ensemble_size) h = list(map(stats.entropy, pred)) results[ensemble_size].extend(h) utils.save_processed_data(results, 'notmnist_sim-trial-%s' % (t + 1))
y = L.Dropout(dropout)(y) deconv22 = L.Conv2DTranspose(32, (4, 4), strides=2, padding='valid') y = deconv22(y) y = L.Activation('relu')(y) y = L.BatchNormalization()(y) y = L.Dropout(dropout)(y) deconv23 = L.Conv2DTranspose(1, (8, 8), strides=4, padding='valid') y = deconv23(y) print deconv23.output_shape y = L.Activation('relu')(y) y_output = L.BatchNormalization()(y) # Merge outputs from 2 channels outputs = L.Average()([x_output, y_output]) outputs = L.Activation(MU.my_softmax)(outputs) model = Model(inputs=[x_inputs, y_inputs], outputs=outputs) opt = K.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0) # opt=K.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08) model.compile(loss=MU.my_kld, optimizer=opt, metrics=[MU.NSS]) d = IU.DatasetWithHeatmap_PastKFrames(LABELS_FILE_TRAIN, LABELS_FILE_VAL, SHAPE, heatmap_shape, GAZE_POS_ASC_FILE, k, stride) of = IU.Dataset_OpticalFlow_PastKFrames(LABELS_FILE_TRAIN, LABELS_FILE_VAL, SHAPE, k, stride)
def model_creation(input_shape1, input_shape2, model_version): """ op_sequence : False, by default. If True, then also uses the num_nodes parameter. num_nodes : The number that denotes that how many values to predict at the output layer. """ past_inp = L.Input(shape=(input_shape1)) fut_inp = L.Input(shape=(input_shape2)) if (model_version == "M1V1"): cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn1 = L.Conv1D(filters=32, kernel_size=3)(cnn1) # cnn1 = L.Dense(32)(cnn1) # cnn1 = L.advanced_activations.LeakyReLU(0.2)(cnn1) cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp) cnn2 = L.Conv1D(filters=32, kernel_size=3)(cnn2) # cnn2 = L.Dense(32)(cnn2) # cnn2 = L.advanced_activations.LeakyReLU(0.2)(cnn2) lstm_inp = L.Average()([cnn1, cnn2]) lstm_out = L.LSTM(32, recurrent_dropout=0.2, return_sequences=True, bias_initializer='ones')(lstm_inp) x1 = L.Average()([lstm_out, lstm_inp]) x1 = L.Flatten()(x1) elif (model_version == "M1V2"): cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn1 = L.Conv1D(filters=32, kernel_size=3)(cnn1) cnn1 = L.Dense(32)(cnn1) cnn1 = L.advanced_activations.LeakyReLU(0.2)(cnn1) cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp) cnn2 = L.Conv1D(filters=32, kernel_size=3)(cnn2) cnn2 = L.Dense(32)(cnn2) cnn2 = L.advanced_activations.LeakyReLU(0.2)(cnn2) x1 = L.Average()([cnn1, cnn2]) x1 = L.Flatten()(x1) elif (model_version == "M1V3"): x1 = L.LSTM(32, recurrent_dropout=0.2, return_sequences=True, bias_initializer='ones')(past_inp) x1 = L.LSTM(32, recurrent_dropout=0.2, return_sequences=True, bias_initializer='ones')(x1) x1 = L.LSTM(32, recurrent_dropout=0.2, bias_initializer='ones')(x1) elif (model_version == "M2V1"): cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn1 = L.Conv1D(filters=32, kernel_size=2)(cnn1) cnn1 = L.Dense(32)(cnn1) cnn1 = L.advanced_activations.LeakyReLU(0.2)(cnn1) lstm_out1 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(cnn1) lstm_out1 = L.Average()([cnn1, lstm_out1]) cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp) cnn2 = L.Conv1D(filters=32, kernel_size=2)(cnn2) cnn2 = L.Dense(32)(cnn2) cnn2 = L.advanced_activations.LeakyReLU(0.2)(cnn2) lstm_out2 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(cnn2) lstm_out2 = L.Average()([cnn2, lstm_out2]) x1 = L.Average()([lstm_out1, lstm_out2]) x1 = L.Flatten()(x1) elif (model_version == "M2V2"): lstm_out1 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(past_inp) lstm_out2 = L.LSTM(32, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones')(fut_inp) x1 = L.Average()([lstm_out1, lstm_out2]) x1 = L.Flatten()(x1) elif (model_version == "M3V1"): # cnn_inp = L.Concatenate(axis=1)([past_inp,fut_inp]) cnn = L.Conv1D(filters=32, kernel_size=5)(past_inp) cnn = L.Conv1D(filters=32, kernel_size=2)(cnn) # cnn = L.Dense(32)(cnn) # cnn = L.advanced_activations.LeakyReLU(0.2)(cnn) lstm = L.Bidirectional( L.LSTM(16, recurrent_dropout=0.3, return_sequences=True, bias_initializer='ones'))(cnn) x1 = L.Average()([cnn, lstm]) x1 = L.Flatten()(x1) elif (model_version == "M3V2"): cnn_inp = L.Concatenate(axis=1)([past_inp, fut_inp]) cnn = L.Conv1D(filters=32, kernel_size=5)(cnn_inp) cnn = L.Conv1D(filters=32, kernel_size=2)(cnn) cnn = L.Dense(32)(cnn) cnn = L.advanced_activations.LeakyReLU(0.2)(cnn) x1 = L.Flatten()(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(256)(x1) x1 = L.advanced_activations.LeakyReLU(0.2)(x1) x1 = L.Dense(1)(x1) main_out = L.advanced_activations.LeakyReLU(0.2)(x1) model = M.Model(inputs=[past_inp, fut_inp], outputs=[main_out], name=model_version) model.summary() model.compile(optimizer='adam', loss=tf.losses.huber_loss, metrics=['mae', rmse]) return model
models = [] inputs = [] outputs = [] setup_keras() for i in range(ensemble_size): net_input, net_output = create_model() inputs.append(net_input) outputs.append(net_output) model = Model(inputs=net_input, outputs=net_output) models.append(model) ensemble_model = nn_layers.Average()(outputs) ensemble_model = Model(inputs=inputs, outputs=ensemble_model) multi_model = Model(inputs=inputs, outputs=outputs) multi_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) multi_model.fit([train_x, train_x, train_x], [train_y, train_y, train_y], verbose=1, validation_data=([val_x, val_x, val_x], [val_y, val_y, val_y]), epochs=5) ntest_x, test_y = process_data(x_test, y_test) predictions = ensemble_model.predict([ntest_x, ntest_x, ntest_x]) correct = np.equal(np.argmax(predictions, 1), np.argmax(test_y, 1)) accuracy = np.mean(correct)