def inference(input_shape, N_CLASSES): '''build the model args: image: image batch, 4D tensor [batch_size, width, height, channels=3], dtype=tf.float32 return: output tensor with the computed logits, float, [batch_size, n_classes] ''' inputs = Input(shape=input_shape) K.set_learning_phase( False) # all new operations will be in test mode from now on ## Conv layer 1 x = dn_layer(inputs=inputs, name='layer1') x = MaxPooling2D((2, 2), strides=(2, 2), name='layer1_maxpool')(x) ## Conv layer 2 x = dn_layer(inputs=x, num_filters=32, name='layer2') x = MaxPooling2D((2, 2), strides=(2, 2), name='layer2_maxpool')(x) ## Conv layer 3 x = dn_layer(inputs=x, num_filters=64, name='layer3') x = MaxPooling2D((2, 2), strides=(2, 2), name='layer3_maxpool')(x) x = GlobalAveragePooling2D(name='GlobalAveragePooling2D')(x) #x = Flatten(name='flatten')(x) x = Dense(N_CLASSES, activation='softmax', name='predictions')(x) model = Model(inputs=inputs, outputs=x) return model
def _call_graph_fn(self, features, labels=None): """Calls graph function. Args: features: `Tensor` or `dict` of tensors labels: `Tensor` or `dict` of tensors """ set_learning_phase(Modes.is_train(self.mode)) kwargs = {} if 'labels' in get_arguments(self._graph_fn): kwargs['labels'] = labels return self._graph_fn(mode=self.mode, features=features, **kwargs)
def build(self,alpha, img_input, temp_softmax): shape = (1, 1, int(1024 * alpha)) """ This looks dangerous. Not sure how the model would get affected with the laarning_phase variable set to True. """ K.set_learning_phase(True) with tf.name_scope('teacher') as scope: self.conv1 = Conv2D( int(32*alpha), (3,3), padding='same', use_bias=False, strides=(1,1), name='teacher_conv1', trainable=self.trainable)(img_input) self.conv2 = BatchNormalization(axis=-1, name='teacher_conv1_bn', trainable=self.trainable)(self.conv1) self.conv3 = Activation(self.relu6, name='teacher_conv1_relu', trainable=self.trainable)(self.conv2) self.conv4 = self._depthwise_conv_block(self.conv3, 64, alpha, depth_multiplier, block_id = 15) self.conv5 = self._depthwise_conv_block(self.conv4, 128, alpha, depth_multiplier,strides=(2, 2), block_id =16) self.conv6 =self. _depthwise_conv_block(self.conv5, 128, alpha, depth_multiplier,block_id =17) self.conv7 = self._depthwise_conv_block(self.conv6, 256, alpha, depth_multiplier, strides=(2,2),block_id =18) self.conv8 = self._depthwise_conv_block(self.conv7, 256, alpha, depth_multiplier, block_id =19) self.conv9 = self._depthwise_conv_block(self.conv8, 512, alpha, depth_multiplier, strides = (2,2), block_id =20) self.conv10 = self._depthwise_conv_block(self.conv9, 512, alpha, depth_multiplier, block_id =21) self.conv11 = self._depthwise_conv_block(self.conv10, 512, alpha, depth_multiplier, block_id =22) self.conv12 = self._depthwise_conv_block(self.conv11, 512, alpha, depth_multiplier, block_id =23) self.conv13 = self._depthwise_conv_block(self.conv12, 512, alpha, depth_multiplier, block_id =24) self.conv14 = self._depthwise_conv_block(self.conv13, 512, alpha, depth_multiplier, block_id =25) self.conv15 = self._depthwise_conv_block(self.conv14, 1024, alpha, depth_multiplier,strides=(2,2), block_id =26) self.conv16 = self._depthwise_conv_block(self.conv15, 1024, alpha, depth_multiplier, block_id =27) self.conv17 = GlobalAveragePooling2D()(self.conv16) self.conv18 = Reshape(shape, name='teacher_reshape_1', trainable=self.trainable)(self.conv17) self.conv19 = Dropout(0.5, name='teacher_dropout', trainable=self.trainable)(self.conv18) self.conv20 = Conv2D(self.num_classes, (1, 1), padding='same', name='teacher_conv_preds', trainable=self.trainable)(self.conv18) self.conv21 = Activation('softmax', name='teacher_act_softmax', trainable=self.trainable)(tf.divide(self.conv20, temp_softmax)) self.conv22 = Reshape((self.num_classes,), name='teacher_reshape_2', trainable=self.trainable)(self.conv21) return self
def _call_graph_fn(self, features, labels=None): """Calls graph function. Creates first one or two graph, i.e. train and target graphs. Return the optimal action given an exploration policy. If `is_dueling` is set to `True`, then another layer is added that represents the state value. Args: inputs: `Tensor` or `dict` of tensors """ set_learning_phase(Modes.is_train(self.mode)) graph_fn = self._build_graph_fn() self._graph_results = graph_fn(mode=self.mode, features=features, labels=labels) return self._build_actions()
def _call_graph_fn(self, features, labels=None): """Calls graph function. Creates first one or two graph, i.e. train and target graphs. Return the optimal action given an exploration policy. If `is_dueling` is set to `True`, then another layer is added that represents the state value. Args: features: `Tensor` or `dict` of tensors labels: `Tensor` or `dict` of tensors """ set_learning_phase(Modes.is_train(self.mode)) graph_fn = self._build_graph_fn() if self.use_target_graph: # We create 2 graphs: a training graph and a target graph, # so that we can copy one graph to another given a frequency. self._train_graph = FunctionModule(mode=self.mode, build_fn=graph_fn, name='train') self._train_results = self._train_graph(features=features, labels=labels) self._target_graph = FunctionModule(mode=self.mode, build_fn=graph_fn, name='target') self._target_results = self._target_graph(features=features, labels=labels) return self._build_actions() else: self._train_results = graph_fn(mode=self.mode, features=features, labels=labels) self._target_results = self._train_results return self._build_actions()
# https://www.tensorflow.org/extend/estimators from __future__ import absolute_import from __future__ import division from __future__ import print_function # numpy import numpy as np # tensorflow import tensorflow as tf # keras from tensorflow.contrib.keras.python.keras.layers import Dense, Conv2D, Dropout, MaxPooling2D, Flatten from tensorflow.contrib.keras.python.keras import backend as K K.set_learning_phase(1) #set learning phase # input data from tensorflow.examples.tutorials.mnist import input_data # estimators from tensorflow.contrib import learn # estimator "builder" from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib import time def time_usage(func): def wrapper(*args, **kwargs): beg_ts = time.time()
def load_inference_model(model_path): K.set_learning_phase(0) return load_model(model_path)
if __name__ == '__main__': batch_size = 16 nb_epoch = 400 optimizer = 'adam' log_dir = './keras_logs/simple11' old_session = K.get_session() X_train, y_train, X_test, y_test = load_data() datagen = ImageDataGenerator(rotation_range=15, zoom_range=0.20) datagen.fit(X_train) with tf.Graph().as_default(): session = tf.Session('') K.set_session(session) K.set_learning_phase(1) es_cb = callbacks.EarlyStopping(monitor='val_loss', patience=200, verbose=0, mode='auto') lr_cb = callbacks.LearningRateScheduler( lambda epoch: float(learning_rates[epoch])) model = mk_model_with_bn() model.summary() model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
def build(self, rgb, num_classes, temp_softmax, train_mode): K.set_learning_phase(True) # conv1_1 with tf.name_scope('mentor_conv1_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 3, 64], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(rgb, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') # out = tf.nn.bias_add(conv, biases) # mean , var = tf.nn.moments(out, axes= [0]) # out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv1_1 = tf.nn.relu(conv, name=scope) self.conv1_1 = BatchNormalization(axis=-1, name='mentor_bn_conv1_1')( self.conv1_1) #self.conv1_1 = Dropout((0.4))(self.conv1_1) self.parameters += [kernel, biases] with tf.name_scope('mentor_conv1_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 64], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.conv1_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv1_2 = tf.nn.relu(conv, name=scope) self.conv1_2 = BatchNormalization(axis=-1, name='mentor_bn_conv1_2')( self.conv1_2) self.parameters += [kernel, biases] self.pool1 = tf.nn.max_pool(self.conv1_2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='mentor_pool1') with tf.name_scope('mentor_conv2_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.pool1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv2_1 = tf.nn.relu(conv, name=scope) self.conv2_1 = BatchNormalization(axis=-1, name='mentor_bn_conv2_1')( self.conv2_1) #self.conv2_1 = Dropout((0.4))(self.conv2_1) self.parameters += [kernel, biases] with tf.name_scope('mentor_conv2_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 128], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.conv2_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv2_2 = tf.nn.relu(conv, name=scope) self.conv2_2 = BatchNormalization(axis=-1, name='mentor_bn_conv2_2')( self.conv2_2) self.parameters += [kernel, biases] self.pool2 = tf.nn.max_pool(self.conv2_2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='mentor_pool2') with tf.name_scope('mentor_conv3_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 256], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.pool2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) # mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv3_1 = tf.nn.relu(conv, name=scope) self.conv3_1 = BatchNormalization(axis=-1, name='mentor_bn_conv3_1')( self.conv3_1) #self.conv3_1 = Dropout((0.4))(self.conv3_1) self.parameters += [kernel, biases] with tf.name_scope('mentor_conv3_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.conv3_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv3_2 = tf.nn.relu(conv, name=scope) self.conv3_2 = BatchNormalization(axis=-1, name='mentor_bn_conv3_2')( self.conv3_2) #self.conv3_2 = Dropout((0.4))(self.conv3_2) self.parameters += [kernel, biases] with tf.name_scope('mentor_conv3_3') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.conv3_2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv3_3 = tf.nn.relu(conv, name=scope) self.conv3_3 = BatchNormalization(axis=-1, name='mentor_bn_conv3_3')( self.conv3_3) self.parameters += [kernel, biases] self.pool3 = tf.nn.max_pool(self.conv3_3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='mentor_pool3') with tf.name_scope('mentor_conv4_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 512], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.pool3, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv4_1 = tf.nn.relu(conv, name=scope) self.conv4_1 = BatchNormalization(axis=-1, name='mentor_bn_conv4_1')( self.conv4_1) #self.conv4_1 = Dropout((0.4))(self.conv4_1) self.parameters += [kernel, biases] # conv5_1 with tf.name_scope('mentor_conv4_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.conv4_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv4_2 = tf.nn.relu(conv, name=scope) self.conv4_2 = BatchNormalization(axis=-1, name='mentor_bn_conv4_2')( self.conv4_2) #self.conv4_2 = Dropout((0.4))(self.conv4_2) self.parameters += [kernel, biases] with tf.name_scope('mentor_conv4_3') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.conv4_2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv4_3 = tf.nn.relu(conv, name=scope) self.conv4_3 = BatchNormalization(axis=-1, name='mentor_bn_conv4_3')( self.conv4_3) self.parameters += [kernel, biases] self.pool4 = tf.nn.max_pool(self.conv4_3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='mentor_pool4') with tf.name_scope('mentor_conv5_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.pool4, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv5_1 = tf.nn.relu(conv, name=scope) self.conv5_1 = BatchNormalization(axis=-1, name='mentor_bn_conv5_1')( self.conv5_1) #self.conv5_1 = Dropout((0.4))(self.conv5_1) self.parameters += [kernel, biases] with tf.name_scope('mentor_conv5_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.conv5_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv5_2 = tf.nn.relu(conv, name=scope) self.conv5_2 = BatchNormalization( axis=-1, name='mentor_batch_norm_conv5_2')(self.conv5_2) #self.conv5_2 = Dropout((0.4))(self.conv5_2) self.parameters += [kernel, biases] with tf.name_scope('mentor_conv5_3') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') conv = tf.nn.conv2d(self.conv5_2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') #out = tf.nn.bias_add(conv, biases) #mean , var = tf.nn.moments(out, axes= [0]) #out = (out - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.conv5_3 = tf.nn.relu(conv, name=scope) self.conv5_3 = BatchNormalization( axis=-1, name='mentor_batch_norm_conv5_3')(self.conv5_3) self.parameters += [kernel, biases] self.pool5 = tf.nn.max_pool(self.conv5_3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='mentor_pool5') # fc1 with tf.name_scope('mentor_fc1') as scope: shape = int(np.prod(self.pool5.get_shape()[1:])) fc1w = tf.Variable(tf.truncated_normal([shape, 4096], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') fc1b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') pool5_flat = tf.reshape(self.pool5, [-1, shape]) fc1l = tf.nn.bias_add(tf.matmul(pool5_flat, fc1w), fc1b) #mean , var = tf.nn.moments(fc1l, axes= [0]) #fc1l = (fc1l - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.fc1 = tf.nn.relu(fc1l) self.fc1 = BatchNormalization(axis=-1, name='mentor_batch_norm_fc1')( self.fc1) # self.fc1 = Dropout((0.4))(self.fc1) #self.fc1 = tf.nn.dropout(self.fc1, 0.5) self.parameters += [fc1w, fc1b] with tf.name_scope('mentor_fc2') as scope: fc2w = tf.Variable(tf.truncated_normal([4096, 4096], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') fc2b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') fc2l = tf.nn.bias_add(tf.matmul(self.fc1, fc2w), fc2b) #mean , var = tf.nn.moments(fc2l, axes= [0]) #fc2l = (fc2l - mean)/tf.sqrt(var + tf.Variable(1e-10)) self.fc2 = tf.nn.relu(fc2l) self.fc2 = BatchNormalization(axis=-1, name='mentor_batch_norm_fc2')( self.fc2) if train_mode == True: self.fc2 = tf.nn.dropout(self.fc2, 0.5) self.parameters += [fc2w, fc2b] with tf.name_scope('mentor_fc3') as scope: fc3w = tf.Variable(tf.truncated_normal([4096, num_classes], dtype=tf.float32, stddev=1e-2), trainable=self.trainable, name='mentor_weights') fc3b = tf.Variable(tf.constant(1.0, shape=[num_classes], dtype=tf.float32), trainable=self.trainable, name='mentor_biases') self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b) #self.fc3l = tf.nn.relu(fc3l) self.parameters += [fc3w, fc3b] self.softmax = tf.nn.softmax(self.fc3l / temp_softmax) return self
def from_config(cls, mode, features, labels, config): # pylint: disable=arguments-differ """Instantiates a Graph container from its config (output of `get_config()`). Arguments: mode: features: labels: config: Model config dictionary. Returns: A model instance. Raises: ValueError: In case of improperly formatted config dict. """ # set the training mode set_learning_phase(Modes.is_train(mode)) if not isinstance(config, GraphConfig): config = GraphConfig.from_dict(config) # layer instances created during # the graph reconstruction process created_layers = {} # Create an input layer based on the defined inputs and features for layer in config.input_layers: layer_name, node_index, tensor_index = cls.get_node_data(layer) if layer_name in features: created_layers[layer_name] = InputLayer( input_tensor=features[layer_name], name=layer_name) elif isinstance(labels, Mapping) and layer_name in labels: created_layers[layer_name] = InputLayer( input_tensor=labels[layer_name], name=layer_name) else: raise ConfigurationError( "Input `{}`is not found".format(layer_name)) def process_layer(layer): """Deserialize a layer, then call it on appropriate inputs. Arguments: layer_data: layer config dict. Raises: ValueError: In case of improperly formatted `layer_data` dict. """ layer_class = layer.IDENTIFIER layer_name = layer.name # Instantiate layer. if layer_class in LAYERS: created_layer = LAYERS[layer_class].from_config(layer) elif layer_class in IMAGE_PROCESSORS: created_layer = IMAGE_PROCESSORS[layer_class].from_config( layer) else: raise ValueError( "The layer `{}` is not supported.".format(layer_class)) created_layers[layer_name] = created_layer # Gather layer inputs. inbound_nodes_data = layer.inbound_nodes input_tensors = [] for input_data in inbound_nodes_data: in_layer_name, in_node_index, in_tensor_index = cls.get_node_data( input_data) if len(input_data) == 3: kwargs = {} elif len(input_data) == 4: kwargs = input_data[3] else: raise ValueError('Improperly formatted model config.') if in_layer_name not in created_layers: raise ValueError('Missing layer: ' + in_layer_name) inbound_layer = created_layers[in_layer_name] inbound_node = inbound_layer.inbound_nodes[in_node_index] input_tensors.append( inbound_node.output_tensors[in_tensor_index]) # Call layer on its inputs, thus creating the node # and building the layer if needed. if input_tensors: if len(input_tensors) == 1: created_layer(input_tensors[0], **kwargs) else: created_layer(input_tensors, **kwargs) for layer in config.layers: process_layer(layer) name = config.name input_tensors = [] output_tensors = [] for layer_data in config.input_layers: layer_name, node_index, tensor_index = cls.get_node_data( layer_data) assert layer_name in created_layers, "Layer `{}` not found".format( layer_name) layer = created_layers[layer_name] layer_output_tensors = layer.inbound_nodes[ node_index].output_tensors input_tensors.append(layer_output_tensors[tensor_index]) for layer_data in config.output_layers: layer_name, node_index, tensor_index = cls.get_node_data( layer_data) assert layer_name in created_layers layer = created_layers[layer_name] layer_output_tensors = layer.inbound_nodes[ node_index].output_tensors output_tensors.append(layer_output_tensors[tensor_index]) return cls(inputs=input_tensors, outputs=output_tensors, name=name)
def build(self, rgb, num_classes, temp_softmax, seed,train_mode): K.set_learning_phase(True) # conv1_1 with tf.name_scope('mentee_conv1_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, self.num_channels, 64], dtype=tf.float32, stddev=1e-2, seed = seed), trainable = self.trainable, name='mentee_weights') conv = tf.nn.conv2d(rgb, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable= self.trainable, name='mentee_biases') out = tf.nn.bias_add(conv, biases) #out = self.extra_regularization(out) self.conv1_1 = tf.nn.relu(out, name=scope) #self.conv1_1 = BatchNormalization(axis = -1, name= 'mentee_bn_conv1_1')(self.conv1_1) self.parameters += [kernel, biases] self.pool1 = tf.nn.max_pool(self.conv1_1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1') #conv2_1 with tf.name_scope('mentee_conv2_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32, stddev=1e-2, seed = seed), trainable = self.trainable, name='mentee_weights') conv = tf.nn.conv2d(self.pool1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable= self.trainable, name='mentee_biases') out = tf.nn.bias_add(conv, biases) self.conv2_1 = tf.nn.relu(out, name=scope) #self.conv2_1 = BatchNormalization(axis = -1, name= 'mentee_bn_conv2_1')(self.conv2_1) self.parameters += [kernel, biases] self.pool2 = tf.nn.max_pool(self.conv2_1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2') with tf.name_scope('mentee_conv3_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 256], dtype=tf.float32, stddev=1e-2, seed = seed), trainable = self.trainable, name='mentee_weights') conv = tf.nn.conv2d(self.pool2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable= self.trainable, name='mentee_biases') out = tf.nn.bias_add(conv, biases) self.conv3_1 = tf.nn.relu(out, name=scope) #self.conv3_1 = BatchNormalization(axis = -1, name= 'mentee_bn_conv3_1')(self.conv3_1) self.parameters += [kernel, biases] self.pool3 = tf.nn.max_pool(self.conv3_1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool3') with tf.name_scope('mentee_conv4_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 512], dtype=tf.float32, stddev=1e-2, seed= seed), trainable = self.trainable, name='mentee_weights') conv = tf.nn.conv2d(self.pool3, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentee_biases') out = tf.nn.bias_add(conv, biases) self.conv4_1 = tf.nn.relu(out, name=scope) #self.conv4_1 = BatchNormalization(axis = -1, name= 'mentee_bn_conv4_1')(self.conv4_1) self.parameters += [kernel, biases] self.pool4 = tf.nn.max_pool(self.conv4_1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool4') with tf.name_scope('mentee_conv5_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-2, seed = seed), trainable = self.trainable, name='mentee_weights') conv = tf.nn.conv2d(self.pool4, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentee_biases') out = tf.nn.bias_add(conv, biases) self.conv5_1 = tf.nn.relu(out, name=scope) #self.conv5_1 = BatchNormalization(axis = -1, name= 'mentee_bn_conv5_1')(self.conv5_1) self.parameters += [kernel, biases] self.pool5 = tf.nn.max_pool(self.conv5_1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool5') with tf.name_scope('mentee_conv6_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-2, seed = seed), trainable = self.trainable, name='mentee_weights') conv = tf.nn.conv2d(self.pool5, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=self.trainable, name='mentee_biases') out = tf.nn.bias_add(conv, biases) self.conv6_1 = tf.nn.relu(out, name=scope) #self.conv6_1 = BatchNormalization(axis = -1, name= 'mentee_bn_conv6_1')(self.conv6_1) self.parameters += [kernel, biases] self.pool6 = tf.nn.max_pool(self.conv6_1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool6') # fc1 with tf.name_scope('mentee_fc1') as scope: shape = int(np.prod(self.pool6.get_shape()[1:])) fc1w = tf.Variable(tf.truncated_normal([shape, 4096], dtype=tf.float32, stddev=1e-2, seed = seed), trainable = self.trainable,name='mentee_weights') fc1b = tf.Variable(tf.constant(0.0, shape=[4096], dtype=tf.float32), trainable=self.trainable, name='mentee_biases') pool6_flat = tf.reshape(self.pool6, [-1, shape]) fc1l = tf.nn.bias_add(tf.matmul(pool6_flat, fc1w), fc1b) self.fc1 = tf.nn.relu(fc1l) #self.fc1 = BatchNormalization(axis = -1, name= 'mentee_bn_fc1')(self.fc1) #if train_mode == True: print("Traine_mode is true") #self.fc1 = tf.nn.dropout(self.fc1, 0.5, seed = seed) self.parameters += [fc1w, fc1b] with tf.name_scope('mentee_fc2') as scope: fc2w = tf.Variable(tf.truncated_normal([4096, 4096], dtype=tf.float32, stddev=1e-2, seed = seed), trainable = self.trainable,name='mentee_weights') fc2b = tf.Variable(tf.constant(0.0, shape=[4096], dtype=tf.float32), trainable=self.trainable, name='mentee_biases') fc2l = tf.nn.bias_add(tf.matmul(self.fc1, fc2w), fc2b) self.fc2 = tf.nn.relu(fc2l) #self.fc2 = BatchNormalization(axis = -1, name= 'mentee_bn_fc2')(self.fc2) """ Dropout and BatchNormalization are added to regularize the network and perform better by generalizing well. However, to demonstrate knowledge transfer effectiveness, no other regularizers are added. """ #if train_mode == True: #self.fc2 = tf.nn.dropout(self.fc2, 0.5, seed = seed) self.parameters += [fc2w, fc2b] with tf.name_scope('mentee_fc3') as scope: fc3w = tf.Variable(tf.truncated_normal([4096, num_classes], dtype=tf.float32, stddev=1e-2, seed = seed), trainable = self.trainable,name='mentee_weights') fc3b = tf.Variable(tf.constant(0.0, shape=[num_classes], dtype=tf.float32), trainable=self.trainable, name='mentee_biases') self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b) #self.fc3 = tf.nn.relu(fc3l) self.parameters += [fc3w, fc3b] self.softmax = tf.nn.softmax(self.fc3l/temp_softmax) return self
os.chdir(r'D:/drivers/') from tensorflow.contrib.keras.python.keras.models import Sequential from tensorflow.contrib.keras.python.keras.layers.core import Dense, Dropout, Flatten from tensorflow.contrib.keras.python.keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D from tensorflow.contrib.keras.python.keras.layers.pooling import AveragePooling2D from tensorflow.contrib.keras.python.keras.layers import InputLayer from tensorflow.contrib.keras.python.keras.models import load_model from tensorflow.contrib.keras.python.keras import optimizers from tensorflow.contrib.keras.python.keras.callbacks import EarlyStopping from tensorflow.contrib.keras.python.keras.callbacks import ModelCheckpoint from tensorflow.contrib.keras.python.keras.callbacks import History from tensorflow.contrib.keras.python.keras import backend as K from tensorflow.contrib.keras.python.keras.layers.core import Lambda import tensorflow as tf K.set_learning_phase(0) def load_image(filepath, size): """"args: filepath, image size Uses OpenCV package to transform image into numerical format. Returns: list of shape size*size*3 """ image = cv2.imread(filepath) res = cv2.resize(image, (size, size), cv2.INTER_LINEAR) return res def load_train_data(size, fp='D:\\drivers\\train'):
def cnn_sentiment_model(inputs, nb_words, embedding_dim=300, static_embedding=True, embedding_weights=None, filter_hs=None, nb_filters=100, emb_size=100, hidden_dropout=0.2, is_training=True, augmentation_function=None, l2_weight=1e-4, img_shape=None, new_shape=None, image_summary=False, batch_norm_decay=0.99, seed=0, embedding_dropout=0.2): from tensorflow.contrib.keras.python.keras.layers import Embedding, Input, Convolution1D, MaxPooling1D, Flatten, \ Dense, Dropout, Activation from tensorflow.contrib.keras.python.keras.initializers import glorot_uniform from tensorflow.contrib.keras.python.keras.layers.merge import Concatenate from tensorflow.contrib.keras.python.keras import backend as K K.set_learning_phase(1 if is_training else 0) sequence_length = img_shape[0] if filter_hs is None: filter_hs = [3, 4, 5] model = inputs def ci(shape, dtype=None, partition_info=None): assert shape[0] == embedding_weights.shape[0] and shape[ 1] == embedding_weights.shape[ 1], 'Shapes are not equal required={} init value={}'.format( shape, embedding_weights.shape) return embedding_weights model = Embedding(nb_words, embedding_dim, input_length=sequence_length, trainable=(not static_embedding), embeddings_initializer='uniform' if embedding_weights is None else ci)(model) if embedding_dropout > 0.0: model = Dropout(embedding_dropout, seed=seed)(model, training=is_training) convs = list() for fsz in filter_hs: conv = Convolution1D( filters=nb_filters, kernel_size=fsz, padding='valid', activation='relu', kernel_initializer=glorot_uniform(seed=seed))(model) pool = MaxPooling1D(pool_size=sequence_length - fsz + 1)(conv) flatten = Flatten()(pool) convs.append(flatten) if len(filter_hs) > 0: graph_out = Concatenate()(convs) else: graph_out = convs[0] model = graph_out model = Dense(emb_size, kernel_initializer=glorot_uniform(seed=seed))(model) model = Dropout(hidden_dropout, seed=seed)(model, training=is_training) model = Activation('relu')(model) return model