def call(self, x): """ Apply the layer to the node embeddings in x. These embeddings are either: * A list of two tensors of shape (N, M) being the embeddings for each of the nodes in the link, where N is the number of links, and M is the node embedding size. * A single tensor of shape (..., N, 2, M) where the axis second from last indexes the nodes in the link and N is the number of links and M the embedding size. """ # Currently GraphSAGE & HinSage output a list of two tensors being the embeddings # for each of the nodes in the link. However, GCN, GAT & other full-batch methods # return a tensor of shape (1, N, 2, M). # Detect and support both inputs if isinstance(x, (list, tuple)): if len(x) != 2: raise ValueError( "Expecting a list of length 2 for link embedding") x0, x1 = x elif isinstance(x, tf.Tensor): if int(x.shape[self.axis]) != 2: raise ValueError( "Expecting a tensor of shape 2 along specified axis for link embedding" ) x0, x1 = tf.unstack(x, axis=self.axis) else: raise TypeError("Expected a list, tuple, or Tensor as input") # Apply different ways to combine the node embeddings to a link embedding. if self.method in ["ip", "dot"]: out = tf.reduce_sum(x0 * x1, axis=-1, keepdims=True) elif self.method == "l1": # l1(u,v)_i = |u_i - v_i| - vector of the same size as u,v out = tf.abs(x0 - x1) elif self.method == "l2": # l2(u,v)_i = (u_i - v_i)^2 - vector of the same size as u,v out = tf.square(x0 - x1) elif self.method in ["mul", "hadamard"]: out = tf.multiply(x0, x1) elif self.method == "concat": out = Concatenate()([x0, x1]) elif self.method == "avg": out = Average()([x0, x1]) else: raise NotImplementedError() # "{}: the requested method '{}' is not known/not implemented".format( # name, edge_embedding_method # ) # ) # Apply activation function out = self.activation(out) return out
def create_mu_block(x_list, config): if config.n_feature_sets > 1: if config.last_layer == 'concatenate': x = Concatenate()(x_list) else: # pad first if prorated max_len = np.max(config.feature_units) # print("Max len {}".format(max_len)) for i, a in enumerate(x_list): # print(i, a) if (max_len - config.feature_units[i]) > 0: left_pad = (int)((max_len - config.feature_units[i]) / 2) right_pad = max_len - config.feature_units[i] - left_pad x_list[i] = tf.pad(a, [[0, 0], [left_pad, right_pad]]) x = Average()(x_list) if config.task == 'regression': x = Dense(1)(x) elif config.task == 'classification': x = Dense(config.n_classes, activation='softmax')(x) else: if config.task == 'regression': x = Dense(1)(x_list[0]) elif config.task == 'classification': x = Dense(config.n_classes, activation='softmax')(x_list[0]) return x
def create_model(image_shape=(224, 224, 3), restart_checkpoint=None, backbone='mobilnetv2', feature_len=128, freeze=False): """ Creates an image encoder. Args: image_shape: input image shape (use [None, None] for resizable network) restart_checkpoint: snapshot to be restored backbone: the backbone CNN (one of mobilenetv2, densent121, custom) feature_len: the length of the additional feature layer freeze: freeze the backbone """ input_img = Input(shape=image_shape) # add the backbone backbone_name = backbone if backbone_name == 'densenet121': print('Using DenseNet121 backbone.') backbone = DenseNet121(input_tensor=input_img, include_top=False) backbone.layers.pop() if freeze: for layer in backbone.layers: layer.trainable = False backbone = backbone.output elif backbone_name == 'mobilenetv2': print('Using MobileNetV2 backbone.') backbone = MobileNetV2(input_tensor=input_img, include_top=False) backbone.layers.pop() if freeze: for layer in backbone.layers: layer.trainable = False backbone = backbone.output elif backbone_name == 'custom': backbone = custom_backbone(input_tensor=input_img) else: raise Exception('Unknown backbone: {}'.format(backbone_name)) # add the head layers gmax = GlobalMaxPool2D()(backbone) gavg = GlobalAvgPool2D()(backbone) gmul = Multiply()([gmax, gavg]) ggavg = Average()([gmax, gavg]) backbone = Concatenate()([gmax, gavg, gmul, ggavg]) backbone = BatchNormalization()(backbone) backbone = Dense(feature_len)(backbone) backbone = Activation('sigmoid')(backbone) encoder = Model(input_img, backbone) if restart_checkpoint: print('Loading weights from {}'.format(restart_checkpoint)) encoder.load_weights(restart_checkpoint, by_name=True, skip_mismatch=True) return encoder
def UnetPlusPlus_mobilenetv2(config, num_classes, weights, base_trainable): base_model, preprocessing_function = classification_networks[ 'mobilenetv2'](input_shape=config.input_shape, weights=weights, include_top=False) base_model.trainable = base_trainable img_height = config.height img_width = config.width conv1_1 = base_model.get_layer('block_1_expand_relu').output conv2_1 = base_model.get_layer('block_3_expand_relu').output conv3_1 = base_model.get_layer('block_6_expand_relu').output conv4_1 = base_model.get_layer('block_13_expand_relu').output conv5_1 = base_model.get_layer('out_relu').output conv1_2, conv1_3, conv1_4, conv1_5 = ret_model_output( config, conv1_1, conv2_1, conv3_1, conv4_1, conv5_1) output_list = get_output_list(config, num_classes, conv1_2, conv1_3, conv1_4, conv1_5) o = 1 if (len(output_list) > 1): o = Average()(output_list) else: o = output_list[0] o = Upsample(img_height, img_width)(o) model = Model(inputs=base_model.input, outputs=o, name='UnetPlusPlus_mobilenetv2') return model, preprocessing_function
def build_CNN_ensemble_model(params): """ Builds ensemble of FFCNN's Parameters ---------- params : dict complete hyper parameter dictionary for model generation and training Required Parameters ---------- ensemble_size : integer Number of feed forward CNNs based on params to include in ensemble Returns ------- model """ x_in = Input(shape=params['data_shape'], name='X') members = [] for i in range(params['ensemble_size']): m = build_CNN_model(params) members.append(m(x_in)) outputs = Average()(members) model = Model(x_in, outputs) return model
def edge_function(x): x0 = x[0] x1 = x[1] if edge_embedding_method == "ip" or edge_embedding_method == "dot": out = Lambda( lambda x: K.sum(x[0] * x[1], axis=-1, keepdims=False))( [x0, x1]) out = Activation(output_act)(out) if output_dim != 1: warnings.warn( "Inner product is a scalar, but output_dim is set to {}. Reverting output_dim to be 1." .format(output_dim)) out = Reshape((1, ))(out) elif edge_embedding_method == "l1": # l1(u,v)_i = |u_i - v_i| - vector of the same size as u,v le = Lambda(lambda x: K.abs(x[0] - x[1]))([x0, x1]) # add dense layer to convert le to the desired output: out = Dense(output_dim, activation=output_act)(le) out = Reshape((output_dim, ))(out) elif edge_embedding_method == "l2": # l2(u,v)_i = (u_i - v_i)^2 - vector of the same size as u,v le = Lambda(lambda x: K.square(x[0] - x[1]))([x0, x1]) # add dense layer to convert le to the desired output: out = Dense(output_dim, activation=output_act)(le) out = Reshape((output_dim, ))(out) elif edge_embedding_method == "mul" or edge_embedding_method == "hadamard": le = Multiply()([x0, x1]) # add dense layer to convert le to the desired output: out = Dense(output_dim, activation=output_act)(le) out = Reshape((output_dim, ))(out) elif edge_embedding_method == "concat": le = Concatenate()([x0, x1]) # add dense layer to convert le to the desired output: out = Dense(output_dim, activation=output_act)(le) out = Reshape((output_dim, ))(out) elif edge_embedding_method == "avg": le = Average()([x0, x1]) # add dense layer to convert le to the desired output: out = Dense(output_dim, activation=output_act)(le) out = Reshape((output_dim, ))(out) else: raise NotImplementedError( "{}: the requested method '{}' is not known/not implemented". format(name, edge_embedding_method)) if clip_limits: out = LeakyClippedLinear(low=clip_limits[0], high=clip_limits[1], alpha=0.1)(out) return out
def average(models: List[training.Model], model_input: Tensor) -> training.Model: outputs = [model.outputs[0] for model in models] y = Average()(outputs) model = Model(model_input, y, name='ensemble') return model
def classify(**args): """ Main method that prepares dataset, builds model, executes training and displays results. :param args: keyword arguments passed from cli parser """ # only allow print-outs if execution has no repetitions allow_print = args['repetitions'] == 1 # determine classification targets and parameters to construct datasets properly cls_target, cls_str = set_classification_targets(args['cls_choice']) d = prepare_dataset(args['dataset_choice'], cls_target, args['batch_size']) print('\n\tTask: Classify «{}» using «{}»\n'.format( cls_str, d['data_str'])) print_dataset_info(d) # build and train inputs = Input(shape=(7810, )) models = [ build_model(i, d['num_classes'], inputs=inputs) for i in range(args['num_models']) ] # combine outputs of all models y = Average()([m.outputs[0] for m in models]) model = Model(inputs, outputs=y, name='multiple') model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) if allow_print: model.summary() print('') plot_model(model, to_file='img/multiple_mlp.png') model.fit(d['train_data'], steps_per_epoch=d['train_steps'], epochs=args['epochs'], verbose=1, class_weight=d['class_weights']) # evaluation model print('Evaluate ...') model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.evaluate(d['eval_data'], steps=d['test_steps'], verbose=1) # predict on testset and calculate classification report and confusion matrix for diagnosis print('Test ...') pred = model.predict(d['test_data'], steps=d['test_steps']) if allow_print: diagnose_output(d['test_labels'], pred.argmax(axis=1), d['classes_trans']) return balanced_accuracy_score(d['test_labels'], pred.argmax(axis=1))
def deep_fuse_layer(inputs, filters, kernel_size, strides): if type(inputs) is not list: raise Exception() avg = Average()(inputs) conv_list = [] for _ in range(len(inputs)): conv_list.append(create_res_conv_block(avg, filters, kernel_size, strides)) return conv_list
def fusion_block(tensors, name, type='add'): if (type == 'add'): return Add(name='add_' + name)(tensors) if (type == 'max'): return Maximum(name='max_' + name)(tensors) if (type == 'con'): return Concatenate(name='conc_' + name)(tensors) if (type == 'avg'): return Average(name='avg_' + name)(tensors)
def call(self, claim, evidence): if self.merging_type == 'concat': new_row = Concatenate(axis=1)([claim, evidence]) elif self.merging_type == 'sum': new_row = Add()([claim, evidence]) elif self.merging_type == 'mean': new_row = Average()([claim, evidence]) if self.cosine_sim_flag: cosine_sim = Dot(1, normalize=True)([claim, evidence]) new_row = Concatenate(axis=1)([new_row, cosine_sim]) return new_row
def discriminator_model(): dm_input = Input(shape=patch_shape) dm = Convolution2D(filters=64, kernel_size=(3, 3), strides=(2, 2), padding="same")(dm_input) dm = BatchNormalization()(dm) dm = LeakyReLU(alpha=0.2)(dm) dm = Convolution2D(filters=2 * 64, kernel_size=(3, 3), strides=(2, 2), padding="same")(dm) dm = BatchNormalization()(dm) dm = LeakyReLU(alpha=0.2)(dm) dm = Convolution2D(filters=4 * 64, kernel_size=(3, 3), strides=(2, 2), padding="same")(dm) dm = BatchNormalization()(dm) dm = LeakyReLU(alpha=0.2)(dm) dm = Convolution2D(filters=4 * 64, kernel_size=(3, 3), strides=(2, 2), padding="same")(dm) dm = BatchNormalization()(dm) dm = LeakyReLU(alpha=0.2)(dm) dm = Flatten()(dm) outputs = Dense(units=1, activation='sigmoid')(dm) model = Model(inputs=dm_input, outputs=outputs, name='Discriminator_Model') # discriminator inputs = Input(shape=image_shape) list_row_idx = [(64 * i, (i + 1) * 64) for i in range(int(image_shape[0] / patch_shape[0]))] list_col_idx = [(64 * i, (i + 1) * 64) for i in range(int(image_shape[1] / patch_shape[1]))] list_patch = [] for row_idx in list_row_idx: for col_idx in list_col_idx: x_patch = Lambda(lambda z: z[:, row_idx[0]:row_idx[1], col_idx[0]: col_idx[1], :])(inputs) list_patch.append(x_patch) x = [model(patch) for patch in list_patch] outputs = Average()(x) model = Model(inputs=inputs, outputs=outputs, name='Discriminator') return model
def call(self, inputs): attention_layers = [] # apply all attention layers to inputs for layer in self.attn_heads: attention_layers.append(layer(inputs)) # now average all their outputs if self.num_heads > 1: return Average()(attention_layers) else: return attention_layers[0]
def build_extractor_29layers_v2(self, name, block, layers): in_img = Input(shape=(*self.in_size_hw, 1)) X = self._mfm(in_img, name = name + '_mfm1', out_channels=48, kernel_size=5, strides=1) X = Average()([MaxPooling2D(pool_size=2, padding='same')(X), AveragePooling2D(pool_size=2, padding='same')(X)]) X = self._make_layer(X, name = name + '_layers1', block=block, num_blocks=layers[0], out_channels=48) X = self._group(X, name = name + '_group1', in_channels=48, out_channels=96, kernel_size=3, strides=1) X = Average()([MaxPooling2D(pool_size=2, padding='same')(X), AveragePooling2D(pool_size=2, padding='same')(X)]) X = self._make_layer(X, name = name + '_layers2', block=block, num_blocks=layers[1], out_channels=96) X = self._group(X, name = name + '_group2', in_channels=96, out_channels=192, kernel_size=3, strides=1) X = Average()([MaxPooling2D(pool_size=2, padding='same')(X), AveragePooling2D(pool_size=2, padding='same')(X)]) X = self._make_layer(X, name = name + '_layers3', block=block, num_blocks=layers[2], out_channels=192) X = self._group(X, name = name + '_group3', in_channels=192, out_channels=128, kernel_size=3, strides=1) X = self._make_layer(X, name = name + '_layers4', block=block, num_blocks=layers[3], out_channels=128) X = self._group(X, name = name + '_group4', in_channels=128, out_channels=128, kernel_size=3, strides=1) X = Average()([MaxPooling2D(pool_size=2, padding='same')(X), AveragePooling2D(pool_size=2, padding='same')(X)]) feat = Dense(256, name = name + '_dense1', kernel_regularizer=regularizers.l2(0.0005))(Flatten()(X)) ret_extractor = Model(inputs=in_img, outputs=feat, name=name) ret_extractor.summary() return ret_extractor
def __tail__(self, x_2, x_1, x, output_size, dropout=False): x_2 = GlobalAveragePooling2D()(x_2) x_1 = GlobalAveragePooling2D()(x_1) x = GlobalAveragePooling2D()(x) if dropout: x_2 = Dropout(.5)(x_2) x_1 = Dropout(.5)(x_1) x = Dropout(.5)(x) x_2 = Dense(output_size, activation='softmax', name='out_2')(x_2) x_1 = Dense(output_size, activation='softmax', name='out_1')(x_1) x = Dense(output_size, activation='softmax', name='out')(x) softmax_avg = Average()([x_2, x_1, x]) return softmax_avg
def define_ensemble_model(models): # Wrap each model so that names are different and do not collide. models = [ Model(inputs=model.input, outputs=model.output, name=f"{model.name}_{model_idx}") for model_idx, model in enumerate(models) ] input = Input(IMAGE_SIZE + (3, )) ensemble_outputs = [model(input) for model in models] avg = Average()(ensemble_outputs) model = Model(inputs=input, outputs=avg) return model
def create_model(backbone, input_shape, lr, metrics): base = create_model_base(backbone, input_shape) # define the 2 inputs (left and right eyes) left_input = Input(shape=input_shape) right_input = Input(shape=input_shape) # get the 2 outputs using shared layers out_left = base(left_input) out_right = base(right_input) # average the predictions merged = Average()([out_left, out_right]) model = Model(inputs=[right_input, left_input], outputs=merged) model.compile(loss='binary_crossentropy', optimizer=Adam(lr=lr), metrics=metrics) model.summary() return model
def build(self, **kwargs) -> Model: opticalflow_model = kwargs['OpticalflowModel'] rgb_model = kwargs['RGBModel'] opticalflow_input = Input(shape=self.opticalflow_input_shape, name='opticalflow_inputs') rgb_input = Input(shape=self.rgb_input_shape, name='rgb_inputs') opticalflow_target = opticalflow_model(opticalflow_input) rgb_target = rgb_model(rgb_input) # average opticalflow and rgb models outputs outputs = Average(name='nsdm_output')([opticalflow_target, rgb_target]) # ensemble model nsdm_model = Model(inputs=[opticalflow_input, rgb_input], outputs=outputs, name='nsdm_model') return nsdm_model
def nn_model(): i = Input(shape=(15, )) # Branch 1 x1 = Dense(128, activation='relu')(i) x1 = BatchNormalization()(x1) x1 = Dense(64, activation='relu')(x1) x1 = BatchNormalization()(x1) x1 = Dense(32, activation='relu')(x1) x1 = BatchNormalization()(x1) x1 = Dense(16, activation='relu')(x1) x1 = BatchNormalization()(x1) x1_output = Dense(1, activation='linear')(x1) # Branch 2 x2 = Dense(128, activation='relu')(i) x2 = BatchNormalization()(x2) x2 = Dense(64, activation='relu')(x2) x2 = BatchNormalization()(x2) x2 = Dense(32, activation='relu')(x2) x2 = BatchNormalization()(x2) x2 = Dense(16, activation='relu')(x2) x2 = BatchNormalization()(x2) x2_output = Dense(1, activation='linear')(x2) # Branch 3 x3 = Dense(128, activation='relu')(i) x3 = BatchNormalization()(x3) x3 = Dense(64, activation='relu')(x3) x3 = BatchNormalization()(x3) x3 = Dense(32, activation='relu')(x3) x3 = BatchNormalization()(x3) x3 = Dense(16, activation='relu')(x3) x3 = BatchNormalization()(x3) x3_output = Dense(1, activation='linear')(x3) scalars = Average()([x1_output, x2_output, x3_output]) return Model(inputs=[i], outputs=[scalars])
def create_model(pretrained_model, num_labels, output_offset, layer_index): model_inputs = pretrained_model.inputs[:2] if is_signed_digit(layer_index): layer_index = int(layer_index) pretrained_output = get_bert_output(pretrained_model, layer_index, output_offset) elif layer_index in ('avg', 'concat'): outputs = [] for i in count(1): try: outputs.append( get_bert_output(pretrained_model, i, output_offset)) except ValueError: break # assume past last layer if layer_index == 'avg': pretrained_output = Average()(outputs) else: assert layer_index == 'concat' pretrained_output = Concatenate()(outputs) model_output = keras.layers.Dense(num_labels, activation='softmax')(pretrained_output) model = keras.models.Model(inputs=model_inputs, outputs=model_output) return model
def create_model(model_type, dataset, validation_dataset, test_dataset, callbacks, batch_size, num_epochs, multi_modal, augmented, perform_test_only, perform_early_fusion, perform_late_fusion, pe_block, path_prefix): time_callback = TimeHistory() input_shape = (vol_x, vol_y, vol_z, n_channels) mirrored_strategy = tf.distribute.MirroredStrategy() with mirrored_strategy.scope(): print_section("Creating and compiling " + model_type + " model") inputs_bmode, inputs_pd = handle_input_fusion(multi_modal, input_shape, batch_size) if multi_modal and perform_late_fusion: # Treat as separate single modal models model_bmode = get_model(model_type, False, False, pe_block, inputs_bmode, None, True) model_pd = get_model(model_type, False, False, pe_block, inputs_pd, None, True) output_conv = Average()([model_bmode, model_pd]) output = Conv3D(1, 1, activation='sigmoid', padding='same')(output_conv) model = Model(inputs=[inputs_bmode, inputs_pd], outputs=[output]) else: model = get_model(model_type, multi_modal, perform_early_fusion, pe_block, inputs_bmode, inputs_pd) model.compile(optimizer=Adam(lr=learning_rate, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, decay=decay), loss=dice_loss, metrics=["binary_crossentropy", dice_coe]) print_section("Generating model summary") model.summary() if not perform_test_only: print_section("Generate model graph image") plot_model(model, to_file=generate_model_image_path(path_prefix), show_shapes=True, show_layer_names=True, rankdir='LR', expand_nested=False, dpi=96) print_section("Training model") num_train_samples = train_samples_total if augmented else train_samples model.fit(dataset, epochs=num_epochs, verbose=1, shuffle=True, steps_per_epoch=calculated_steps_per_epoch( num_train_samples, batch_size), validation_data=validation_dataset, validation_steps=calculated_steps_per_epoch( validation_samples, batch_size), validation_freq=1, callbacks=[time_callback] + callbacks) print_section("Loading model weights") model.load_weights(generate_checkpoint_path(path_prefix)) print_section('Testing model') imgs_mask_test = model.predict(test_dataset, batch_size=batch_size, verbose=1) print_section('Saving predictions') pred_label_list = save_test_images(imgs_mask_test, path_prefix) print_section('Evaluating against ground truth') test_df = pd.read_csv(os.path.join(dataset_output_path, 'test.csv')) ground_truth_label_list = test_df['label'].values.tolist() perf_metrics = compare_segmentations(pred_label_list, ground_truth_label_list) print_section('Saving results') create_results_csv(path_prefix, perf_metrics, list(zip(pred_label_list, ground_truth_label_list)), sum(time_callback.times)) if not perform_test_only: print_section("Printing stats") print("Each epoch time", time_callback.times) print("Total Time Taken (s)", sum(time_callback.times))
def get_test_model_exhaustive(): """Returns a exhaustive test model.""" input_shapes = [(2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10), (7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15), (14, 15), (16, ), (16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4), (1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4), (1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3), (4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3), (9, 16, 1), (1, 9, 16)] inputs = [Input(shape=s) for s in input_shapes] outputs = [] outputs.append(Conv1D(1, 3, padding='valid')(inputs[6])) outputs.append(Conv1D(2, 1, padding='same')(inputs[6])) outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6])) outputs.append(ZeroPadding1D(2)(inputs[6])) outputs.append(Cropping1D((2, 3))(inputs[6])) outputs.append(MaxPooling1D(2)(inputs[6])) outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6])) outputs.append(MaxPooling1D(2, data_format="channels_first")(inputs[6])) outputs.append(AveragePooling1D(2)(inputs[6])) outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6])) outputs.append( AveragePooling1D(2, data_format="channels_first")(inputs[6])) outputs.append(GlobalMaxPooling1D()(inputs[6])) outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6])) outputs.append(GlobalAveragePooling1D()(inputs[6])) outputs.append( GlobalAveragePooling1D(data_format="channels_first")(inputs[6])) outputs.append(Conv2D(4, (3, 3))(inputs[4])) outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4])) outputs.append( Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4])) outputs.append( Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4])) outputs.append(SeparableConv2D(3, (3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((1, 2))(inputs[4])) outputs.append(MaxPooling2D((2, 2))(inputs[4])) # todo: check if TensorFlow >= 2.1 supports this # outputs.append(MaxPooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default MaxPoolingOp only supports NHWC on device type CPU outputs.append( MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4])) outputs.append(AveragePooling2D((2, 2))(inputs[4])) # todo: check if TensorFlow >= 2.1 supports this # outputs.append(AveragePooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default AvgPoolingOp only supports NHWC on device type CPU outputs.append( AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4])) outputs.append(GlobalAveragePooling2D()(inputs[4])) outputs.append( GlobalAveragePooling2D(data_format="channels_first")(inputs[4])) outputs.append(GlobalMaxPooling2D()(inputs[4])) outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4])) outputs.append(Permute((3, 4, 1, 5, 2))(inputs[0])) outputs.append(Permute((1, 5, 3, 2, 4))(inputs[0])) outputs.append(Permute((3, 4, 1, 2))(inputs[2])) outputs.append(Permute((2, 1, 3))(inputs[4])) outputs.append(Permute((2, 1))(inputs[6])) outputs.append(Permute((1, ))(inputs[8])) outputs.append(Permute((3, 1, 2))(inputs[31])) outputs.append(Permute((3, 1, 2))(inputs[32])) outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[31]))) outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[32]))) outputs.append(BatchNormalization()(inputs[0])) outputs.append(BatchNormalization(axis=1)(inputs[0])) outputs.append(BatchNormalization(axis=2)(inputs[0])) outputs.append(BatchNormalization(axis=3)(inputs[0])) outputs.append(BatchNormalization(axis=4)(inputs[0])) outputs.append(BatchNormalization(axis=5)(inputs[0])) outputs.append(BatchNormalization()(inputs[2])) outputs.append(BatchNormalization(axis=1)(inputs[2])) outputs.append(BatchNormalization(axis=2)(inputs[2])) outputs.append(BatchNormalization(axis=3)(inputs[2])) outputs.append(BatchNormalization(axis=4)(inputs[2])) outputs.append(BatchNormalization()(inputs[4])) # todo: check if TensorFlow >= 2.1 supports this # outputs.append(BatchNormalization(axis=1)(inputs[4])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now. outputs.append(BatchNormalization(axis=2)(inputs[4])) outputs.append(BatchNormalization(axis=3)(inputs[4])) outputs.append(BatchNormalization()(inputs[6])) outputs.append(BatchNormalization(axis=1)(inputs[6])) outputs.append(BatchNormalization(axis=2)(inputs[6])) outputs.append(BatchNormalization()(inputs[8])) outputs.append(BatchNormalization(axis=1)(inputs[8])) outputs.append(BatchNormalization()(inputs[27])) outputs.append(BatchNormalization(axis=1)(inputs[27])) outputs.append(BatchNormalization()(inputs[14])) outputs.append(BatchNormalization(axis=1)(inputs[14])) outputs.append(BatchNormalization(axis=2)(inputs[14])) outputs.append(BatchNormalization()(inputs[16])) # todo: check if TensorFlow >= 2.1 supports this # outputs.append(BatchNormalization(axis=1)(inputs[16])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now. outputs.append(BatchNormalization(axis=2)(inputs[16])) outputs.append(BatchNormalization(axis=3)(inputs[16])) outputs.append(BatchNormalization()(inputs[18])) outputs.append(BatchNormalization(axis=1)(inputs[18])) outputs.append(BatchNormalization(axis=2)(inputs[18])) outputs.append(BatchNormalization(axis=3)(inputs[18])) outputs.append(BatchNormalization(axis=4)(inputs[18])) outputs.append(BatchNormalization()(inputs[20])) outputs.append(BatchNormalization(axis=1)(inputs[20])) outputs.append(BatchNormalization(axis=2)(inputs[20])) outputs.append(BatchNormalization(axis=3)(inputs[20])) outputs.append(BatchNormalization(axis=4)(inputs[20])) outputs.append(BatchNormalization(axis=5)(inputs[20])) outputs.append(Dropout(0.5)(inputs[4])) outputs.append(ZeroPadding2D(2)(inputs[4])) outputs.append(ZeroPadding2D((2, 3))(inputs[4])) outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4])) outputs.append(Cropping2D(2)(inputs[4])) outputs.append(Cropping2D((2, 3))(inputs[4])) outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4])) outputs.append(Dense(3, use_bias=True)(inputs[13])) outputs.append(Dense(3, use_bias=True)(inputs[14])) outputs.append(Dense(4, use_bias=False)(inputs[16])) outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18])) outputs.append(Dense(4, use_bias=False)(inputs[20])) outputs.append(Reshape(((2 * 3 * 4 * 5 * 6), ))(inputs[0])) outputs.append(Reshape((2, 3 * 4 * 5 * 6))(inputs[0])) outputs.append(Reshape((2, 3, 4 * 5 * 6))(inputs[0])) outputs.append(Reshape((2, 3, 4, 5 * 6))(inputs[0])) outputs.append(Reshape((2, 3, 4, 5, 6))(inputs[0])) outputs.append(Reshape((16, ))(inputs[8])) outputs.append(Reshape((2, 8))(inputs[8])) outputs.append(Reshape((2, 2, 4))(inputs[8])) outputs.append(Reshape((2, 2, 2, 2))(inputs[8])) outputs.append(Reshape((2, 2, 1, 2, 2))(inputs[8])) outputs.append(RepeatVector(3)(inputs[8])) outputs.append( UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4])) outputs.append( UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4])) outputs.append( UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4])) outputs.append( UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4])) outputs.append(ReLU()(inputs[0])) for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]: outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]])) for axis in [-4, -3, -2, -1, 1, 2, 3, 4]: outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]])) for axis in [-3, -2, -1, 1, 2, 3]: outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]])) for axis in [-2, -1, 1, 2]: outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]])) for axis in [-1, 1]: outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]])) for axis in [-1, 2]: outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]])) for axis in [-1, 3]: outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]])) for axis in [-1, 4]: outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]])) for axis in [-1, 5]: outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]])) outputs.append(UpSampling1D(size=2)(inputs[6])) # outputs.append(UpSampling1D(size=2)(inputs[8])) # ValueError: Input 0 of layer up_sampling1d_1 is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 16] outputs.append(Multiply()([inputs[10], inputs[11]])) outputs.append(Multiply()([inputs[11], inputs[10]])) outputs.append(Multiply()([inputs[11], inputs[13]])) outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]])) outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]])) shared_conv = Conv2D(1, (1, 1), padding='valid', name='shared_conv', activation='relu') up_scale_2 = UpSampling2D((2, 2)) x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8) x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8) x3 = Conv2D(1, (1, 1), padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8) x = Concatenate()([x1, x2, x3]) # (3, 8, 8) outputs.append(x) x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8) outputs.append(x) x = Dropout(0.5)(x) outputs.append(x) x = Concatenate()([MaxPooling2D((2, 2))(x), AveragePooling2D((2, 2))(x)]) # (6, 4, 4) outputs.append(x) x = Flatten()(x) # (1, 1, 96) x = Dense(4, use_bias=False)(x) outputs.append(x) x = Dense(3)(x) # (1, 1, 3) outputs.append(x) outputs.append(Add()([inputs[26], inputs[30], inputs[30]])) outputs.append(Subtract()([inputs[26], inputs[30]])) outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]])) outputs.append(Average()([inputs[26], inputs[30], inputs[30]])) outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]])) outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]])) intermediate_input_shape = (3, ) intermediate_in = Input(intermediate_input_shape) intermediate_x = intermediate_in intermediate_x = Dense(8)(intermediate_x) intermediate_x = Dense(5, name='duplicate_layer_name')(intermediate_x) intermediate_model = Model(inputs=[intermediate_in], outputs=[intermediate_x], name='intermediate_model') intermediate_model.compile(loss='mse', optimizer='nadam') x = intermediate_model(x) # (1, 1, 5) intermediate_model_2 = Sequential() intermediate_model_2.add(Dense(7, input_shape=(5, ))) intermediate_model_2.add(Dense(5, name='duplicate_layer_name')) intermediate_model_2.compile(optimizer='rmsprop', loss='categorical_crossentropy') x = intermediate_model_2(x) # (1, 1, 5) intermediate_model_3_nested = Sequential() intermediate_model_3_nested.add(Dense(7, input_shape=(6, ))) intermediate_model_3_nested.compile(optimizer='rmsprop', loss='categorical_crossentropy') intermediate_model_3 = Sequential() intermediate_model_3.add(Dense(6, input_shape=(5, ))) intermediate_model_3.add(intermediate_model_3_nested) intermediate_model_3.add(Dense(8)) intermediate_model_3.compile(optimizer='rmsprop', loss='categorical_crossentropy') x = intermediate_model_3(x) # (1, 1, 8) x = Dense(3)(x) # (1, 1, 3) shared_activation = Activation('tanh') outputs = outputs + [ Activation('tanh')(inputs[25]), Activation('hard_sigmoid')(inputs[25]), Activation('selu')(inputs[25]), Activation('sigmoid')(inputs[25]), Activation('softplus')(inputs[25]), Activation('softmax')(inputs[25]), Activation('relu')(inputs[25]), Activation('relu6')(inputs[25]), Activation('swish')(inputs[25]), Activation('exponential')(inputs[25]), Activation('gelu')(inputs[25]), Activation('softsign')(inputs[25]), LeakyReLU()(inputs[25]), ReLU()(inputs[25]), ReLU(max_value=0.4, negative_slope=1.1, threshold=0.3)(inputs[25]), ELU()(inputs[25]), PReLU()(inputs[24]), PReLU()(inputs[25]), PReLU()(inputs[26]), shared_activation(inputs[25]), Activation('linear')(inputs[26]), Activation('linear')(inputs[23]), x, shared_activation(x), ] model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive') model.compile(loss='mse', optimizer='nadam') # fit to dummy data training_data_size = 2 data_in = generate_input_data(training_data_size, input_shapes) initial_data_out = model.predict(data_in) data_out = generate_output_data(training_data_size, initial_data_out) model.fit(data_in, data_out, epochs=10) return model
import tensorflow as tf import tensorflow.keras.backend as K from tensorflow.keras.layers import Add, Activation, Concatenate, Conv1D, Conv2D, Conv3D, \ Input, Subtract, Multiply, Average, Maximum, Minimum, LeakyReLU, PReLU, ELU, ThresholdedReLU, Softmax, \ ZeroPadding1D, ZeroPadding2D, ZeroPadding3D, Reshape DEFAULT_TF_GRAPH = tf.get_default_graph() @pytest.mark.parametrize('layer, input_layer, batch_size', [ (Concatenate(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Add(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Add(), [Input((6, 6, 5)), Input((1, 1, 5))], 32), (Subtract(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Multiply(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Average(), [Input( (6, 6, 5)), Input( (6, 6, 5)), Input((6, 6, 5))], 32), (Maximum(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Minimum(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Activation('relu'), Input((6, 6, 5)), 32), (Reshape((36, 5)), Input((6, 6, 5)), 32), (Reshape((20, 6, 5)), Input((6, 5, 4, 5)), 32), (GatherLayer([1, 4, 2], axis=1), Input((6, 5)), 32), (GatherLayer([1, 4, 2], axis=2), Input((6, 5)), 32), (GatherLayer([1, 4, 2], axis=2), Input((6, 7, 5)), 32), (LeakyReLU(), Input((6, 6, 5)), 32), (ELU(), Input((6, 6, 5)), 32), (ThresholdedReLU(), Input((6, 6, 5)), 32), (Softmax(), Input((6, 6, 5)), 32), (DownShiftLayer(), Input((6, 6, 5)), 32),
conv_3_flow = Conv2D(512, (3, 3), strides=1, activation='relu')(pool_2_flow) #Layer4 conv_4_flow = Conv2D(512, (3, 3), strides=1, activation='relu')(conv_3_flow) #Layer5 conv_5_flow = Conv2D(512, (3, 3), strides=1, activation='relu')(conv_4_flow) pool_3_flow = MaxPooling2D((2, 2))(conv_5_flow) flat_flow = Flatten()(pool_3_flow) #Layer6 fc_1_flow = Dense(4096, activation='relu')(flat_flow) #Layer7 fc_2_flow = Dense(2048, activation='relu')(fc_1_flow) #output layer out_flow = Dense(101, activation='softmax')(fc_2_flow) #Taking the output of both the streams and combining them out = Average()([out_frames, out_flow]) model = Model(inputs=[inp_frames, inp_flow], outputs=out) opti_flow = tf.keras.optimizers.Adam(learning_rate=1e-5) #compiling the model by using categorical_crossentropy loss model.compile(optimizer=opti_flow, loss='categorical_crossentropy', metrics=['mae', 'accuracy']) model.summary() #visualizing the model on tensorboard tensorboard = TensorBoard(log_dir="logs\{}".format(time()), write_graph=True) #calling the datagenerator and passing the inputs to our model for training i = 0 hist_frames = []
def get_test_model_exhaustive(): """Returns a exhaustive test model.""" input_shapes = [ (2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10), (7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15), (14, 15), (16, ), (16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4), (1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4), (1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3), (4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3), ] inputs = [Input(shape=s) for s in input_shapes] outputs = [] outputs.append(Conv1D(1, 3, padding='valid')(inputs[6])) outputs.append(Conv1D(2, 1, padding='same')(inputs[6])) outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6])) outputs.append(ZeroPadding1D(2)(inputs[6])) outputs.append(Cropping1D((2, 3))(inputs[6])) outputs.append(MaxPooling1D(2)(inputs[6])) outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6])) outputs.append(AveragePooling1D(2)(inputs[6])) outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6])) outputs.append(GlobalMaxPooling1D()(inputs[6])) outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6])) outputs.append(GlobalAveragePooling1D()(inputs[6])) outputs.append( GlobalAveragePooling1D(data_format="channels_first")(inputs[6])) outputs.append(Conv2D(4, (3, 3))(inputs[4])) outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4])) outputs.append( Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4])) outputs.append( Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4])) outputs.append(SeparableConv2D(3, (3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((1, 2))(inputs[4])) outputs.append(MaxPooling2D((2, 2))(inputs[4])) outputs.append( MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4])) outputs.append(AveragePooling2D((2, 2))(inputs[4])) outputs.append( AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4])) outputs.append(GlobalAveragePooling2D()(inputs[4])) outputs.append( GlobalAveragePooling2D(data_format="channels_first")(inputs[4])) outputs.append(GlobalMaxPooling2D()(inputs[4])) outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4])) outputs.append(BatchNormalization()(inputs[4])) outputs.append(Dropout(0.5)(inputs[4])) outputs.append(ZeroPadding2D(2)(inputs[4])) outputs.append(ZeroPadding2D((2, 3))(inputs[4])) outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4])) outputs.append(Cropping2D(2)(inputs[4])) outputs.append(Cropping2D((2, 3))(inputs[4])) outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4])) outputs.append(Dense(3, use_bias=True)(inputs[13])) outputs.append(Dense(3, use_bias=True)(inputs[14])) outputs.append(Dense(4, use_bias=False)(inputs[16])) outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18])) outputs.append(Dense(4, use_bias=False)(inputs[20])) outputs.append( UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4])) outputs.append( UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4])) outputs.append( UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4])) outputs.append( UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4])) for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]: outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]])) for axis in [-4, -3, -2, -1, 1, 2, 3, 4]: outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]])) for axis in [-3, -2, -1, 1, 2, 3]: outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]])) for axis in [-2, -1, 1, 2]: outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]])) for axis in [-1, 1]: outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]])) for axis in [-1, 2]: outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]])) for axis in [-1, 3]: outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]])) for axis in [-1, 4]: outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]])) for axis in [-1, 5]: outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]])) outputs.append(UpSampling1D(size=2)(inputs[6])) outputs.append(Multiply()([inputs[10], inputs[11]])) outputs.append(Multiply()([inputs[11], inputs[10]])) outputs.append(Multiply()([inputs[11], inputs[13]])) outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]])) outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]])) shared_conv = Conv2D(1, (1, 1), padding='valid', name='shared_conv', activation='relu') up_scale_2 = UpSampling2D((2, 2)) x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8) x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8) x3 = Conv2D(1, (1, 1), padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8) x = Concatenate()([x1, x2, x3]) # (3, 8, 8) outputs.append(x) x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8) outputs.append(x) x = Dropout(0.5)(x) outputs.append(x) x = Concatenate()([MaxPooling2D((2, 2))(x), AveragePooling2D((2, 2))(x)]) # (6, 4, 4) outputs.append(x) x = Flatten()(x) # (1, 1, 96) x = Dense(4, use_bias=False)(x) outputs.append(x) x = Dense(3)(x) # (1, 1, 3) outputs.append(x) outputs.append(Add()([inputs[26], inputs[30], inputs[30]])) outputs.append(Subtract()([inputs[26], inputs[30]])) outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]])) outputs.append(Average()([inputs[26], inputs[30], inputs[30]])) outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]])) outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]])) intermediate_input_shape = (3, ) intermediate_in = Input(intermediate_input_shape) intermediate_x = intermediate_in intermediate_x = Dense(8)(intermediate_x) intermediate_x = Dense(5)(intermediate_x) intermediate_model = Model(inputs=[intermediate_in], outputs=[intermediate_x], name='intermediate_model') intermediate_model.compile(loss='mse', optimizer='nadam') x = intermediate_model(x) # (1, 1, 5) intermediate_model_2 = Sequential() intermediate_model_2.add(Dense(7, input_shape=(5, ))) intermediate_model_2.add(Dense(5)) intermediate_model_2.compile(optimizer='rmsprop', loss='categorical_crossentropy') x = intermediate_model_2(x) # (1, 1, 5) x = Dense(3)(x) # (1, 1, 3) shared_activation = Activation('tanh') outputs = outputs + [ Activation('tanh')(inputs[25]), Activation('hard_sigmoid')(inputs[25]), Activation('selu')(inputs[25]), Activation('sigmoid')(inputs[25]), Activation('softplus')(inputs[25]), Activation('softmax')(inputs[25]), Activation('relu')(inputs[25]), LeakyReLU()(inputs[25]), ELU()(inputs[25]), PReLU()(inputs[24]), PReLU()(inputs[25]), PReLU()(inputs[26]), shared_activation(inputs[25]), Activation('linear')(inputs[26]), Activation('linear')(inputs[23]), x, shared_activation(x), ] model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive') model.compile(loss='mse', optimizer='nadam') # fit to dummy data training_data_size = 1 data_in = generate_input_data(training_data_size, input_shapes) initial_data_out = model.predict(data_in) data_out = generate_output_data(training_data_size, initial_data_out) model.fit(data_in, data_out, epochs=10) return model
def build_xnet(use_backbone, backbone, classes, skip_connection_layers, decoder_filters=(256, 128, 64, 32), upsample_rates=(2, 2, 2, 2), n_upsample_blocks=4, block_type='upsampling', activation='sigmoid', input_shape=(None, None, 3), use_batchnorm=True, attention=False, deep_supervision=False): downterm = [None] * (n_upsample_blocks + 1) dsterm = [None] * (n_upsample_blocks) interm = [[None] * (n_upsample_blocks - i + 1) for i in range(n_upsample_blocks + 1)] if block_type == 'transpose': up_block = Transpose2D_block else: up_block = Upsample2D_block # Using Backbone for the Encoder if use_backbone: input = backbone.input downsampling_layers = skip_connection_layers output = backbone.output if 'vgg' not in backbone.name else backbone.layers[ -2].output # convert layer names to indices downsampling_idx = ([ get_layer_number(backbone, l) if isinstance(l, str) else l for l in downsampling_layers ]) downsampling_list = [ backbone.layers[downsampling_idx[i]].output for i in range(len(downsampling_idx)) ] for i in range(len(downsampling_idx)): downterm[n_upsample_blocks - i - 1] = downsampling_list[i] downterm[-1] = output # Using Conv+relu for the Encoder else: encoder_filters = (decoder_filters[0] * 2, ) + decoder_filters input = Input(shape=input_shape) for i in range(n_upsample_blocks + 1): if i == 0: x = conv_block(encoder_filters[n_upsample_blocks - i], i, 0, amount=2, use_batchnorm=use_batchnorm, type_block='encoder')(input) else: down_rate = to_tuple(upsample_rates[n_upsample_blocks - i - 1]) x = MaxPool2D(pool_size=down_rate, name='encoder_stage{}-0_maxpool'.format(i))(x) x = conv_block(encoder_filters[n_upsample_blocks - i], i, 0, amount=2, use_batchnorm=use_batchnorm, type_block='encoder')(x) downterm[i] = x skip_connection_layers = tuple([i.name for i in downterm]) output = downterm[-1] for i in range(len(skip_connection_layers)): interm[i][0] = downterm[i] interm[-1][0] = output down_rate = to_tuple(upsample_rates[-1]) for j in range(n_upsample_blocks): for i in range(n_upsample_blocks - j): upsample_rate = to_tuple(upsample_rates[n_upsample_blocks - i - 1]) if deep_supervision and i == 0: dsterm[j - 1] = interm[i][j] if attention: interm[i][j] = attention_block( decoder_filters[n_upsample_blocks - i - 1], interm[i][j], i, j)(interm[i + 1][j]) if interm[i][j + 1] is None: interm[i][j + 1] = up_block( decoder_filters[n_upsample_blocks - i - 1], i, j + 1, upsample_rate=upsample_rate, skip=interm[i][:j + 1], use_batchnorm=use_batchnorm)(interm[i + 1][j]) # Deep Supervision if deep_supervision and n_upsample_blocks > 1: # Currently only VGG or not using backbone dsterm[-1] = interm[0][-1] x = DeepSupervision(classes)(dsterm) x = Average(name='average_ds')(x) else: if use_backbone and 'vgg' not in backbone.name: x = up_block(decoder_filters[-1], 0, n_upsample_blocks + 1, upsample_rate=down_rate, use_batchnorm=use_batchnorm)(interm[0][-1]) else: x = interm[0][-1] x = Conv2D(classes, (3, 3), padding='same', name='final_conv')(x) x = Activation(activation, name=activation)(x) return Model(input, x)
def __init__(self, config, latent_code_garms_sz=1024, garmparams_sz=config.PCA_, name=None): super(PoseShapeOffsetModel, self).__init__(name=name) self.config = config self.latent_code_garms_sz = latent_code_garms_sz self.garmparams_sz = garmparams_sz self.latent_code_betas_sz = 128 # ToDo: Minor: Remove hard coded colors. Should be same as rendered colors in input self.colormap = tf.cast([ np.array([255, 255, 255]), np.array([65, 0, 65]), np.array([0, 65, 65]), np.array([145, 65, 0]), np.array([145, 0, 65]), np.array([0, 145, 65]) ], tf.float32) / 255. with open('assets/hresMapping.pkl', 'rb') as f: _, self.faces = pkl.load(f, encoding="latin1") self.faces = np.int32(self.faces) # Define network layers self.top_ = SingleImageNet(self.latent_code_garms_sz, self.latent_code_betas_sz) for n in self.config.garmentKeys: gn = GarmentNet(self.config.PCA_, n, self.garmparams_sz) self.garmentModels.append(gn) self.smpl = SMPL('assets/neutral_smpl.pkl', theta_in_rodrigues=False, theta_is_perfect_rotmtx=False, isHres=True, scale=True) self.smpl_J = SmplBody25Layer(theta_in_rodrigues=False, theta_is_perfect_rotmtx=False, isHres=True) self.J_layers = [NameLayer('J_{}'.format(i)) for i in range(NUM)] self.lat_betas = Dense(self.latent_code_betas_sz, kernel_initializer=initializers.RandomNormal( 0, 0.00005), activation='relu') self.betas = Dense(10, kernel_initializer=initializers.RandomNormal( 0, 0.000005), name='betas') init_trans = np.array([0, 0.2, -2.]) init_pose = np.load('assets/mean_a_pose.npy') init_pose[:3] = 0 init_pose = tf.reshape( batch_rodrigues(init_pose.reshape(-1, 3).astype(np.float32)), (-1, )) self.pose_trans = tf.concat((init_pose, init_trans), axis=0) self.lat_pose = Dense(self.latent_code_betas_sz, kernel_initializer=initializers.RandomNormal( 0, 0.000005), activation='relu') self.lat_pose_layer = Dense( 24 * 3 * 3 + 3, kernel_initializer=initializers.RandomNormal(0, 0.000005), name='pose_trans') self.cut_trans = Lambda(lambda z: z[:, -3:]) self.trans_layers = [ NameLayer('trans_{}'.format(i)) for i in range(NUM) ] self.cut_poses = Lambda(lambda z: z[:, :-3]) self.reshape_pose = Reshape((24, 3, 3)) self.pose_layers = [NameLayer('pose_{}'.format(i)) for i in range(NUM)] # Optional: Condition garment on betas, probably not self.latent_code_offset_ShapeMerged = Dense(self.latent_code_garms_sz + self.latent_code_betas_sz, activation='relu') self.latent_code_offset_ShapeMerged_2 = Dense( self.latent_code_garms_sz + self.latent_code_betas_sz, activation='relu', name='latent_code_offset_ShapeMerged') self.avg = Average() self.flatten = Flatten() self.concat = Concatenate() self.scatters = [] for vs in self.vertSpread: self.scatters.append(Scatter_(vs, self.config.NVERTS))
def unetpp(multi_modal, early_fusion, project_excite, inputs_bmode, inputs_pd, cascade=False): set_image_data_format('channels_last') multi_stage_fusion = multi_modal and not early_fusion # Input if multi_modal and early_fusion: conv, pool, pool_b = fusion_encoder_block(n_filters[0] // 2, multi_modal, project_excite, inputs_bmode, inputs_pd) conv0_0, pool0, pool0_b = fusion_encoder_block(n_filters[0], multi_stage_fusion, project_excite, conv, None) else: conv0_0, pool0, pool0_b = fusion_encoder_block(n_filters[0], multi_stage_fusion, project_excite, inputs_bmode, inputs_pd) # Fused Encode Block 1 conv1_0, pool1, pool1_b = fusion_encoder_block(n_filters[1], multi_stage_fusion, project_excite, pool0, pool0_b) # Dense Conv Block 0_1 up0_1 = concatenate([ Conv3DTranspose(n_filters[0], 2, strides=(2, 2, 2), padding='same')(conv1_0), conv0_0 ], axis=-1) conv0_1 = conv_block(up0_1, n_filters[0]) # Fused Encode Block 2 conv2_0, pool2, pool2_b = fusion_encoder_block(n_filters[2], multi_stage_fusion, project_excite, pool1, pool1_b) # Dense Conv Block 1_1 up1_1 = concatenate([ Conv3DTranspose(n_filters[1], 2, strides=(2, 2, 2), padding='same')(conv2_0), conv1_0 ], axis=-1) conv1_1 = conv_block(up1_1, n_filters[1]) # Dense Conv Block 0_2 up0_2 = concatenate([ Conv3DTranspose(n_filters[0], 2, strides=(2, 2, 2), padding='same')(conv1_1), conv0_1 ], axis=-1) conv0_2 = conv_block(up0_2, n_filters[0]) # Fused Encode Block 3 conv3_0, pool3, pool3_b = fusion_encoder_block(n_filters[3], multi_stage_fusion, project_excite, pool2, pool2_b) # Dense Conv Block 2_1 up2_1 = concatenate([ Conv3DTranspose(n_filters[2], 2, strides=(2, 2, 2), padding='same')(conv3_0), conv2_0 ], axis=-1) conv2_1 = conv_block(up2_1, n_filters[2]) # Dense Conv Block 1_2 up1_2 = concatenate([ Conv3DTranspose(n_filters[1], 2, strides=(2, 2, 2), padding='same')(conv2_1), conv1_1 ], axis=-1) conv1_2 = conv_block(up1_2, n_filters[1]) # Dense Conv Block 0_3 up0_3 = concatenate([ Conv3DTranspose(n_filters[0], 2, strides=(2, 2, 2), padding='same')(conv1_2), conv0_2 ], axis=-1) conv0_3 = conv_block(up0_3, n_filters[0]) # Bottom Conv Block conv4_0 = conv_block(pool3, n_filters[4], project_excite) # Decode stage 1 up3_1 = concatenate([ Conv3DTranspose(n_filters[3], 2, strides=(2, 2, 2), padding='same')(conv4_0), conv3_0 ], axis=-1) conv3_1 = conv_block(up3_1, n_filters[3], project_excite) # Decode stage 2 up2_2 = concatenate([ Conv3DTranspose(n_filters[2], 2, strides=(2, 2, 2), padding='same')(conv3_1), conv2_1 ], axis=-1) conv2_2 = conv_block(up2_2, n_filters[2], project_excite) # Decode stage 3 up1_3 = concatenate([ Conv3DTranspose(n_filters[1], 2, strides=(2, 2, 2), padding='same')(conv2_2), conv1_2 ], axis=-1) conv1_3 = conv_block(up1_3, n_filters[1], project_excite) # Decode stage 4 up0_4 = concatenate([ Conv3DTranspose(n_filters[0], 2, strides=(2, 2, 2), padding='same')(conv1_3), conv0_3 ], axis=-1) conv0_4 = conv_block(up0_4, n_filters[0], project_excite) if cascade: return Average()([conv0_1, conv0_2, conv0_3, conv0_4]) # Average all segmentation branches from top row output_1 = Conv3D(1, 1, activation='sigmoid', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(conv0_1) output_2 = Conv3D(1, 1, activation='sigmoid', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(conv0_2) output_3 = Conv3D(1, 1, activation='sigmoid', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(conv0_3) output_4 = Conv3D(1, 1, activation='sigmoid', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(conv0_4) # If training with deep_supervision, average all skip connection convolution layers in the top row if deep_supervision: output = Average()([output_1, output_2, output_3, output_4]) else: output = output_4 if multi_modal: model = Model(inputs=[inputs_bmode, inputs_pd], outputs=[output]) else: model = Model(inputs=[inputs_bmode], outputs=[output]) return model
def model(self, inputs, training_schedule, trainable=True, build=False): _concat_axis = 3 alpha = 1.0 if 'warped' in inputs and 'flow' in inputs and 'brightness_error' in inputs: concat_inputs = tf.concat([ inputs['input_a'], inputs['input_b'], inputs['warped'], inputs['flow'], inputs['brightness_error'] ], axis=_concat_axis) else: concat_inputs = tf.concat([inputs['input_a'], inputs['input_b']], axis=_concat_axis) if build is True: #INPUT_SHAPE = #(384, 512,6) concat_inputs = tf.concat( [inputs['input_a'], inputs['input_b']], axis=_concat_axis) #Input(shape=INPUT_SHAPE) ################################################# # ENCODER ################################################# if trainable is False: # all new operations will be in test mode from now on K.set_learning_phase(0) # Conv1_dw / Conv1 conv1_dw = _depthwise_convolution2D(concat_inputs, alpha, 6, (3, 3), strides=(2, 2), training=trainable) conv1 = _convolution2D(conv1_dw, alpha, 32, (1, 1), strides=(1, 1), training=trainable) # Conv2_dw / Conv2 conv2_dw = _depthwise_convolution2D(conv1, alpha, 32, (3, 3), strides=(2, 2), training=trainable) conv2 = _convolution2D(conv2_dw, alpha, 64, (1, 1), strides=(1, 1), training=trainable) # Conv3_dw / Conv3 conv3_dw = _depthwise_convolution2D(conv2, alpha, 64, (3, 3), strides=(2, 2), training=trainable) conv3 = _convolution2D(conv3_dw, alpha, 128, (1, 1), strides=(1, 1), training=trainable) # Conv4a_dw / Conv4a conv4a_dw = _depthwise_convolution2D(conv3, alpha, 128, (3, 3), strides=(2, 2), training=trainable) conv4a = _convolution2D(conv4a_dw, alpha, 256, (1, 1), strides=(1, 1), training=trainable) # Conv4b_dw / Conv4b conv4b_dw = _depthwise_convolution2D(conv4a, alpha, 256, (3, 3), strides=(1, 1), training=trainable) conv4b = _convolution2D(conv4b_dw, alpha, 256, (1, 1), strides=(1, 1), training=trainable) # Conv5a_dw / Conv5a conv5a_dw = _depthwise_convolution2D(conv4b, alpha, 256, (3, 3), strides=(2, 2), training=trainable) conv5a = _convolution2D(conv5a_dw, alpha, 512, (1, 1), strides=(1, 1), training=trainable) # Conv5b_dw / Conv5b conv5b_dw = _depthwise_convolution2D(conv5a, alpha, 512, (3, 3), strides=(1, 1), training=trainable) conv5b = _convolution2D(conv5b_dw, alpha, 512, (1, 1), strides=(1, 1), training=trainable) # Conv6a_dw / Conv6a conv6a_dw = _depthwise_convolution2D(conv5b, alpha, 512, (3, 3), strides=(2, 2), training=trainable) conv6a = _convolution2D(conv6a_dw, alpha, 1024, (1, 1), strides=(1, 1), training=trainable) # Conv6b_dw / Conv6b conv6b_dw = _depthwise_convolution2D(conv6a, alpha, 1024, (3, 3), strides=(1, 1), training=trainable) conv6b = _convolution2D(conv6b_dw, alpha, 1024, (1, 1), strides=(1, 1), training=trainable) ################################################# # DECODER ################################################# # Conv7_dw / Conv7 conv7_dw = _depthwise_convolution2D(conv6b, alpha, 1024, (3, 3), strides=(1, 1), training=trainable) conv7 = _convolution2D(conv7_dw, alpha, 256, (1, 1), strides=(1, 1), training=trainable) # Conv8_dw /Conv8 conv7_resized_tensor = Lambda(resize_like, arguments={ 'ref_tensor': conv7, 'scale': 2 })(conv7) concat_op1 = Concatenate(axis=_concat_axis)( [conv7_resized_tensor, conv5b]) conv8_dw = _depthwise_convolution2D(concat_op1, alpha, 768, (3, 3), strides=(1, 1), training=trainable) conv8 = _convolution2D(conv8_dw, alpha, 128, (1, 1), strides=(1, 1), training=trainable) # Conv9_dw /Conv9 conv8_resized_tensor = Lambda(resize_like, arguments={ 'ref_tensor': conv8, 'scale': 2 })(conv8) concat_op2 = Concatenate(axis=_concat_axis)( [conv8_resized_tensor, conv4b]) conv9_dw = _depthwise_convolution2D(concat_op2, alpha, 384, (3, 3), strides=(1, 1), training=trainable) conv9 = _convolution2D(conv9_dw, alpha, 64, (1, 1), strides=(1, 1), training=trainable) # Conv10_dw / Conv10 coonv9_resized_tensor = Lambda(resize_like, arguments={ 'ref_tensor': conv9, 'scale': 2 })(conv9) concat_op3 = Concatenate(axis=_concat_axis)( [coonv9_resized_tensor, conv3]) conv10_dw = _depthwise_convolution2D(concat_op3, alpha, 192, (3, 3), strides=(1, 1), training=trainable) conv10 = _convolution2D(conv10_dw, alpha, 32, (1, 1), strides=(1, 1), training=trainable) # Conv11_dw / Con11 conv10_resized_tensor = Lambda(resize_like, arguments={ 'ref_tensor': conv10, 'scale': 2 })(conv10) concat_op3 = Concatenate(axis=_concat_axis)( [conv10_resized_tensor, conv2]) conv11_dw = _depthwise_convolution2D(concat_op3, alpha, 96, (3, 3), strides=(1, 1), training=trainable) conv11 = _convolution2D(conv11_dw, alpha, 16, (1, 1), strides=(1, 1), training=trainable) ################################################## # Optical Flow Predictions ################################################## # Conv12_dw / conv12 conv12_dw = _depthwise_convolution2D(conv7, alpha, 256, (3, 3), strides=(1, 1), training=trainable) conv12 = _convolution2D(conv12_dw, alpha, 2, (1, 1), strides=(1, 1), training=trainable) # Conv13_dw / conv13 conv13_dw = _depthwise_convolution2D(conv8, alpha, 128, (3, 3), strides=(1, 1), training=trainable) conv13 = _convolution2D(conv13_dw, alpha, 2, (1, 1), strides=(1, 1), training=trainable) # Conv14_dw / conv14 conv14_dw = _depthwise_convolution2D(conv9, alpha, 64, (3, 3), strides=(1, 1), training=trainable) conv14 = _convolution2D(conv14_dw, alpha, 2, (1, 1), strides=(1, 1), training=trainable) # Conv15_dw / con15 conv15_dw = _depthwise_convolution2D(conv10, alpha, 32, (3, 3), strides=(1, 1), training=trainable) conv15 = _convolution2D(conv15_dw, alpha, 2, (1, 1), strides=(1, 1), training=trainable) # Conv16_dw / conv16 conv16_dw = _depthwise_convolution2D(conv11, alpha, 16, (3, 3), strides=(1, 1), training=trainable) conv16 = _convolution2D(conv16_dw, alpha, 2, (1, 1), strides=(1, 1), training=trainable) ################################################### # Multiple Optical Flow Predictions Fusion ################################################### conv12_resized_tensor_x16 = Lambda(resize_like, arguments={ 'ref_tensor': conv12, 'scale': 16 })(conv12) conv13_resized_tensor_x8 = Lambda(resize_like, arguments={ 'ref_tensor': conv13, 'scale': 8 })(conv13) conv14_resized_tensor_x4 = Lambda(resize_like, arguments={ 'ref_tensor': conv14, 'scale': 4 })(conv14) conv15_resized_tensor_x2 = Lambda(resize_like, arguments={ 'ref_tensor': conv15, 'scale': 2 })(conv15) # 96x128x2 flow = Average(name='average_layer')([ conv12_resized_tensor_x16, conv13_resized_tensor_x8, conv14_resized_tensor_x4, conv15_resized_tensor_x2, conv16 ]) #flow = tf.image.resize_bilinear(average, # tf.stack([height, width]), # align_corners=True) # Fuse groundtrunth resolution with prediction #flow = Lambda(resize_like, arguments={'ref_tensor':average, 'scale': 4})(average) return {'inputs': concat_inputs, 'flow': flow}
def create_range_view_branch(input_shapes=[(375, 1242, 3), (375, 1242, 1), (375, 1242, 1), (375, 1242, 1), (224, 256 , 2), (112, 128 , 2), (56 , 64 , 2), (224, 256 , 3), (112, 128 , 3), (56 , 64 , 3),], input_names=['rgb_img_input', 'depth_map_input', 'intensity_map_input', 'height_map_input', 'mapping_2x', 'mapping_4x', 'mapping_8x', 'geo_2x', 'geo_4x', 'geo_8x',], input_types=['float32', 'float32', 'float32', 'float32', 'int32', 'int32', 'int32', 'float32', 'float32', 'float32',]): init_inputs = [] inputs_dict = {} for name, shape, dtype in zip(input_names, input_shapes, input_types): inp = create_input_layer(input_shape=shape, name=name, dtype=dtype) init_inputs.append(inp) inputs_dict[name] = inp conv_inputs = [] for input in init_inputs: if 'mapping' not in input.name and 'geo' not in input.name: conv_inputs.append(conv_block(input, 32, 7, 2)) # /2 x = Average()(conv_inputs) l1 = x for i in range(4): # same res x = create_res_conv_block(x, 32, 3) l2 = x for i in range(4): # /4 x = create_res_conv_block(x, 64, 3, i==0) l3 = x for i in range(6): # /8 x = create_res_conv_block(x, 128, 3, i==0) l4 = x out_map_sz = (112, 128) TOP_DOWN_PYRAMID_SIZE = 96 L2 = tf.image.resize(l2, (l3.shape[1], l3.shape[2]), 'nearest') L2 = conv_block(L2, TOP_DOWN_PYRAMID_SIZE, 3, 1) L2 = conv_block(L2, TOP_DOWN_PYRAMID_SIZE, 1, 1) L3 = conv_block(l3, TOP_DOWN_PYRAMID_SIZE, 1, 1) L4 = tf.image.resize(l4, (l3.shape[1], l3.shape[2]), 'nearest') L4 = conv_block(L4, TOP_DOWN_PYRAMID_SIZE, 3, 1) L4 = conv_block(L4, TOP_DOWN_PYRAMID_SIZE, 1, 1) RV_OUT = Add()([L2, L3, L4]) RV2BEV_2x = Lambda(tf.gather_nd, arguments={'indices': inputs_dict['mapping_2x'], 'batch_dims': 1})(RV_OUT) RV2BEV_2x = Concatenate()([RV2BEV_2x, inputs_dict['geo_2x']]) # add geometric feature RV2BEV_2x = transform_rangeview_to_bev(RV2BEV_2x, 128) RV2BEV_4x = Lambda(tf.gather_nd, arguments={'indices': inputs_dict['mapping_4x'], 'batch_dims': 1})(RV_OUT) RV2BEV_4x = Concatenate()([RV2BEV_4x, inputs_dict['geo_4x']]) # add geometric feature RV2BEV_4x = transform_rangeview_to_bev(RV2BEV_4x, 192) RV2BEV_8x = Lambda(tf.gather_nd, arguments={'indices': inputs_dict['mapping_8x'], 'batch_dims': 1})(RV_OUT) RV2BEV_8x = Concatenate()([RV2BEV_8x, inputs_dict['geo_8x']]) # add geometric feature RV2BEV_8x = transform_rangeview_to_bev(RV2BEV_8x, 256) # print(RV2BEV_2x.shape, RV2BEV_4x.shape, RV2BEV_8x.shape) return { 'inputs': init_inputs, 'outputs': [ RV2BEV_2x, RV2BEV_4x, RV2BEV_8x ], }