def __init__(self, name, inputs, tower_setup, n_features, concat, activation="relu", filter_size=(3, 3), l2=Layer.L2_DEFAULT): super(Upsampling, self).__init__() filter_size = list(filter_size) assert isinstance(concat, list) assert len(concat) > 0 curr, n_features_inp = prepare_input(inputs) concat_inp, n_features_concat = prepare_input(concat) curr = tf.image.resize_nearest_neighbor(curr, tf.shape(concat_inp)[1:3]) curr = tf.concat([curr, concat_inp], axis=3) n_features_curr = n_features_inp + n_features_concat with tf.variable_scope(name): W = self.create_weight_variable( "W", filter_size + [n_features_curr, n_features], l2, tower_setup) b = self.create_bias_variable("b", [n_features], tower_setup) curr = conv2d(curr, W) + b curr = get_activation(activation)(curr) self.outputs = [curr]
def _init_(self, name, inputs, tower_setup, variant, add_conv5): super(ResNet, self)._init_() inp, n_features_inp = prepare_input(inputs) # for grayscale if n_features_inp == 1: inp = tf.concat([inp, inp, inp], axis=-1) else: assert n_features_inp == 3 with tf.variable_scope(name): if variant == "resnet50": net = pretrained_resnet50_conv4(inp, tower_setup) elif variant == "resnet101": net = pretrained_resnet101_conv4(inp, tower_setup) elif variant == "xception": net = pretrained_xception(inp, tower_setup) else: assert False, "Unknown ResNet variant" if add_conv5: net = add_resnet_conv5(net[-1], tower_setup) self.outputs = [net[-1]] if tower_setup.is_training: vars_to_regularize = tf.trainable_variables( name + "/(?:group1|group2|group3)/.*W") regularizers = [ 1e-4 * tf.nn.l2_loss(W) for W in vars_to_regularize ] regularization_loss = tf.add_n(regularizers, "regularization_loss") self.regularizers.append(regularization_loss)
def __init__(self, name, inputs, n_features, tower_setup, filter_size=(3, 3), strides=(1, 1), activation="relu", # dropout=0.0, batch_norm=False, bias=False, batch_norm_decay=Layer.BATCH_NORM_DECAY_DEFAULT, l2=Layer.L2_DEFAULT, padding="SAME", W_initializer=None, freeze_batchnorm_override=None): super(ConvTranspose, self).__init__() # TODO this uses the tensorpack order of operations curr, n_features_inp = prepare_input(inputs) filter_size = list(filter_size) strides = list(strides) with tf.variable_scope(name): W = self.create_weight_variable("W", filter_size + [n_features, n_features_inp], l2, tower_setup, initializer=W_initializer) b = None if bias: b = self.create_bias_variable("b", [n_features], tower_setup) # curr = apply_dropout(curr, dropout) curr = conv2d_transpose(curr, W, strides, padding=padding) if bias: curr = tf.nn.bias_add(curr, b) if batch_norm: curr = self.create_and_apply_batch_norm(curr, n_features, batch_norm_decay, tower_setup, freeze_batchnorm_override=freeze_batchnorm_override) curr = get_activation(activation)(curr) self.outputs = [curr]
def __init__(self, name, inputs, dataset, n_features, tower_setup, filter_size=(1, 1), input_activation=None, dilation=None, l2=Layer.L2_DEFAULT, dropout=0.0): super().__init__() if n_features == -1: n_features = dataset.num_classes() filter_size = list(filter_size) inp, n_features_inp = prepare_input(inputs) if input_activation is not None: inp = get_activation(input_activation)(inp) inp = apply_dropout(inp, dropout) with tf.variable_scope(name): W = self.create_weight_variable( "W", filter_size + [n_features_inp, n_features], l2, tower_setup) b = self.create_bias_variable("b", [n_features], tower_setup) if dilation is None: output = conv2d(inp, W) + b else: output = conv2d_dilated(inp, W, dilation) + b self.outputs = [output]
def __init__(self, name, inputs, n_features, tower_setup, activation="relu", dropout=0.0, batch_norm=False, batch_norm_decay=Layer.BATCH_NORM_DECAY_DEFAULT, l2=Layer.L2_DEFAULT): super(FullyConnected, self).__init__() inp, n_features_inp = prepare_input(inputs) with tf.variable_scope(name): inp = apply_dropout(inp, dropout) if batch_norm: inp = tf.expand_dims(inp, axis=0) inp = tf.expand_dims(inp, axis=0) inp = self.create_and_apply_batch_norm(inp, n_features_inp, batch_norm_decay, tower_setup) inp = tf.squeeze(inp, axis=[0, 1]) W = self.create_weight_variable("W", [n_features_inp, n_features], l2, tower_setup) b = self.create_bias_variable("b", [n_features], tower_setup) z = tf.matmul(inp, W) + b h = get_activation(activation)(z) self.outputs = [h] self.n_features = n_features
def __init__(self, name, inputs, tower_setup, factor=2): super().__init__() with tf.variable_scope(name): inp, dim = prepare_input(inputs) height = tf.shape(inp)[1] width = tf.shape(inp)[2] resized_input = tf.image.resize_images( inputs[0], (height * factor, width * factor), tf.image.ResizeMethod.NEAREST_NEIGHBOR) self.outputs.append(resized_input)
def __init__(self, name, inputs, tower_setup, n_convs=2, n_features=None, dilations=None, strides=None, filter_size=None, activation="relu", dropout=0.0, batch_norm_decay=Layer.BATCH_NORM_DECAY_DEFAULT, l2=Layer.L2_DEFAULT): super().__init__() curr, n_features_inp = prepare_input(inputs) res = curr assert n_convs >= 1, n_convs if dilations is not None: assert strides is None elif strides is None: strides = [[1, 1]] * n_convs if filter_size is None: filter_size = [[3, 3]] * n_convs if n_features is None: n_features = n_features_inp if not isinstance(n_features, list): n_features = [n_features] * n_convs with tf.variable_scope(name): curr = self.create_and_apply_batch_norm(curr, n_features_inp, batch_norm_decay, tower_setup, "bn0") curr = get_activation(activation)(curr) if tower_setup.is_training: curr = apply_dropout(curr, dropout) if strides is None: strides_res = [1, 1] else: strides_res = np.prod(strides, axis=0).tolist() if (n_features[-1] != n_features_inp) or (strides_res != [1, 1]): W0 = self.create_weight_variable("W0", [1, 1] + [n_features_inp, n_features[-1]], l2, tower_setup) if dilations is None: res = conv2d(curr, W0, strides_res) else: res = conv2d(curr, W0) W1 = self.create_weight_variable("W1", filter_size[0] + [n_features_inp, n_features[0]], l2, tower_setup) if dilations is None: curr = conv2d(curr, W1, strides[0]) else: curr = conv2d_dilated(curr, W1, dilations[0]) for idx in range(1, n_convs): curr = self.create_and_apply_batch_norm(curr, n_features[idx - 1], batch_norm_decay, tower_setup, "bn" + str(idx + 1)) curr = get_activation(activation)(curr) Wi = self.create_weight_variable("W" + str(idx + 1), filter_size[idx] + [n_features[idx - 1], n_features[idx]], l2, tower_setup) if dilations is None: curr = conv2d(curr, Wi, strides[idx]) else: curr = conv2d_dilated(curr, Wi, dilations[idx]) curr += res self.outputs = [curr]
def __init__(self, name, inputs, n_features, tower_setup, filter_size=(3, 3), old_order=False, strides=(1, 1), dilation=None, pool_size=(1, 1), pool_strides=None, activation="relu", dropout=0.0, batch_norm=False, bias=False, batch_norm_decay=Layer.BATCH_NORM_DECAY_DEFAULT, l2=Layer.L2_DEFAULT, padding="SAME", W_initializer=None, b_initializer=None, freeze_batchnorm_override=None): super(Conv, self).__init__() # mind the order of dropout, conv, activation and batchnorm! # batchnorm -> activation -> dropout -> conv -> pool # if old_order: dropout -> conv -> batchnorm -> activation -> pool (used for example in tensorpack) curr, n_features_inp = prepare_input(inputs) filter_size = list(filter_size) strides = list(strides) pool_size = list(pool_size) if pool_strides is None: pool_strides = pool_size with tf.variable_scope(name): if isinstance(W_initializer, list): W_initializer = tf.constant_initializer(W_initializer) W = self.create_weight_variable("W", filter_size + [n_features_inp, n_features], l2, tower_setup, initializer=W_initializer) b = None if bias: if b_initializer is not None: b_initializer = tf.constant_initializer(b_initializer) b = self.create_bias_variable("b", [n_features], tower_setup, initializer=b_initializer) else: b = self.create_bias_variable("b", [n_features], tower_setup) if old_order: curr = apply_dropout(curr, dropout) if dilation is None: curr = conv2d(curr, W, strides, padding=padding) else: curr = conv2d_dilated(curr, W, dilation, padding=padding) if bias: curr += b if batch_norm: curr = self.create_and_apply_batch_norm(curr, n_features, batch_norm_decay, tower_setup, freeze_batchnorm_override=freeze_batchnorm_override) curr = get_activation(activation)(curr) else: if batch_norm: curr = self.create_and_apply_batch_norm(curr, n_features_inp, batch_norm_decay, tower_setup, freeze_batchnorm_override=freeze_batchnorm_override) curr = get_activation(activation)(curr) curr = apply_dropout(curr, dropout) if dilation is None: curr = conv2d(curr, W, strides, padding=padding) else: curr = conv2d_dilated(curr, W, dilation, padding=padding) if bias: curr += b if pool_size != [1, 1]: curr = max_pool(curr, pool_size, pool_strides) self.outputs = [curr]
def __init__(self, name, inputs, tower_setup): super(ResNet50, self).__init__() inp, n_features_inp = prepare_input(inputs) #for grayscale if n_features_inp == 1: inp = tf.concat([inp, inp, inp], axis=-1) else: assert n_features_inp == 3 with tf.variable_scope(name): self.outputs = pretrained_resnet50_conv4(inp, tower_setup) self.outputs.append( add_resnet_conv5(self.outputs[-1], tower_setup)[-1])
def __init__(self, name, inputs, tower_setup, n_features, filter_size=(5, 5), l2=Layer.L2_DEFAULT): super(ConvLSTM, self).__init__() filter_size = list(filter_size) inp, n_features_inp = prepare_input(inputs) shape = smart_shape(inp) batch_size = 1 height = shape[1] width = shape[2] # input is assumed to have shape (time, height, width, feature) # we need to introduce a batch 1 dimension -> (time, batch=1, height, width, feature) inp = inp[:, tf.newaxis] with tf.variable_scope(name): # TODO: maybe we need a different weight init here because we don't use relu W = self.create_weight_variable( "W_lstm", filter_size + [n_features_inp + n_features, 4 * n_features], l2, tower_setup) b = self.create_bias_variable("b_lstm", [4 * n_features], tower_setup) lstm = ConvLSTMCell(n_features, height, width, filter_size, W=W, b=b) initial_state = lstm.zero_state(batch_size, tower_setup.dtype) if not tower_setup.is_training: c0 = tf.placeholder_with_default(initial_state[0], initial_state[0].get_shape(), "c0_placeholder") h0 = tf.placeholder_with_default(initial_state[1], initial_state[1].get_shape(), "h0_placeholder") self.placeholders.append(c0) self.placeholders.append(h0) initial_state = LSTMStateTuple(c0, h0) y, (c, h) = dynamic_conv_rnn(lstm, inp, initial_state=initial_state, time_major=True) y = tf.reshape(y, shape[:-1] + [n_features]) self.outputs.append(y) self.extractions[Extractions.RECURRENT_STATE] = (c, h)
def __init__(self, name, inputs, tower_setup): super(ResNet50Conv4, self).__init__() inp, n_features_inp = prepare_input(inputs) # for grayscale if n_features_inp == 1: inp = tf.concat([inp, inp, inp], axis=-1) else: assert n_features_inp == 3 with tf.variable_scope(name): self.outputs = [pretrained_resnet50_conv4(inp, tower_setup)[-1]] if tower_setup.is_training: vars_to_regularize = tf.trainable_variables( name + "/(?:group1|group2|group3)/.*W") regularizers = [ 1e-4 * tf.nn.l2_loss(W) for W in vars_to_regularize ] regularization_loss = tf.add_n(regularizers, "regularization_loss") self.regularizers.append(regularization_loss)
def __init__(self, name, inputs, n_features, tower_setup, filter_size=(3, 3, 3), old_order=False, strides=(1, 1, 1), pool_size=(1, 1, 1), pool_strides=None, activation="relu", dropout=0.0, batch_norm=False, bias=False, batch_norm_decay=Layer.BATCH_NORM_DECAY_DEFAULT, l2=Layer.L2_DEFAULT, padding="SAME", init_type="random"): super(SepConv3DOverBatch, self).__init__() # mind the order of dropout, conv, activation and batchnorm! # batchnorm -> activation -> dropout -> conv -> pool # if old_order: dropout -> conv -> batchnorm -> activation -> pool (used for example in tensorpack) curr, n_features_inp = prepare_input(inputs) filter_size = list(filter_size) strides = list(strides) pool_size = list(pool_size) if pool_strides is None: pool_strides = pool_size assert not batch_norm assert dropout == 0.0 assert old_order # Introduce a fake "batch dimension", previous dimension becomes time curr = tf.expand_dims(curr, axis=0) with tf.variable_scope(name): # Depthwise conv3d curr_split = tf.split(curr, n_features_inp, axis=-1) if init_type == "identity": filter_initializer = np.zeros(filter_size + [1, 1], dtype=np.float32) filter_initializer[int(filter_size[0]/2.0), int(filter_size[1]/2.0), int(filter_size[2]/2.0), 0, 0] = 1.0 elif init_type == "avgtime": filter_initializer = np.zeros(filter_size + [1, 1], dtype=np.float32) filter_initializer[:, int(filter_size[1]/2.0), int(filter_size[2]/2.0), 0, 0] = 1.0/filter_size[0] else: # random filter_initializer = None for channel in range(n_features_inp): with tf.variable_scope("channel_" + str(channel)): curr = curr_split[channel] W = self.create_weight_variable("W", filter_size + [1, 1], l2, tower_setup, initializer=filter_initializer) b = None if bias: b = self.create_bias_variable("b", [n_features], tower_setup) curr = conv3d(curr, W, strides, padding=padding) if bias: curr += b curr_split[channel] = curr # 1x1 conv3d over all channels curr = tf.concat(curr_split, axis=-1) if init_type == "identity" or init_type == "avgtime": assert n_features == n_features_inp init_weights = np.zeros([1, 1, 1, n_features_inp, n_features], dtype=np.float32) for i in range(n_features): init_weights[:, :, :, i, i] = 1.0 W = self.create_weight_variable("W", [1, 1, 1, n_features_inp, n_features], l2, tower_setup, initializer=init_weights) else: # random W = self.create_weight_variable("W", [1, 1, 1, n_features_inp, n_features], l2, tower_setup) b = None if bias: b = self.create_bias_variable("b", [n_features], tower_setup) curr = conv3d(curr, W, strides, padding=padding) if bias: curr += b curr = get_activation(activation)(curr) if pool_size != [1, 1, 1]: curr = max_pool3d(curr, pool_size, pool_strides) # Remove the fake dimension again curr = tf.squeeze(curr, axis=0) self.outputs = [curr]
def __init__(self, name, inputs, n_features, tower_setup, filter_size=(3, 3, 3), old_order=False, strides=(1, 1, 1), pool_size=(1, 1, 1), pool_strides=None, activation="relu", dropout=0.0, batch_norm=False, bias=False, batch_norm_decay=Layer.BATCH_NORM_DECAY_DEFAULT, l2=Layer.L2_DEFAULT, padding="SAME", init_type="random"): super(Conv3DOverBatch, self).__init__() # mind the order of dropout, conv, activation and batchnorm! # batchnorm -> activation -> dropout -> conv -> pool # if old_order: dropout -> conv -> batchnorm -> activation -> pool (used for example in tensorpack) curr, n_features_inp = prepare_input(inputs) filter_size = list(filter_size) strides = list(strides) pool_size = list(pool_size) if pool_strides is None: pool_strides = pool_size # Introduce a fake "batch dimension", previous dimension becomes time curr = tf.expand_dims(curr, axis=0) with tf.variable_scope(name): if init_type == "identity": assert n_features == n_features_inp identity_filter = np.zeros(filter_size, dtype=np.float32) identity_filter[int(filter_size[0]/2.0), int(filter_size[1]/2.0), int(filter_size[2]/2.0)] = 1.0 init_weights = np.zeros(list(filter_size) + [n_features_inp, n_features], dtype=np.float32) for i in range(n_features): init_weights[:, :, :, i, i] = identity_filter W = self.create_weight_variable("W", filter_size + [n_features_inp, n_features], l2, tower_setup, initializer=init_weights) elif init_type == "avgtime": assert n_features == n_features_inp avgtime_filter = np.zeros(filter_size, dtype=np.float32) avgtime_filter[:, int(filter_size[1]/2.0), int(filter_size[2]/2.0)] = 1.0/filter_size[0] init_weights = np.zeros(list(filter_size) + [n_features_inp, n_features], dtype=np.float32) for i in range(n_features): init_weights[:, :, :, i, i] = avgtime_filter # Maybe we'ld like sth else than per-channel averaging? W = self.create_weight_variable("W", filter_size + [n_features_inp, n_features], l2, tower_setup, initializer=init_weights) else: # random W = self.create_weight_variable("W", filter_size + [n_features_inp, n_features], l2, tower_setup) b = None if bias: b = self.create_bias_variable("b", [n_features], tower_setup) if old_order: curr = apply_dropout(curr, dropout) curr = conv3d(curr, W, strides, padding=padding) if bias: curr += b if batch_norm: curr = self.create_and_apply_batch_norm(curr, n_features, batch_norm_decay, tower_setup) curr = get_activation(activation)(curr) else: if batch_norm: curr = self.create_and_apply_batch_norm(curr, n_features_inp, batch_norm_decay, tower_setup) curr = get_activation(activation)(curr) curr = apply_dropout(curr, dropout) curr = conv3d(curr, W, strides, padding=padding) if bias: curr += b if pool_size != [1, 1, 1]: curr = max_pool3d(curr, pool_size, pool_strides) # Remove the fake dimension again curr = tf.squeeze(curr, axis=0) self.outputs = [curr]
def __init__(self, name, inputs, tower_setup, n_features, filter_size=(5, 5), l2=Layer.L2_DEFAULT): super(BidirectionalConvLSTM, self).__init__() filter_size = list(filter_size) inp, n_features_inp = prepare_input(inputs) shape = smart_shape(inp) batch_size = 1 height = shape[1] width = shape[2] # input is assumed to have shape (time, height, width, feature) # we need to introduce a batch 1 dimension -> (time, batch=1, height, width, feature) inp = inp[:, tf.newaxis] with tf.variable_scope(name): inp_fwd = inp inp_bwd = tf.reverse(inp, axis=[0]) W_fwd = self.create_weight_variable( "W_lstm_fwd", filter_size + [n_features_inp + n_features, 4 * n_features], l2, tower_setup) W_bwd = self.create_weight_variable( "W_lstm_bwd", filter_size + [n_features_inp + n_features, 4 * n_features], l2, tower_setup) b_fwd = self.create_bias_variable("b_lstm_fwd", [4 * n_features], tower_setup) b_bwd = self.create_bias_variable("b_lstm_bwd", [4 * n_features], tower_setup) lstm_fwd = ConvLSTMCell(n_features, height, width, filter_size, W=W_fwd, b=b_fwd) lstm_bwd = ConvLSTMCell(n_features, height, width, filter_size, W=W_bwd, b=b_bwd) initial_state_fwd = lstm_fwd.zero_state(batch_size, tower_setup.dtype) initial_state_bwd = lstm_bwd.zero_state(batch_size, tower_setup.dtype) y_fwd, _ = dynamic_conv_rnn(lstm_fwd, inp_fwd, initial_state=initial_state_fwd, time_major=True) y_bwd, _ = dynamic_conv_rnn(lstm_bwd, inp_bwd, initial_state=initial_state_bwd, time_major=True) y_bwd = tf.reverse(y_bwd, axis=[0]) y_fwd = tf.reshape(y_fwd, shape[:-1] + [n_features]) y_bwd = tf.reshape(y_bwd, shape[:-1] + [n_features]) y = tf.concat([y_fwd, y_bwd], axis=-1) self.outputs.append(y)