def get_symbol(input_blob):
filter_list = [64, 64, 128, 256, 512]
num_stages = 4
units = [3, 13, 30, 3]
num_classes = config.emb_size
fc_type = config.fc_type
bn_is_training = config.bn_is_training
data_format = config.data_format
if data_format.upper() == "NCHW":
input_blob = flow.transpose(
input_blob, name="transpose", perm=[0, 3, 1, 2]
)
input_blob = _conv2d_layer(
name="conv0",
input=input_blob,
filters=filter_list[0],
kernel_size=3,
strides=[1, 1],
padding="same",
data_format=data_format,
use_bias=False,
dilation_rate=1,
activation=None,
)
input_blob = _batch_norm(
input_blob, epsilon=2e-5, is_training=bn_is_training, data_format=data_format, name="bn0"
)
input_blob = _prelu(input_blob, data_format=data_format, name="relu0")
for i in range(num_stages):
input_blob = residual_unit_v3(
input_blob,
filter_list[i + 1],
[2, 2],
False,
bn_is_training=bn_is_training,
data_format=data_format,
name="stage%d_unit%d" % (i + 1, 1),
)
for j in range(units[i] - 1):
input_blob = residual_unit_v3(
input_blob,
filter_list[i + 1],
[1, 1],
True,
bn_is_training=bn_is_training,
data_format=data_format,
name="stage%d_unit%d" % (i + 1, j + 2),
)
fc1 = get_fc1(input_blob, num_classes, fc_type)
return fc1
def combine_heads(self, x):
"""Combine tensor that has been split.
Args:
x: A tensor [batch_size, num_heads, length, hidden_size/num_heads]
Returns:
A tensor with shape [batch_size, length, hidden_size]
"""
with flow.scope.namespace("combine_heads"):
batch_size = x.shape[0]
length = x.shape[2]
x = flow.transpose(
x, [0, 2, 1, 3]) # --> [batch, length, num_heads, depth]
return flow.reshape(x, [batch_size, length, self.hidden_size])
def alexnet_inference(
image: flow.typing.Numpy.Placeholder(image_shape, dtype=flow.float32),
label: flow.typing.Numpy.Placeholder(label_shape, dtype=flow.int32),
) -> flow.typing.Numpy:
input_lbns["image"] = image.logical_blob_name
input_lbns["label"] = label.logical_blob_name
image = flow.transpose(image, perm=(0, 3, 1, 2))
loss = alexnet(image, label, trainable=False)
# reduce_mean calculate reduce_count in python api, we should only set attribute for op in python,
# so reduce_count is out of date when we have loaded model and set new batch_size.
# We will modify implementation of reduce_mean
# output = flow.math.reduce_mean(loss)
output = loss
output_lbns["output"] = output.logical_blob_name
return output
def _cosine_distance(input1, input2):
"""Computes cosine distance.
Args:
input1 : 2-D feature matrix.
input2 : 2-D feature matrix.
Returns:
distance matrix.
"""
input1_normed = math.l2_normalize(input1, axis=1)
input2_normed = math.l2_normalize(input2, axis=1)
distmat = math.subtract(
1,
flow.matmul(input1_normed, flow.transpose(input2_normed, perm=(1, 0))))
return distmat
def self_attn_qk_v_fw_bw(h: flow.typing.Numpy.Placeholder(
shape=(seq_len, batch_size, hidden_size),
dtype=flow.float32)) -> typing.Tuple[flow.typing.Numpy,
flow.typing.Numpy]:
var = flow.get_variable(
"var",
shape=(1, ),
dtype=flow.float32,
initializer=flow.constant_initializer(1.0, dtype=flow.float32),
trainable=True,
)
h = h * var
# save grad
if fused:
flow.watch_diff(h, test_global_storage.Setter("h_grad_fused"))
else:
flow.watch_diff(h, test_global_storage.Setter("h_grad"))
if fp16:
h = flow.amp_white_identity(h)
alpha = get_alpha(head_size)
if fused:
qmk, v = flow.nn.fused_self_attention_query_mul_key_and_value(
h, head_size=head_size, alpha=alpha)
else:
# (s, b, H) -> (s, b, n, 3 * h) -> (s, b, n, h) -> (b, n, s, h)
h = flow.reshape(h, (seq_len, batch_size, -1, 3 * head_size))
q, k, v = (flow.transpose(
flow.slice(
h,
begin=[None, None, None, head_size * i],
size=[None, None, None, head_size],
),
perm=[1, 2, 0, 3],
) for i in range(3))
qmk = flow.matmul(q, k, transpose_b=True, alpha=alpha)
# calc loss for grad
h = flow.matmul(qmk, v)
loss = flow.math.reduce_sum(h)
flow.optimizer.SGD(get_lr_scheduler(), momentum=0).minimize(loss)
return qmk, v
def validation_dataset_reader(val_dataset_dir,
val_batch_size=1,
val_data_part_num=1):
# lfw: (12000L, 3L, 112L, 112L)
# cfp_fp: (14000L, 3L, 112L, 112L)
# agedb_30: (12000L, 3L, 112L, 112L)
if os.path.exists(val_dataset_dir):
print("Loading validation data from {}".format(val_dataset_dir))
else:
raise Exception("Invalid validation dataset dir", val_dataset_dir)
color_space = "RGB"
ofrecord = flow.data.ofrecord_reader(
val_dataset_dir,
batch_size=val_batch_size,
data_part_num=val_data_part_num,
part_name_suffix_length=1,
shuffle_after_epoch=False,
)
image = flow.data.OFRecordImageDecoder(ofrecord,
"encoded",
color_space=color_space)
issame = flow.data.OFRecordRawDecoder(ofrecord,
"issame",
shape=(),
dtype=flow.int32)
rsz, scale, new_size = flow.image.Resize(image,
target_size=(112, 112),
channels=3)
normal = flow.image.CropMirrorNormalize(
rsz,
color_space=color_space,
crop_h=0,
crop_w=0,
crop_pos_y=0.5,
crop_pos_x=0.5,
mean=[127.5, 127.5, 127.5],
std=[128.0, 128.0, 128.0],
output_dtype=flow.float,
)
normal = flow.transpose(normal, name="transpose_val", perm=[0, 2, 3, 1])
return issame, normal
def TransposeJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"input",
shape=input_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=2, maxval=5),
trainable=True,
)
loss = flow.transpose(x, perm)
flow.losses.add_loss(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def build_network(self):
if self.need_transpose:
images = flow.transpose(self.images, name="transpose", perm=[0, 3, 1,
2])
else:
images = self.images
conv1 = _conv2d(images, 64, kernel_size=7, strides=2,
padding=([0, 0], [0, 0], [3, 3], [3, 3]),
groups=1, use_bias=False, trainable=self.trainable, name="conv1")
bn1 = _batch_norm(conv1, trainable=self.trainable, training=self.training, name="bn1")
relu = flow.nn.relu(bn1, name="relu1")
max_pool = flow.nn.max_pool2d(relu, ksize=3, strides=2,
padding=[[0, 0], [0, 0], [1, 1], [1, 1]], data_format="NCHW", name="max_pool")
layer1 = self._make_layer(max_pool, 64, self.layers[0],
self.num_group, layer_num="layer1")
layer2 = self._make_layer(layer1[-1], 128, self.layers[1],
self.num_group, strides=2, layer_num="layer2")
layer3 = self._make_layer(layer2[-1], 256, self.layers[2],
self.num_group, strides=2, layer_num="layer3")
layer4 = self._make_layer(layer3[-1], 512, self.layers[3],
self.num_group, strides=2, layer_num="layer4")
# debug mode: dump data for debugging
# with flow.watch_scope(blob_watcher=blob_watched,
# diff_blob_watcher=diff_blob_watched):
# bn1_identity = flow.identity(layer4[-1], name="layer4_last_out")
avg_pool = flow.nn.avg_pool2d(layer4[-1], 7, strides=1, padding="VALID",
data_format="NCHW", name="avg_pool")
reshape = flow.reshape(avg_pool, (avg_pool.shape[0], -1))
fc = flow.layers.dense(reshape, units=self.num_classes, use_bias=True,
kernel_initializer=_get_initializer("dense_weight"),
bias_initializer=_get_initializer("dense_bias"),
trainable=self.trainable,
kernel_regularizer=_get_regularizer("dense_weight"),
bias_regularizer=_get_regularizer("dense_bias"),
name="fc")
return fc
def TransposeJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"input",
shape=input_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=2, maxval=5),
trainable=True,
)
loss = flow.transpose(x, perm)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [1e-4]), momentum=0
).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def YoloPredictLayer(in_blob, origin_image_info, i, trainable):
global layer_number
layer_name = 'yolo-layer' + str(layer_number)
#placeholder for a reshape from (n,h,w,255)->(n,h,w*3,85)
blob = flow.transpose(in_blob,
name=layer_name + '-yolo_transpose',
perm=[0, 2, 3, 1])
reshape_blob = flow.reshape(blob,
shape=(blob.shape[0], -1, 85),
name=layer_name + '-yolo_reshape')
position = flow.slice(reshape_blob, [None, 0, 0], [None, -1, 4],
name=layer_name + '-yolo_slice_pos')
xy = flow.slice(position, [None, 0, 0], [None, -1, 2],
name=layer_name + '-yolo_slice_xy')
wh = flow.slice(position, [None, 0, 2], [None, -1, 2],
name=layer_name + '-yolo_slice_wh')
xy = flow.math.sigmoid(xy, name=layer_name + '-yolo_ligistic_xy')
position = flow.concat([xy, wh], axis=2, name=layer_name + '-yolo_concat')
confidence = flow.slice(reshape_blob, [None, 0, 4], [None, -1, 81],
name=layer_name + '-yolo_slice_prob')
confidence = flow.math.sigmoid(confidence,
name=layer_name + '-yolo_ligistic_prob')
#[out_bbox, out_probs, valid_num] = flow.detection.yolo_detect(bbox=position, probs=confidence, origin_image_info=origin_image_info, image_height=608, image_width=608, layer_height=yolo_conf[i]['layer_height'], layer_width=yolo_conf[i]['layer_width'], prob_thresh=0.5, num_classes=80, max_out_boxes = max_out_boxes, anchor_boxes=yolo_conf[i]['anchor_boxes_size'])
[out_bbox, out_probs, valid_num
] = flow.yolo_detect(bbox=position,
probs=confidence,
origin_image_info=origin_image_info,
image_height=608,
image_width=608,
layer_height=yolo_conf[i]['layer_height'],
layer_width=yolo_conf[i]['layer_width'],
prob_thresh=0.5,
num_classes=80,
max_out_boxes=max_out_boxes,
anchor_boxes=yolo_conf[i]['anchor_boxes_size'],
name=str(layer_name) + "yolo_detect")
#print("out_bbox.shape",out_bbox.shape)
return out_bbox, out_probs, valid_num
def __getitem__(self, index):
"""
Args:
index (int): Index
Returns:
tuple: (clip, pid, camid) where pid is identity of the clip.
"""
img_paths, pid, camid = self.dataset[index]
if self.temporal_transform is not None:
img_paths = self.temporal_transform(img_paths)
clip = self.loader(img_paths)
if self.spatial_transform is not None:
self.spatial_transform.randomize_parameters()
clip = [self.spatial_transform(img) for img in clip]
# trans T x C x H x W to C x T x H x W
clip = flow.stack(clip, axis=0)
clip = flow.transpose(clip, perm=[1, 0, 2, 3])
return clip, pid, camid
def split_heads(self, x):
"""Split x into different heads, and transpose the resulting value.
The tensor is transposed to insure the inner dimensions hold the correct
values during the matrix multiplication.
Args:
x: A tensor with shape [batch_size, length, hidden_size]
Returns:
A tensor with shape [batch_size, num_heads, length, hidden_size/num_heads]
"""
with flow.scope.namespace("split_heads"):
batch_size = x.shape[0]
length = x.shape[1]
# Calculate depth of last dimension after it has been split.
depth = (self.hidden_size // self.num_heads)
# Split the last dimension
x = flow.reshape(x, [batch_size, length, self.num_heads, depth])
# Transpose the result
return flow.transpose(x, [0, 2, 1, 3])
def eval_searchtransfer(lrsr_lv3_unfold: tp.Numpy.Placeholder(
(1, 2304, 1600)), refsr_lv3_unfold: tp.Numpy.Placeholder(
(1, 2304, 1600)), ref_lv3_unfold: tp.Numpy.Placeholder(
(1, 2304, 1600)), ref_lv2_unfold: tp.Numpy.Placeholder(
(1, 4608, 1600)), ref_lv1_unfold: tp.Numpy.Placeholder(
(1, 9216, 1600))) -> Tuple[tp.Numpy, tp.Numpy,
tp.Numpy, tp.Numpy]:
refsr_lv3_unfold = flow.transpose(refsr_lv3_unfold, perm=[0, 2, 1])
refsr_lv3_unfold = flow.math.l2_normalize(
refsr_lv3_unfold, axis=2) # [N, Hr*Wr, C*k*k]
lrsr_lv3_unfold = flow.math.l2_normalize(lrsr_lv3_unfold,
axis=1) # [N, C*k*k, H*W]
R_lv3 = flow.matmul(refsr_lv3_unfold,
lrsr_lv3_unfold) # [N, Hr*Wr, H*W]
R_lv3_star = flow.math.reduce_max(R_lv3, axis=1) # [N, H*W]
R_lv3_star_arg = flow.math.argmax(R_lv3, axis=1) # [N, H*W]
T_lv3_unfold = self.bis(ref_lv3_unfold, R_lv3_star_arg)
T_lv2_unfold = self.bis(ref_lv2_unfold, R_lv3_star_arg)
T_lv1_unfold = self.bis(ref_lv1_unfold, R_lv3_star_arg)
return R_lv3_star, T_lv3_unfold, T_lv2_unfold, T_lv1_unfold
def one_hot(
indices: remote_blob_util.BlobDef,
depth: int,
on_value: Union[int, float] = 1,
off_value: Union[int, float] = 0,
axis: int = -1,
dtype: Optional[dtype_util.dtype] = None,
name: Optional[str] = None,
) -> remote_blob_util.BlobDef:
out_ndims = len(indices.shape) + 1
if axis < 0:
axis += out_ndims
assert axis >= 0 and axis < out_ndims, ValueError(
"Expected axis to between [%d, %d). But received: %d " %
(-out_ndims, out_ndims, axis))
out = (flow.user_op_builder(name if name is not None else id_util.
UniqueStr("OneHot_")).Op("one_hot").Input(
"indices",
[indices]).Attr("depth", int(depth)).Attr(
"floating_on_value",
float(on_value)).Attr(
"integer_on_value",
int(on_value)).Attr(
"floating_off_value",
float(off_value)).Attr(
"integer_off_value",
int(off_value)).Attr(
"dtype", dtype).
Output("out").Build().InferAndTryRun().RemoteBlobList()[0])
if axis != (out_ndims - 1):
dim_list = list(range(0, out_ndims))
dim_list.insert(axis, out_ndims - 1)
dim_list.pop()
return flow.transpose(out, dim_list)
else:
return out
def upsample(
x: remote_blob_util.BlobDef,
size: Sequence[int] = (2, 2),
data_format: str = "NCHW",
interpolation: str = "nearest",
name: str = "Upsample2D",
):
r"""Upsample Operation
Args:
x ([type]): Input `Blob`.
size (tuple, optional): (height_scale,width_scale) Defaults to (2, 2).
data_format (str, optional): A string specifies the format of the input `Blob`, one of "NCHW" or "NHWC" (default: "NCHW"). "NCHW" cooresponds to channels_first, i.e. the input `Blob` with shape (batch_size, channels, height, width).
"NHWC" cooresponds to channels_last, i.e. the input `Blob` with shape (batch_size, height, width, channels).. Defaults to "NCHW".
interpolation (str, optional): Image interpolation algorithm to enlarge the image size. Defaults to "nearest".
name ([type], optional): This layer's name. Defaults to None.
Raises:
ValueError: interpolation must be "nearest" or "bilinear".
ValueError: data_format must be "NHWC" or "NCHW"
Returns:
[type]: remote_blob_util.BlobDef: A `Blob` with new shape of input. if input size is(2,2),then the new shape is [N, C, 2H, 2W].
"""
if isinstance(size, int):
height_scale = size
width_scale = size
else:
assert isinstance(size, (list, tuple))
assert len(size) == 2
height_scale = size[0]
width_scale = size[1]
if interpolation != "nearest" and interpolation != "bilinear":
raise ValueError('interpolation must be "nearest" or "bilinear".')
if data_format.upper() != "NCHW" and data_format.upper() != "NHWC":
raise ValueError('data_format must be "NHWC" or "NCHW".')
need_transpose = 0
if data_format.upper() == "NHWC":
need_transpose = 1
if need_transpose:
x = flow.transpose(x, perm=[0, 3, 1, 2])
op = (
flow.user_op_builder(name)
.Op("upsample")
.Input("x", [x])
.Output("y")
.Attr("height_scale", float(height_scale))
.Attr("width_scale", float(width_scale))
.Attr("data_format", "channels_first")
.Attr("interpolation", interpolation)
.Build()
)
output = op.InferAndTryRun().SoleOutputBlob()
if need_transpose:
output = flow.transpose(output, perm=[0, 2, 3, 1])
return output
def trt_transpose_job(x=flow.FixedTensorDef(input_shape, dtype=dtype)):
return flow.transpose(x, perm=permute)
def vgg16bn(self,
images,
trainable=True,
need_transpose=False,
channel_last=False,
training=True,
wd=1.0 / 32768,
reuse=False):
def _get_regularizer():
return flow.regularizers.l2(0.00005)
def conv2d_layer(
name,
input,
filters,
kernel_size=3,
strides=1,
padding="SAME",
data_format="NCHW",
dilation_rate=1,
activation="Relu",
use_bias=True,
weight_initializer=flow.variance_scaling_initializer(
2, 'fan_out', 'random_normal', data_format="NCHW"),
bias_initializer=flow.zeros_initializer(),
weight_regularizer=_get_regularizer(), # weight_decay
bias_regularizer=_get_regularizer(),
bn=True,
reuse=False):
name_ = name if reuse == False else name + "_reuse"
weight_shape = (filters, input.shape[1], kernel_size, kernel_size)
weight = flow.get_variable(
name + "_weight",
shape=weight_shape,
dtype=input.dtype,
initializer=weight_initializer,
)
output = flow.nn.conv2d(input,
weight,
strides,
padding,
data_format,
dilation_rate,
name=name_)
if use_bias:
bias = flow.get_variable(
name + "_bias",
shape=(filters, ),
dtype=input.dtype,
initializer=bias_initializer,
)
output = flow.nn.bias_add(output, bias, data_format)
if activation is not None:
if activation == "Relu":
if bn:
# use of_layers(layers) batch_norm
output = layers.batch_norm(output,
name + "_bn",
reuse=reuse)
output = flow.nn.relu(output)
else:
output = flow.nn.relu(output)
else:
raise NotImplementedError
return output
def _conv_block(in_blob, index, filters, conv_times, reuse=False):
conv_block = []
conv_block.insert(0, in_blob)
for i in range(conv_times):
conv_i = conv2d_layer(name="conv{}".format(index),
input=conv_block[i],
filters=filters,
kernel_size=3,
strides=1,
bn=True,
reuse=reuse)
conv_block.append(conv_i)
index += 1
return conv_block
if need_transpose:
images = flow.transpose(images,
name="transpose",
perm=[0, 3, 1, 2])
if channel_last:
# if channel_last=True, then change mode from 'nchw'to 'nhwc'
images = flow.transpose(images,
name="transpose",
perm=[0, 2, 3, 1])
conv1 = _conv_block(images, 0, 64, 2, reuse=reuse)
# pool1 = flow.nn.max_pool2d(conv1[-1], 2, 2, "VALID", "NCHW", name="pool1")
pool1 = layers.max_pool2d(conv1[-1], 2, 2, name="pool1", reuse=reuse)
conv2 = _conv_block(pool1, 2, 128, 2, reuse=reuse)
# pool2 = flow.nn.max_pool2d(conv2[-1], 2, 2, "VALID", "NCHW", name="pool2")
pool2 = layers.max_pool2d(conv2[-1], 2, 2, name="pool2", reuse=reuse)
conv3 = _conv_block(pool2, 4, 256, 3, reuse=reuse)
# pool3 = flow.nn.max_pool2d(conv3[-1], 2, 2, "VALID", "NCHW", name="pool3")
pool3 = layers.max_pool2d(conv3[-1], 2, 2, name="pool3", reuse=reuse)
conv4 = _conv_block(pool3, 7, 512, 3, reuse=reuse)
# pool4 = flow.nn.max_pool2d(conv4[-1], 2, 2,
# "VALID", "NCHW", name="pool4")
pool4 = layers.max_pool2d(conv4[-1], 2, 2, name="pool4", reuse=reuse)
conv5 = _conv_block(pool4, 10, 512, 3, reuse=reuse)
# pool5 = flow.nn.max_pool2d(conv5[-1], 2, 2, "VALID", "NCHW", name="pool5")
pool5 = layers.max_pool2d(conv5[-1], 2, 2, name="pool5", reuse=reuse)
return pool5
def build_network(self, input_data, need_transpose, data_format, class_num=1000, prefix="", **configs):
self.config_map.update(configs)
if need_transpose:
input_data = flow.transpose(input_data, name="transpose", perm=[0, 3, 1, 2])
first_c = int(round(self.config_map['firstconv_filter_num']*self.multiplier))
first_layer = mobilenet_unit(
data=input_data,
num_filter=first_c,
kernel=(3,3),
stride=(2,2),
pad="same",
data_format=data_format,
if_act=True,
prefix= prefix+'-Conv'
)
last_bottleneck_layer = first_layer
in_c = first_c
for i, layer_setting in enumerate(self.config_map['bottleneck_params_list']):
t, c, s, sc = layer_setting
if i == 0:
last_bottleneck_layer = inverted_residual_unit(
data=last_bottleneck_layer,
num_in_filter=in_c,
num_filter=int(round(c*self.multiplier)),
ifshortcut=sc,
stride=(s,s),
kernel=(3,3),
pad="same",
expansion_factor=t,
prefix= prefix+'-expanded_conv',
data_format=data_format,
has_expand=0
)
in_c = int(round(c*self.multiplier))
else:
last_bottleneck_layer = inverted_residual_unit(
data=last_bottleneck_layer,
num_in_filter=in_c,
num_filter=int(round(c*self.multiplier)),
ifshortcut=sc,
stride=(s,s),
kernel=(3,3),
pad="same",
expansion_factor=t,
data_format=data_format,
prefix= prefix+'-expanded_conv_%d'%i
)
in_c = int(round(c*self.multiplier))
last_fm = mobilenet_unit(
data=last_bottleneck_layer,
num_filter=int(1280 * self.multiplier) if self.multiplier > 1.0 else 1280,
kernel=(1,1),
stride=(1,1),
pad="valid",
data_format=data_format,
if_act=True,
prefix=prefix+'-Conv_1'
)
# global average pooling
pool_size = int(self.data_wh[0] / 32)
pool = flow.nn.avg_pool2d(
last_fm, ksize=pool_size, strides=1, padding="VALID", data_format="NCHW", name="pool5",
)
fc = flow.layers.dense(
flow.reshape(pool, (pool.shape[0], -1)),
units=class_num,
use_bias=False,
kernel_initializer=_get_initializer("dense_weight"),
bias_initializer=_get_initializer("bias"),
kernel_regularizer=_get_regularizer("dense_weight"),
bias_regularizer=_get_regularizer("bias"),
name=prefix+'-fc',
)
return fc
def _AttentionLayer(from_blob,
to_blob,
addr_blob,
num_attention_heads=1,
size_per_head=512,
query_act=op_conf_util.kNone,
key_act=op_conf_util.kNone,
value_act=op_conf_util.kNone,
attention_probs_dropout_prob=0.0,
initializer_range=0.02,
do_return_2d_tensor=False,
batch_size=None,
from_seq_length=None,
to_seq_length=None):
def TransposeForScores(input_blob, num_attention_heads, seq_length, width):
output_blob = flow.reshape(
input_blob, [-1, seq_length, num_attention_heads, width])
output_blob = flow.transpose(output_blob, perm=[0, 2, 1, 3])
return output_blob
from_blob_2d = flow.reshape(from_blob,
[-1, num_attention_heads * size_per_head])
to_blob_2d = flow.reshape(to_blob,
[-1, num_attention_heads * size_per_head])
query_blob = _FullyConnected(
from_blob_2d,
input_size=num_attention_heads * size_per_head,
units=num_attention_heads * size_per_head,
activation=query_act,
name="query",
weight_initializer=CreateInitializer(initializer_range))
key_blob = _FullyConnected(
to_blob_2d,
input_size=num_attention_heads * size_per_head,
units=num_attention_heads * size_per_head,
activation=key_act,
name="key",
weight_initializer=CreateInitializer(initializer_range))
value_blob = _FullyConnected(
to_blob_2d,
input_size=num_attention_heads * size_per_head,
units=num_attention_heads * size_per_head,
activation=value_act,
name="value",
weight_initializer=CreateInitializer(initializer_range))
query_blob = TransposeForScores(query_blob, num_attention_heads,
from_seq_length, size_per_head)
key_blob = TransposeForScores(key_blob, num_attention_heads, to_seq_length,
size_per_head)
attention_scores_blob = flow.matmul(query_blob, key_blob, transpose_b=True)
attention_scores_blob = attention_scores_blob * (
1.0 / math.sqrt(float(size_per_head)))
attention_scores_blob = attention_scores_blob + addr_blob
attention_probs_blob = flow.nn.softmax(attention_scores_blob)
attention_probs_blob = _Dropout(attention_probs_blob,
attention_probs_dropout_prob)
value_blob = flow.reshape(
value_blob, [-1, to_seq_length, num_attention_heads, size_per_head])
value_blob = flow.transpose(value_blob, perm=[0, 2, 1, 3])
context_blob = flow.matmul(attention_probs_blob, value_blob)
context_blob = flow.transpose(context_blob, perm=[0, 2, 1, 3])
if do_return_2d_tensor:
context_blob = flow.reshape(context_blob,
[-1, num_attention_heads * size_per_head])
else:
context_blob = flow.reshape(
context_blob,
[-1, from_seq_length, num_attention_heads * size_per_head])
return context_blob
def Resnet100(input_blob,
embedding_size,
fc_type="GDC",
bn_is_training=True,
**kw):
filter_list = [64, 64, 128, 256, 512]
num_stages = 4
units = [3, 13, 30, 3]
input_blob = flow.transpose(input_blob,
name="transpose",
perm=[0, 3, 1, 2])
input_blob = _conv2d_layer(
name="conv0",
input=input_blob,
filters=filter_list[0],
kernel_size=3,
strides=[1, 1],
padding="same",
use_bias=False,
dilation_rate=1,
activation=None,
)
input_blob = _batch_norm(input_blob,
epsilon=2e-5,
is_training=bn_is_training,
name="bn0")
input_blob = _prelu(input_blob, name="relu0")
for i in range(num_stages):
input_blob = residual_unit_v3(
input_blob,
filter_list[i + 1],
[2, 2],
False,
bn_is_training=bn_is_training,
name="stage%d_unit%d" % (i + 1, 1),
)
for j in range(units[i] - 1):
input_blob = residual_unit_v3(
input_blob,
filter_list[i + 1],
[1, 1],
True,
bn_is_training=bn_is_training,
name="stage%d_unit%d" % (i + 1, j + 2),
)
if fc_type == "GDC":
input_blob = Linear(
input_blob,
num_filter=512,
num_group=512,
kernel=7,
pad="valid",
stride=[1, 1],
bn_is_training=bn_is_training,
name="conv_6dw7_7",
)
input_blob = flow.reshape(input_blob, (input_blob.shape[0], -1))
pre_fc1 = flow.layers.dense(
inputs=input_blob,
units=embedding_size,
activation=None,
use_bias=True,
kernel_initializer=_get_initializer(),
bias_initializer=flow.zeros_initializer(),
kernel_regularizer=_get_regularizer(),
bias_regularizer=_get_regularizer(),
trainable=True,
name="pre_fc1",
)
fc1 = _batch_norm(
pre_fc1,
epsilon=2e-5,
center=True,
scale=False,
is_training=bn_is_training,
name="fc1",
)
elif fc_type == "E":
input_blob = _batch_norm(input_blob,
epsilon=2e-5,
is_training=bn_is_training,
name="bn1")
input_blob = _dropout(input_blob, dropout_prob=0.4)
input_blob = flow.reshape(input_blob, (input_blob.shape[0], -1))
pre_fc1 = flow.layers.dense(
inputs=input_blob,
units=embedding_size,
activation=None,
use_bias=True,
kernel_initializer=_get_initializer(),
bias_initializer=flow.zeros_initializer(),
kernel_regularizer=_get_regularizer(),
bias_regularizer=_get_regularizer(),
trainable=True,
name="pre_fc1",
)
fc1 = _batch_norm(
pre_fc1,
epsilon=2e-5,
center=True,
scale=False,
is_training=bn_is_training,
name="fc1",
)
elif fc_type == "FC":
input_blob = _batch_norm(input_blob,
epsilon=2e-5,
is_training=bn_is_training,
name="bn1")
input_blob = flow.reshape(input_blob, (input_blob.shape[0], -1))
pre_fc1 = flow.layers.dense(
inputs=input_blob,
units=embedding_size,
activation=None,
use_bias=True,
kernel_initializer=_get_initializer(),
bias_initializer=flow.zeros_initializer(),
kernel_regularizer=_get_regularizer(),
bias_regularizer=_get_regularizer(),
trainable=True,
name="pre_fc1",
)
fc1 = _batch_norm(
pre_fc1,
epsilon=2e-5,
center=True,
scale=False,
is_training=bn_is_training,
name="fc1",
)
else:
print("unimplemented")
return fc1
def conv3d(
inputs: remote_blob_util.BlobDef,
filters: int,
kernel_size: Union[int, Sequence[int]] = 1,
strides: Union[int, Sequence[int]] = 1,
padding: Union[str, Tuple[IntPair, IntPair, IntPair, IntPair,
IntPair]] = "VALID",
data_format: str = "NCDHW",
dilation_rate: Optional[Union[int, IntPair]] = None,
groups: int = 1,
activation: Optional[Callable[[remote_blob_util.BlobDef, str],
remote_blob_util.BlobDef]] = None,
use_bias: bool = True,
kernel_initializer: Optional[op_conf_util.InitializerConf] = None,
bias_initializer: Optional[op_conf_util.InitializerConf] = None,
kernel_regularizer: Optional[op_conf_util.RegularizerConf] = None,
bias_regularizer: Optional[op_conf_util.RegularizerConf] = None,
trainable: bool = True,
name: str = "Conv3d",
weight_name: Optional[str] = None,
bias_name: Optional[str] = None,
) -> remote_blob_util.BlobDef:
r"""3D convolution layer.
Args:
inputs (remote_blob_util.BlobDef): A 5D input `Blob`.
filters (int): An integer specifies the dimensionality of the output space.
kernel_size (Union[int, List[int], Sequence[int]], optional): An integer or tuple/list specifies the height and width of the convolution window.
When it is an integer, a square window is applied to the input. Defaults to 1.
strides (Union[int, List[int], Sequence[int]], optional): An integer or tuple/list specifies the strides of the convolution window along the height and width.
When it is an integer, the same value for the all spatial dimesions is applied. Defaults to 1.
padding (str, Tuple[IntPair, IntPair, IntPair, IntPair, IntPair], optional): padding: `string` `"SAME"` or `"SAME_LOWER"` or `"SAME_UPPER"` or `"VALID" or Tuple[IntPair, IntPair, IntPair, IntPair, IntPair]` indicating the type of padding algorithm to use, or a list indicating the explicit paddings at the start and end of each dimension. Defaults to "VALID".
data_format (str, optional): A string specifies the format of the input `Blob`, one of "NCDHW" or "NDHWC" (default: "NCDHW"). "NCDHW" cooresponds to channels_first, i.e. the input `Blob` with shape (batch_size, channels, depth, height, width).
"NDHWC" cooresponds to channels_last, i.e. the input `Blob` with shape (batch_size, channels, depth, height, width). Defaults to "NCDHW".
dilation_rate (int, optional): An integer or tuple/list specifies the dilation rate for the dilated convolution. When it is an integer, the same dilation rate is applied for the all dimensions. Defaults to 1.
groups (int, optional): A positive integer specifies number of groups for the Group conv. Defaults to 1.
activation (Optional[ Callable[[remote_blob_util.BlobDef, str], remote_blob_util.BlobDef] ], optional): Activation function. Defaults to None.
use_bias (bool, optional): A boolean specifies whether to use a bias vector. Defaults to True.
kernel_initializer (Optional[op_conf_util.InitializerConf], optional): Initializer for the kernel weights matrix. Defaults to None.
bias_initializer (Optional[op_conf_util.InitializerConf], optional): Initializer for the bias vector. Defaults to None.
kernel_regularizer (Optional[op_conf_util.RegularizerConf], optional): Regularizer for the kernel weights matrix. Defaults to None.
bias_regularizer (Optional[op_conf_util.RegularizerConf], optional): Regularizer for the bias vector . Defaults to None.
trainable (bool, optional): A boolean specifies whether to train variables. Defaults to True.
name (Optional[str], optional): This layer's name. Defaults to None.
weight_name (Optional[str], optional): This weight's name. Defaults to None.
bias_name (Optional[str], optional): This bias's name. Defaults to None.
Raises:
ValueError: If the type of kernel_size is not one of integer, list, tuple.
ValueError: The number of groups must be positive and number of filters must be divisible by it.
ValueError: If data_format is not one of 'NCDHW', 'NDHWC'.
ValueError: If number of input channels is not divisible by number of groups or less than number of groups.
ValueError: Number of group must be one when data_format is 'NDHWC'.
Returns:
remote_blob_util.BlobDef: A 5D `Blob` with the shape of (batch_size, filters, new_height, new_width).
"""
need_transpose = 0
if data_format.upper(
) == "NDHWC": # NDHWC is not supported before cudnn 8.0
need_transpose = 1
data_format = "NCDHW"
if need_transpose:
inputs = flow.transpose(inputs, perm=[0, 4, 1, 2, 3])
if isinstance(kernel_size, int):
kernel_size = (kernel_size, kernel_size, kernel_size)
else:
assert isinstance(kernel_size, (list, tuple))
assert len(kernel_size) == 3
kernel_size = tuple(kernel_size)
assert isinstance(groups, int)
assert groups > 0
assert groups <= filters
assert filters % groups == 0
if data_format.upper() == "NCDHW":
assert groups <= inputs.shape[1]
assert inputs.shape[1] % groups == 0
weight_shape = (filters, inputs.shape[1] // groups) + kernel_size
elif data_format.upper() == "NDHWC":
assert groups == 1
assert groups <= inputs.shape[3]
assert inputs.shape[3] % groups == 0
weight_shape = (
filters,
kernel_size[0],
kernel_size[1],
kernel_size[2],
inputs.shape[4] // groups,
)
else:
raise ValueError("data_format must be in NCHW or NHWC")
if kernel_initializer is None:
kernel_initializer = flow.xavier_uniform_initializer(
data_format=data_format)
if weight_name is None:
with flow.scope.namespace(name):
weight = flow.get_variable(
name="weight",
shape=weight_shape,
dtype=inputs.dtype,
initializer=kernel_initializer,
regularizer=kernel_regularizer,
trainable=trainable,
model_name="weight",
reuse=False,
)
else:
weight = flow.get_variable(
name=weight_name,
shape=weight_shape,
dtype=inputs.dtype,
initializer=kernel_initializer,
regularizer=kernel_regularizer,
trainable=trainable,
model_name="weight",
reuse=False,
)
output = flow.nn.conv3d(
inputs,
weight,
strides,
padding,
data_format,
dilation_rate,
groups=groups,
name=name,
)
if use_bias:
if bias_initializer is None:
bias_initializer = flow.constant_initializer(0)
if bias_name is None:
with flow.scope.namespace(name):
bias = flow.get_variable(
name="bias",
shape=(filters, ),
dtype=inputs.dtype,
initializer=bias_initializer,
regularizer=bias_regularizer,
trainable=trainable,
model_name="bias",
reuse=False,
)
else:
bias = flow.get_variable(
name=bias_name,
shape=(filters, ),
dtype=inputs.dtype,
initializer=bias_initializer,
regularizer=bias_regularizer,
trainable=trainable,
model_name="bias",
reuse=False,
)
with flow.scope.namespace(name):
output = flow.nn.bias_add(output,
bias,
data_format,
name="bias_add")
if callable(activation):
with flow.scope.namespace(name):
output = activation(output, name="activation")
if need_transpose:
output = flow.transpose(output, perm=[0, 2, 3, 4, 1])
return output
def build_network(self,
input_data,
data_format,
class_num=1000,
prefix="",
**configs):
self.config_map.update(configs)
# input_data = flow.math.multiply(input_data, 2.0 / 255.0)
# input_data = flow.math.add(input_data, -1)
if data_format == "NCHW":
input_data = flow.transpose(input_data,
name="transpose",
perm=[0, 3, 1, 2])
first_c = int(
round(self.config_map["firstconv_filter_num"] * self.multiplier))
first_layer = mobilenet_unit(
data=input_data,
num_filter=first_c,
kernel=(3, 3),
stride=(2, 2),
pad="same",
data_format=data_format,
if_act=True,
prefix=prefix + "-Conv",
)
last_bottleneck_layer = first_layer
in_c = first_c
for i, layer_setting in enumerate(
self.config_map["bottleneck_params_list"]):
t, c, s, sc = layer_setting
if i == 0:
last_bottleneck_layer = inverted_residual_unit(
data=last_bottleneck_layer,
num_in_filter=in_c,
num_filter=int(round(c * self.multiplier)),
ifshortcut=sc,
stride=(s, s),
kernel=(3, 3),
pad="same",
expansion_factor=t,
prefix=prefix + "-expanded_conv",
data_format=data_format,
has_expand=0,
)
in_c = int(round(c * self.multiplier))
else:
last_bottleneck_layer = inverted_residual_unit(
data=last_bottleneck_layer,
num_in_filter=in_c,
num_filter=int(round(c * self.multiplier)),
ifshortcut=sc,
stride=(s, s),
kernel=(3, 3),
pad="same",
expansion_factor=t,
data_format=data_format,
prefix=prefix + "-expanded_conv_%d" % i,
)
in_c = int(round(c * self.multiplier))
last_fm = mobilenet_unit(
data=last_bottleneck_layer,
# num_filter=int(1280 * self.multiplier) if self.multiplier > 1.0 else 1280,
# gr to confirm
num_filter=int(256 *
self.multiplier) if self.multiplier > 1.0 else 256,
kernel=(1, 1),
stride=(1, 1),
pad="valid",
data_format=data_format,
if_act=True,
prefix=prefix + "-Conv_1",
)
base_only = True
if base_only:
return last_fm
else:
raise NotImplementedError
def vgg(images,
cfg,
optimizer,
trainable=True,
need_transpose=False,
training=True,
wd=1.0 / 32768,
model_weight=True,
bn=True):
if need_transpose:
images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
conv1 = _conv_block(images, 0, cfg, 2, optimizer, model_weight, bn=bn)
pool1 = flow.nn.max_pool2d(conv1[-1], 2, 2, "VALID", "NCHW", name="pool1")
conv2 = _conv_block(pool1, 2, cfg, 2, optimizer, model_weight, bn=bn)
pool2 = flow.nn.max_pool2d(conv2[-1], 2, 2, "VALID", "NCHW", name="pool2")
conv3 = _conv_block(pool2, 4, cfg, 3, optimizer, model_weight, bn=bn)
pool3 = flow.nn.max_pool2d(conv3[-1], 2, 2, "VALID", "NCHW", name="pool3")
conv4 = _conv_block(pool3, 7, cfg, 3, optimizer, model_weight, bn=bn)
pool4 = flow.nn.max_pool2d(conv4[-1], 2, 2, "VALID", "NCHW", name="pool4")
conv5 = _conv_block(pool4, 10, cfg, 3, optimizer, model_weight, bn=bn)
pool5 = flow.nn.max_pool2d(conv5[-1], 2, 2, "VALID", "NCHW", name="pool5")
pool5 = flow.reshape(pool5, [pool5.shape[0], -1])
dense0 = flow.layers.dense(
inputs=pool5,
units=cfg[13],
activation=flow.nn.relu,
use_bias=True,
kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1),
trainable=trainable,
name="dense0",
)
dense1 = flow.layers.dense(
inputs=dense0,
units=cfg[14],
activation=flow.nn.relu,
use_bias=True,
kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1),
trainable=trainable,
name="dense1",
)
dense2 = flow.layers.dense(
inputs=dense1,
units=cfg[15],
use_bias=True,
kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1),
trainable=trainable,
name="dense2",
)
# flow.watch(fc8)
def getTypeAndShape(inputs, units):
in_shape = inputs.shape
in_num_axes = len(in_shape)
inputs = (flow.reshape(inputs,
(-1,
in_shape[-1])) if in_num_axes > 2 else inputs)
shape = (units, inputs.shape[1])
dtype = inputs.dtype
return shape, dtype
if model_weight == True:
shape_list = []
dtype_list = []
shape_weight, dtype = getTypeAndShape(pool5, cfg[13])
shape_list.append(shape_weight)
dtype_list.append(dtype)
shape_weight, dtype = getTypeAndShape(dense0, cfg[14])
shape_list.append(shape_weight)
dtype_list.append(dtype)
shape_weight, dtype = getTypeAndShape(dense1, cfg[15])
shape_list.append(shape_weight)
dtype_list.append(dtype)
modelWeight.addDense(dtype_old=dtype_list,
shape=shape_list,
optimizer=optimizer,
dense_num=3)
# shape_weight,dtype=getTypeAndShape(pool5,4096)
# modelWeight.add('fc1'+'-weight',dtype,shape_weight)
# modelWeight.add('fc1'+'-bias',dtype,(4096,))
# shape_weight,dtype=getTypeAndShape(fc6,4096)
# modelWeight.add('fc2'+'-weight',dtype,shape_weight)
# modelWeight.add('fc2'+'-bias',dtype,(4096,))
# shape_weight,dtype=getTypeAndShape(fc7,1000)
# modelWeight.add('fc_final'+'-weight',dtype,shape_weight)
# modelWeight.add('fc_final'+'-bias',dtype,(1000,))
return dense2
def MobileFacenet(input_blob, embedding_size=10, bn_is_training=True):
units = [1, 4, 6, 2]
input_blob = flow.transpose(input_blob,
name="transpose",
perm=[0, 3, 1, 2])
conv_1 = Conv(
input_blob,
num_filter=64,
kernel=3,
stride=[2, 2],
pad="same",
bn_is_training=bn_is_training,
name="conv_1",
)
if units[0] == 1:
conv_2_dw = Conv(
conv_1,
num_filter=64,
kernel=3,
stride=[1, 1],
pad="same",
num_group=64,
bn_is_training=bn_is_training,
name="conv_2_dw",
)
else:
conv_2_dw = Residual(
conv_1,
num_block=units[0],
num_out=64,
kernel=3,
stride=[1, 1],
pad="same",
num_group=64,
bn_is_training=bn_is_training,
name="res_2",
)
conv_23 = DResidual_v1(
conv_2_dw,
num_out=64,
kernel=3,
stride=[2, 2],
pad="same",
num_group=128,
bn_is_training=bn_is_training,
name="dconv_23",
)
conv_3 = Residual(
conv_23,
num_block=units[1],
num_out=64,
kernel=3,
stride=[1, 1],
pad="same",
num_group=128,
bn_is_training=bn_is_training,
name="res_3",
)
conv_34 = DResidual_v1(
conv_3,
num_out=128,
kernel=3,
stride=[2, 2],
pad="same",
num_group=256,
bn_is_training=bn_is_training,
name="dconv_34",
)
conv_4 = Residual(
conv_34,
num_block=units[2],
num_out=128,
kernel=3,
stride=[1, 1],
pad="same",
num_group=256,
bn_is_training=bn_is_training,
name="res_4",
)
conv_45 = DResidual_v1(
conv_4,
num_out=128,
kernel=3,
stride=[2, 2],
pad="same",
num_group=512,
bn_is_training=bn_is_training,
name="dconv_45",
)
conv_5 = Residual(
conv_45,
num_block=units[3],
num_out=128,
kernel=3,
stride=[1, 1],
pad="same",
num_group=256,
bn_is_training=bn_is_training,
name="res_5",
)
conv_6_sep = Conv(
conv_5,
num_filter=512,
kernel=1,
pad="valid",
stride=[1, 1],
bn_is_training=bn_is_training,
name="conv_6sep",
)
conv_6_dw = Linear(
conv_6_sep,
num_filter=512,
num_group=512,
kernel=7,
pad="valid",
stride=[1, 1],
bn_is_training=bn_is_training,
name="conv_6dw7_7",
)
conv_6_dw = flow.reshape(conv_6_dw, (conv_6_dw.shape[0], -1))
conv_6_f = flow.layers.dense(
inputs=conv_6_dw,
units=embedding_size,
activation=None,
use_bias=True,
kernel_initializer=_get_initializer(),
bias_initializer=flow.zeros_initializer(),
kernel_regularizer=_get_regularizer(),
bias_regularizer=_get_regularizer(),
trainable=True,
name="pre_fc1",
)
fc1 = _batch_norm(
conv_6_f,
epsilon=2e-5,
scale=False,
center=True,
is_training=bn_is_training,
name="fc1",
)
return fc1
def inceptionv3(images,
trainable=True,
need_transpose=False,
channel_last=False):
if need_transpose:
images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
if channel_last:
# if channel_last=True, then change mode from 'nchw' to 'nhwc'
images = flow.transpose(images, name="transpose", perm=[0, 2, 3, 1])
with flow.scope.namespace("InceptionV3"):
# conv0: 299 x 299 x 3
conv0 = conv2d_layer("conv0",
images,
filters=32,
kernel_size=3,
strides=2,
padding="VALID")
conv1 = conv2d_layer("conv1",
conv0,
filters=32,
kernel_size=3,
strides=1,
padding="VALID")
conv2 = conv2d_layer("conv2",
conv1,
filters=64,
kernel_size=3,
strides=1,
padding="SAME")
pool1 = flow.nn.max_pool2d(conv2,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="pool1")
conv3 = conv2d_layer("conv3",
pool1,
filters=80,
kernel_size=1,
strides=1,
padding="VALID")
conv4 = conv2d_layer("conv4",
conv3,
filters=192,
kernel_size=3,
strides=1,
padding="VALID")
pool2 = flow.nn.max_pool2d(conv4,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="pool2")
# mixed_0 ~ mixed_2
mixed_0 = InceptionA(pool2, 0)
mixed_1 = InceptionA(mixed_0, 1)
mixed_2 = InceptionA(mixed_1, 2)
# mixed_3
mixed_3 = InceptionB(mixed_2, 3)
# mixed_4 ~ mixed_7
mixed_4 = InceptionC(mixed_3, 4, 128)
mixed_5 = InceptionC(mixed_4, 5, 160)
mixed_6 = InceptionC(mixed_5, 6, 160)
mixed_7 = InceptionC(mixed_6, 7, 192)
# mixed_8
mixed_8 = InceptionD(mixed_7, 8)
# mixed_9 ~ mixed_10
mixed_9 = InceptionE(mixed_8, 9, 'avg')
mixed_10 = InceptionE(mixed_9, 10, 'max')
pool3 = flow.nn.avg_pool2d(mixed_10,
ksize=8,
strides=1,
padding="VALID",
data_format="NCHW",
name="pool3")
# TODO: Need to transpose weight when converting model from TF to OF if
# you want to use layers.dense interface.
fc1 = flow.layers.dense(
inputs=flow.reshape(pool3, [pool3.shape[0], -1]),
units=1000,
activation=None,
use_bias=True,
kernel_initializer=flow.truncated_normal(0.816496580927726),
bias_initializer=flow.constant_initializer(),
trainable=trainable,
name="fc1",
)
return fc1
def alexnet(images, need_transpose=False, channel_last=False, training=True):
if need_transpose:
images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
if channel_last:
# if channel_last=True, then change mode from 'nchw' to 'nhwc'
images = flow.transpose(images, name="transpose", perm=[0, 2, 3, 1])
conv1 = conv2d_layer("conv1",
images,
filters=64,
kernel_size=11,
strides=4,
padding="VALID")
pool1 = flow.nn.avg_pool2d(conv1, 3, 2, "VALID", "NCHW", name="pool1")
conv2 = conv2d_layer("conv2", pool1, filters=192, kernel_size=5)
pool2 = flow.nn.avg_pool2d(conv2, 3, 2, "VALID", "NCHW", name="pool2")
conv3 = conv2d_layer("conv3", pool2, filters=384)
conv4 = conv2d_layer("conv4", conv3, filters=384)
conv5 = conv2d_layer("conv5", conv4, filters=256)
pool5 = flow.nn.avg_pool2d(conv5, 3, 2, "VALID", "NCHW", name="pool5")
if len(pool5.shape) > 2:
pool5 = flow.reshape(pool5, shape=(pool5.shape[0], -1))
fc1 = flow.layers.dense(
inputs=pool5,
units=4096,
activation=flow.nn.relu,
use_bias=True,
#kernel_initializer=flow.random_uniform_initializer(),
kernel_initializer=_get_kernel_initializer(),
bias_initializer=_get_bias_initializer(),
kernel_regularizer=_get_regularizer(),
bias_regularizer=_get_regularizer(),
name="fc1",
)
dropout1 = flow.nn.dropout(fc1, rate=0.5)
fc2 = flow.layers.dense(
inputs=dropout1,
units=4096,
activation=flow.nn.relu,
use_bias=True,
kernel_initializer=_get_kernel_initializer(),
bias_initializer=_get_bias_initializer(),
kernel_regularizer=_get_regularizer(),
bias_regularizer=_get_regularizer(),
name="fc2",
)
dropout2 = flow.nn.dropout(fc2, rate=0.5)
fc3 = flow.layers.dense(
inputs=dropout2,
units=1000,
activation=None,
use_bias=False,
kernel_initializer=_get_kernel_initializer(),
kernel_regularizer=_get_regularizer(),
bias_initializer=False,
name="fc3",
)
return fc3
def alexnet(images, labels, trainable=True):
transposed = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
conv1 = _conv2d_layer("conv1",
transposed,
filters=64,
kernel_size=11,
strides=4,
padding="VALID")
pool1 = flow.nn.avg_pool2d(conv1, 3, 2, "VALID", "NCHW", name="pool1")
conv2 = _conv2d_layer("conv2", pool1, filters=192, kernel_size=5)
pool2 = flow.nn.avg_pool2d(conv2, 3, 2, "VALID", "NCHW", name="pool2")
conv3 = _conv2d_layer("conv3", pool2, filters=384)
conv4 = _conv2d_layer("conv4", conv3, filters=384)
conv5 = _conv2d_layer("conv5", conv4, filters=256)
pool5 = flow.nn.avg_pool2d(conv5, 3, 2, "VALID", "NCHW", name="pool5")
def _get_initializer():
return flow.random_uniform_initializer()
if len(pool5.shape) > 2:
pool5 = flow.reshape(pool5, shape=(pool5.shape[0], -1))
fc1 = flow.layers.dense(
inputs=pool5,
units=4096,
activation=flow.math.relu,
use_bias=False,
kernel_initializer=_get_initializer(),
bias_initializer=False,
trainable=trainable,
name="fc1",
)
dropout1 = fc1
fc2 = flow.layers.dense(
inputs=dropout1,
units=4096,
activation=flow.math.relu,
use_bias=False,
kernel_initializer=_get_initializer(),
bias_initializer=False,
trainable=trainable,
name="fc2",
)
dropout2 = fc2
fc3 = flow.layers.dense(
inputs=dropout2,
units=1001,
activation=None,
use_bias=False,
kernel_initializer=_get_initializer(),
bias_initializer=False,
trainable=trainable,
name="fc3",
)
return fc3
def forward(self, inputs, targets):
n = inputs.shape[0]
# Compute pairwise distance, replace by the official when merged
tempname = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S.%f')
shape_tensor = flow.constant(value=0.0,
dtype=flow.float32,
shape=(n, n))
if self.distance == 'euclidean':
blob_2 = flow.get_variable(
"blob_2_" + tempname,
shape=inputs.shape,
initializer=flow.constant_initializer(2),
dtype=inputs.dtype)
dist = flow.math.pow(inputs, blob_2)
dist = flow.math.reduce_sum(dist, axis=1, keepdims=True)
dist = flow.broadcast_like(dist, shape_tensor)
tempdist = flow.transpose(dist)
dist = dist + tempdist
inputs_t = flow.transpose(inputs)
dist = addmm(dist, inputs, inputs_t, beta=1, alpha=-2)
dist = flow.clamp(dist, min_value=1e-12)
dist = flow.math.sqrt(dist)
elif self.distance == 'cosine':
#fnorm=flow.math.l2_normalize(inputs, axis=1)
fnorm = flow.math.reduce_mean(flow.math.divide(
inputs, flow.math.l2_normalize(inputs, axis=1)),
axis=1,
keepdims=True)
expand_fnorm = flow.broadcast_like(fnorm,
like=inputs,
broadcast_axes=[1])
l2norm = flow.math.divide(inputs, expand_fnorm)
l2norm_t = flow.transpose(l2norm, perm=(1, 0))
dist = flow.math.negative(flow.matmul(l2norm, l2norm_t))
# For each anchor, find the hardest positive and negative
mask = math.equal(
flow.broadcast_like(targets, like=shape_tensor,
broadcast_axes=[1]),
flow.transpose(flow.broadcast_like(targets,
like=shape_tensor,
broadcast_axes=[1]),
perm=(1, 0),
batch_axis_non_change=True))
mask_rev = math.not_equal(
flow.broadcast_like(targets, like=shape_tensor,
broadcast_axes=[1]),
flow.transpose(flow.broadcast_like(targets,
like=shape_tensor,
broadcast_axes=[1]),
perm=(1, 0),
batch_axis_non_change=True))
dist_ap, dist_an = [], []
for i in range(n):
temp_dist = flow.slice_v2(dist, [(i, i + 1, 1)])
temp_mask = flow.slice_v2(mask, [(i, i + 1, 1)])
temp_mask_rev = flow.slice_v2(mask_rev, [(i, i + 1, 1)])
temp_dist_ap = flow.expand_dims(
math.reduce_max(
flow.gather_nd(temp_dist, flow.where(temp_mask))), 0)
temp_dist_an = flow.expand_dims(
math.reduce_min(
flow.gather_nd(temp_dist, flow.where(temp_mask_rev))), 0)
dist_ap.append(temp_dist_ap)
dist_an.append(temp_dist_an)
dist_ap = flow.concat(dist_ap, 0)
dist_an = flow.concat(dist_an, 0)
y = flow.ones_like(dist_an)
return self._MarginRankingLoss(dist_an, dist_ap, y)
def TransposeForScores(input_blob, num_attention_heads, seq_length, width):
output_blob = flow.reshape(
input_blob, [-1, seq_length, num_attention_heads, width])
output_blob = flow.transpose(output_blob, perm=[0, 2, 1, 3])
return output_blob
def ccmp(self, input, kernel_size, stride):
input = flow.transpose(input, perm=[0, 3, 2, 1])
input = flow.nn.max_pool2d(input, kernel_size, stride, padding="VALID")
input = flow.transpose(input, perm=[0, 3, 2, 1])
return input
