def onnx_extract_residual(graph, node, layers):
'''
Construct residual layer from ONNX op
Parameters
----------
graph : ONNX GraphProto
Specifies a GraphProto object.
node : ONNX NodeProto
Specifies a NodeProto object.
layers : list of Layers
Specifies the existing layers of a model.
Returns
-------
:class:`Res`
'''
previous = onnx_find_previous_compute_layer(graph, node)
if not previous:
src_names = [find_input_layer_name(graph)]
else:
src_names = [p.name for p in previous]
src = [get_dlpy_layer(layers, i) for i in src_names]
return Res(name=node.name, act='identity', src_layers=src)
def test_model22(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
pool1 = Pooling(2)
model1.add(pool1)
conv1 = Conv2d(1, 1, act='identity', src_layers=[pool1])
model1.add(conv1)
model1.add(Res(act='relu', src_layers=[conv1, pool1]))
model1.add(Pooling(2))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load')
self.s.table.loadtable(caslib=caslib,
casout={'name': 'eee', 'replace': True},
path=path)
r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1)
self.assertTrue(r.severity == 0)
model1.deploy(self.data_dir, output_format='onnx')
def test_model3(self):
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
pool1 = Pooling(2)
model1.add(pool1)
conv1 = Conv2d(1, 1, act='identity', src_layers=[pool1])
model1.add(conv1)
model1.add(Res(act='relu', src_layers=[conv1, pool1]))
model1.add(Pooling(2))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
#
# unittest.TestCase.skipTest(self, "Skipping temporarily due to server error.")
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
import tempfile
tmp_dir_to_dump = tempfile.gettempdir()
model1.deploy(tmp_dir_to_dump, output_format='onnx')
import os
os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx"))
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib',
message_level='error',
caslib=caslib)
def downsampling_bottleneck(x, in_depth, out_depth, projection_ratio=4):
'''
Defines the down-sampling bottleneck of ENet
Parameters
----------
x : class:`Layer'
Previous layer to this block
in_depth : int
Depth of the layer fed into this block
out_depth : int
Depth of the output layer of this block
projection_ratio : int, optional
Used to calculate the reduced_depth for intermediate convolution layers
Default: 4
Returns
-------
:class:`Res`
'''
reduced_depth = int(in_depth // projection_ratio)
conv1 = Conv2d(reduced_depth,
3,
stride=2,
padding=1,
act='identity',
include_bias=False)(x)
bn1 = BN(act='relu')(conv1)
conv2 = Conv2d(reduced_depth,
3,
stride=1,
act='identity',
include_bias=False)(bn1)
bn2 = BN(act='relu')(conv2)
conv3 = Conv2d(out_depth, 1, stride=1, act='identity',
include_bias=False)(bn2)
bn3 = BN(act='relu')(conv3)
pool1 = Pooling(2, stride=2)(x)
conv4 = Conv2d(out_depth, 1, stride=1, act='identity',
include_bias=False)(pool1)
bn4 = BN(act='relu')(conv4)
res = Res()([bn3, bn4])
return res
def _add_bert_embedding_layer(self, layer_name, layer_info):
# find input layer(s)
input_layers = self._find_layer_def([
self.ds_layers['token_input'], self.ds_layers['position_input'],
self.ds_layers['segment_input']
], layer_info)
# need tensors for input layers
for ii, ilayer in enumerate(input_layers):
input_layers[ii] = input_layers[ii].input_tenor
# define layer information
new_layer_info = collections.OrderedDict()
# residual layer to sum embeddings
lname = layer_name + '_sum'
new_layer_info['sum_layer'] = dict(
name=lname,
type=BertCommon['layer_types']['noparms'],
dim=None,
ldef=Res(name=lname, act='identity')(input_layers))
# layer normalization layer
lname = layer_name + '_ln'
new_layer_info['layer_norm_layer'] = dict(
name=lname,
type=BertCommon['layer_types']['layernorm'],
dim=[self._config['hidden_size']],
ldef=LayerNormalization(
name=lname,
act='identity',
epsilon=self._config['layer_norm_eps'],
token_size=self._config['hidden_size'],
dropout=self._config['hidden_dropout_prob'])(
new_layer_info['sum_layer']['ldef']))
return new_layer_info
def _inverted_res_block(inputs, in_channels, expansion, stride, alpha,
filters, block_id):
"""
Inverted Residual Block
Parameters
----------
inputs:
Input tensor
in_channels:
Specifies the number of input tensor's channel
expansion:
expansion factor always applied to the input size.
stride:
the strides of the convolution
alpha:
width multiplier.
filters:
the dimensionality of the output space.
block_id:
block id used for naming layers
"""
pointwise_conv_filters = int(filters * alpha)
pointwise_filters = _make_divisible(pointwise_conv_filters, 8)
x = inputs
prefix = 'block_{}_'.format(block_id)
n_groups = in_channels
if block_id:
# Expand
n_groups = expansion * in_channels
x = Conv2d(expansion * in_channels,
1,
include_bias=False,
act='identity',
name=prefix + 'expand')(x)
x = BN(name=prefix + 'expand_BN', act='identity')(x)
else:
prefix = 'expanded_conv_'
# Depthwise
x = GroupConv2d(n_groups,
n_groups,
3,
stride=stride,
act='identity',
include_bias=False,
name=prefix + 'depthwise')(x)
x = BN(name=prefix + 'depthwise_BN', act='relu')(x)
# Project
x = Conv2d(pointwise_filters,
1,
include_bias=False,
act='identity',
name=prefix + 'project')(x)
x = BN(name=prefix + 'project_BN',
act='identity')(x) # identity activation on narrow tensor
if in_channels == pointwise_filters and stride == 1:
return Res(name=prefix + 'add')([inputs, x]), pointwise_filters
return x, pointwise_filters
def test_res_layer2(self):
if not __dev__:
with self.assertRaises(DLPyError):
Res(not_a_parameter=1)
def test_res_layer1(self):
if not __dev__:
dict1 = Res(name='res',
src_layers=[Conv2d(name='conv',
n_filters=32)]).to_model_params()
self.assertTrue(self.sample_syntax['res1'] == dict1)
def _MBConvBlock(inputs, in_channels, out_channels, ksize, stride, expansion, se_ratio, stage_id, block_id,
noskip=False, activation_fn='relu'):
'''
Inverted Residual Block
Parameters
----------
inputs: input tensor
Speecify input tensor for block.
in_channels: integer
Specifies the number of input tensor's channel.
out_channels: integer
Specifies the number of output tensor's channel
ksize:
Specifies the kernel size of the convolution
stride: integer
Specifies the stride of the convolution
expansion: double
Specifies the expansion factor for the input layer.
se_ratio: double
Specifies the ratio to squeeze the input filters for squeeze-and-excitation block.
stage_id: integer
Specifies stage id for naming layers
block_id:
Specifies block id for naming layers
noskip: bool
Specifies whether the skip connection is used. By default, the skip connection is used.
activation_fn:
Specifies activation function
'''
# mobilenetv2 block is also known as inverted residual block, which consists of three convolutions:
# the first is 1*1 convolution for expansion
# the second is depthwise convolution
# the third is 1*1 convolution without any non-linearity for projection
x = inputs
prefix = 'stage_{}_block_{}'.format(stage_id, block_id)
n_groups = in_channels # for expansion=1, n_groups might be different from pointwise_filters
if expansion > 1:
# For MobileNet V2, expansion>1 when stage>0
n_groups = int(expansion * in_channels) ## update n_groups
x = Conv2d(n_groups, 1, include_bias=False, act='identity',
name=prefix + 'expand')(x)
x = BN(name=prefix + 'expand_BN', act='identity')(x)
# Depthwise convolution
x = GroupConv2d(n_groups, n_groups, ksize, stride=stride, act='identity',
include_bias=False, name=prefix + 'depthwise')(x)
x = BN(name=prefix + 'depthwise_BN', act=activation_fn)(x)
# Squeeze-Excitation
if 0 < se_ratio <= 1:
se_input = x # features to be squeezed
x = GlobalAveragePooling2D(name=prefix + "global_avg_pool")(x)
# Squeeze
channels_se = max(1, int(in_channels * se_ratio))
x = Conv2d(channels_se, 1, include_bias=True, act=activation_fn, name=prefix + 'squeeze')(x)
x = Conv2d(n_groups, 1, include_bias=True, act='sigmoid', name=prefix + 'excitation')(x)
x = Reshape(name=prefix + 'reshape', width=n_groups, height=1, depth=1)(x)
x = Scale(name=prefix + 'scale')([se_input, x]) # x = out*w
# Project
x = Conv2d(out_channels, 1, include_bias=False, act='identity', name=prefix + 'project')(x)
x = BN(name=prefix + 'project_BN', act='identity')(x) # identity activation on narrow tensor
# Prepare output for MBConv block
if in_channels == out_channels and stride == 1 and (not noskip):
# dropout can be added.
return Res(name=prefix + 'add_se_residual')([x, inputs])
else:
return x
def _shuffle_unit(inputs,
in_channels,
out_channels,
groups,
bottleneck_ratio,
strides=2,
stage=1,
block=1):
"""
create a shuffle unit
Parameters
----------
inputs:
Input tensor of with `channels_last` data format
in_channels:
number of input channels
out_channels:
number of output channels
strides:
An integer or tuple/list of 2 integers,
groups:
number of groups per channel
bottleneck_ratio: float
bottleneck ratio implies the ratio of bottleneck channels to output channels.
stage:
stage number
block:
block number
"""
prefix = 'stage%d/block%d' % (stage, block)
# if strides >= 2:
# out_channels -= in_channels
# default: 1/4 of the output channel of a ShuffleNet Unit
bottleneck_channels = int(out_channels * bottleneck_ratio)
groups = (1 if stage == 2 and block == 1 else groups)
# x = _group_conv(inputs, in_channels, out_channels = bottleneck_channels,
# groups = (1 if stage == 2 and block == 1 else groups),
# name = '%s/1x1_gconv_1' % prefix)
x = GroupConv2d(bottleneck_channels,
n_groups=(1 if stage == 2 and block == 1 else groups),
act='identity',
width=1,
height=1,
stride=1,
include_bias=False,
name='%s/1x1_gconv_1' % prefix)(inputs)
x = BN(act='relu', name='%s/bn_gconv_1' % prefix)(x)
x = ChannelShuffle(n_groups=groups,
name='%s/channel_shuffle' % prefix)(x)
# depthwise convolutioin
x = GroupConv2d(x.shape[-1],
n_groups=x.shape[-1],
width=3,
height=3,
include_bias=False,
stride=strides,
act='identity',
name='%s/1x1_dwconv_1' % prefix)(x)
x = BN(act=block_act, name='%s/bn_dwconv_1' % prefix)(x)
out_channels = out_channels if strides == 1 else out_channels - in_channels
x = GroupConv2d(out_channels,
n_groups=groups,
width=1,
height=1,
stride=1,
act='identity',
include_bias=False,
name='%s/1x1_gconv_2' % prefix)(x)
x = BN(act=block_act, name='%s/bn_gconv_2' % prefix)(x)
if strides < 2:
ret = Res(act='relu', name='%s/add' % prefix)([x, inputs])
else:
avg = Pooling(width=3,
height=3,
stride=2,
pool='mean',
name='%s/avg_pool' % prefix)(inputs)
ret = Concat(act='relu', name='%s/concat' % prefix)([x, avg])
return ret
def _add_bert_encoding_layer(self, layer_base, src_layers, layer_info):
# find input layer(s)
input_layers = self._find_layer_def(src_layers, layer_info)
hidden_act = self._config['hidden_act']
hidden_dropout_prob = self._config['hidden_dropout_prob']
num_attention_heads = self._config['num_attention_heads']
hidden_size = self._config['hidden_size']
lnorm_eps = self._config['layer_norm_eps']
intermediate_size = self._config['intermediate_size']
# define layer info
new_layer_info = collections.OrderedDict()
# multi-head attention (BertSelfAttention and part of BertSelfOutput)
# NOTE: dropout performed on the Q*K^T matrix AND the output of the final dense layer
lname = layer_base + '_mha'
new_layer_info['mh_attention'] = dict(
name=lname,
type=BertCommon['layer_types']['mhattention'],
dim=[self._config['hidden_size'], self._config['hidden_size']],
ldef=MultiHeadAttention(name=lname,
n=hidden_size,
n_attn_heads=num_attention_heads,
act='auto',
dropout=hidden_dropout_prob)(input_layers))
# first residual (BertSelfOutput)
lname = layer_base + '_sum1'
new_layer_info['sum1'] = dict(
name=lname,
type=BertCommon['layer_types']['noparms'],
dim=None,
ldef=Res(name=lname,
act='identity')(input_layers +
[new_layer_info['mh_attention']['ldef']]))
# first layer normalization (BertSelfOutput)
lname = layer_base + '_ln1'
new_layer_info['layer_norm1'] = dict(
name=lname,
type=BertCommon['layer_types']['layernorm'],
dim=[self._config['hidden_size']],
ldef=LayerNormalization(name=lname,
act='identity',
token_size=self._config['hidden_size'],
epsilon=lnorm_eps)(
new_layer_info['sum1']['ldef']))
# intermediate fully-connected layer (BertIntermediate)
lname = layer_base + '_intm_dense'
new_layer_info['intermediate_fc'] = dict(
name=lname,
type=BertCommon['layer_types']['dense'],
dim=[intermediate_size, self._config['hidden_size']],
ldef=Dense(name=lname, act=hidden_act, n=intermediate_size)(
new_layer_info['layer_norm1']['ldef']))
# final fully-connected layer (BertOutput)
lname = layer_base + '_final_dense'
new_layer_info['final_fc'] = dict(
name=lname,
type=BertCommon['layer_types']['dense'],
dim=[self._config['hidden_size'], intermediate_size],
ldef=Dense(name=lname,
act='identity',
n=hidden_size,
dropout=hidden_dropout_prob)(
new_layer_info['intermediate_fc']['ldef']))
# second residual (BertOutput)
lname = layer_base + '_sum2'
new_layer_info['sum2'] = dict(
name=lname,
type=BertCommon['layer_types']['noparms'],
dim=None,
ldef=Res(name=lname, act='identity')([
new_layer_info['layer_norm1']['ldef'],
new_layer_info['final_fc']['ldef']
]))
# second layer normalization (BertOutput)
lname = layer_base + '_ln2'
new_layer_info['layer_norm2'] = dict(
name=lname,
type=BertCommon['layer_types']['layernorm'],
dim=[self._config['hidden_size']],
ldef=LayerNormalization(name=lname,
act='identity',
token_size=self._config['hidden_size'],
epsilon=lnorm_eps)(
new_layer_info['sum2']['ldef']))
return new_layer_info
