def onnx_extract_residual(graph, node, layers): ''' Construct residual layer from ONNX op Parameters ---------- graph : ONNX GraphProto Specifies a GraphProto object. node : ONNX NodeProto Specifies a NodeProto object. layers : list of Layers Specifies the existing layers of a model. Returns ------- :class:`Res` ''' previous = onnx_find_previous_compute_layer(graph, node) if not previous: src_names = [find_input_layer_name(graph)] else: src_names = [p.name for p in previous] src = [get_dlpy_layer(layers, i) for i in src_names] return Res(name=node.name, act='identity', src_layers=src)
def test_model22(self): model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7)) pool1 = Pooling(2) model1.add(pool1) conv1 = Conv2d(1, 1, act='identity', src_layers=[pool1]) model1.add(conv1) model1.add(Res(act='relu', src_layers=[conv1, pool1])) model1.add(Pooling(2)) model1.add(Dense(2)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={'name': 'eee', 'replace': True}, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1) self.assertTrue(r.severity == 0) model1.deploy(self.data_dir, output_format='onnx')
def test_model3(self): try: import onnx except: unittest.TestCase.skipTest(self, "onnx not found in the libraries") model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7)) pool1 = Pooling(2) model1.add(pool1) conv1 = Conv2d(1, 1, act='identity', src_layers=[pool1]) model1.add(conv1) model1.add(Res(act='relu', src_layers=[conv1, pool1])) model1.add(Pooling(2)) model1.add(Dense(2)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) # # unittest.TestCase.skipTest(self, "Skipping temporarily due to server error.") r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1) self.assertTrue(r.severity == 0) import tempfile tmp_dir_to_dump = tempfile.gettempdir() model1.deploy(tmp_dir_to_dump, output_format='onnx') import os os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx")) if (caslib is not None) and tmp_caslib: self.s.retrieve('table.dropcaslib', message_level='error', caslib=caslib)
def downsampling_bottleneck(x, in_depth, out_depth, projection_ratio=4): ''' Defines the down-sampling bottleneck of ENet Parameters ---------- x : class:`Layer' Previous layer to this block in_depth : int Depth of the layer fed into this block out_depth : int Depth of the output layer of this block projection_ratio : int, optional Used to calculate the reduced_depth for intermediate convolution layers Default: 4 Returns ------- :class:`Res` ''' reduced_depth = int(in_depth // projection_ratio) conv1 = Conv2d(reduced_depth, 3, stride=2, padding=1, act='identity', include_bias=False)(x) bn1 = BN(act='relu')(conv1) conv2 = Conv2d(reduced_depth, 3, stride=1, act='identity', include_bias=False)(bn1) bn2 = BN(act='relu')(conv2) conv3 = Conv2d(out_depth, 1, stride=1, act='identity', include_bias=False)(bn2) bn3 = BN(act='relu')(conv3) pool1 = Pooling(2, stride=2)(x) conv4 = Conv2d(out_depth, 1, stride=1, act='identity', include_bias=False)(pool1) bn4 = BN(act='relu')(conv4) res = Res()([bn3, bn4]) return res
def _add_bert_embedding_layer(self, layer_name, layer_info): # find input layer(s) input_layers = self._find_layer_def([ self.ds_layers['token_input'], self.ds_layers['position_input'], self.ds_layers['segment_input'] ], layer_info) # need tensors for input layers for ii, ilayer in enumerate(input_layers): input_layers[ii] = input_layers[ii].input_tenor # define layer information new_layer_info = collections.OrderedDict() # residual layer to sum embeddings lname = layer_name + '_sum' new_layer_info['sum_layer'] = dict( name=lname, type=BertCommon['layer_types']['noparms'], dim=None, ldef=Res(name=lname, act='identity')(input_layers)) # layer normalization layer lname = layer_name + '_ln' new_layer_info['layer_norm_layer'] = dict( name=lname, type=BertCommon['layer_types']['layernorm'], dim=[self._config['hidden_size']], ldef=LayerNormalization( name=lname, act='identity', epsilon=self._config['layer_norm_eps'], token_size=self._config['hidden_size'], dropout=self._config['hidden_dropout_prob'])( new_layer_info['sum_layer']['ldef'])) return new_layer_info
def _inverted_res_block(inputs, in_channels, expansion, stride, alpha, filters, block_id): """ Inverted Residual Block Parameters ---------- inputs: Input tensor in_channels: Specifies the number of input tensor's channel expansion: expansion factor always applied to the input size. stride: the strides of the convolution alpha: width multiplier. filters: the dimensionality of the output space. block_id: block id used for naming layers """ pointwise_conv_filters = int(filters * alpha) pointwise_filters = _make_divisible(pointwise_conv_filters, 8) x = inputs prefix = 'block_{}_'.format(block_id) n_groups = in_channels if block_id: # Expand n_groups = expansion * in_channels x = Conv2d(expansion * in_channels, 1, include_bias=False, act='identity', name=prefix + 'expand')(x) x = BN(name=prefix + 'expand_BN', act='identity')(x) else: prefix = 'expanded_conv_' # Depthwise x = GroupConv2d(n_groups, n_groups, 3, stride=stride, act='identity', include_bias=False, name=prefix + 'depthwise')(x) x = BN(name=prefix + 'depthwise_BN', act='relu')(x) # Project x = Conv2d(pointwise_filters, 1, include_bias=False, act='identity', name=prefix + 'project')(x) x = BN(name=prefix + 'project_BN', act='identity')(x) # identity activation on narrow tensor if in_channels == pointwise_filters and stride == 1: return Res(name=prefix + 'add')([inputs, x]), pointwise_filters return x, pointwise_filters
def test_res_layer2(self): if not __dev__: with self.assertRaises(DLPyError): Res(not_a_parameter=1)
def test_res_layer1(self): if not __dev__: dict1 = Res(name='res', src_layers=[Conv2d(name='conv', n_filters=32)]).to_model_params() self.assertTrue(self.sample_syntax['res1'] == dict1)
def _MBConvBlock(inputs, in_channels, out_channels, ksize, stride, expansion, se_ratio, stage_id, block_id, noskip=False, activation_fn='relu'): ''' Inverted Residual Block Parameters ---------- inputs: input tensor Speecify input tensor for block. in_channels: integer Specifies the number of input tensor's channel. out_channels: integer Specifies the number of output tensor's channel ksize: Specifies the kernel size of the convolution stride: integer Specifies the stride of the convolution expansion: double Specifies the expansion factor for the input layer. se_ratio: double Specifies the ratio to squeeze the input filters for squeeze-and-excitation block. stage_id: integer Specifies stage id for naming layers block_id: Specifies block id for naming layers noskip: bool Specifies whether the skip connection is used. By default, the skip connection is used. activation_fn: Specifies activation function ''' # mobilenetv2 block is also known as inverted residual block, which consists of three convolutions: # the first is 1*1 convolution for expansion # the second is depthwise convolution # the third is 1*1 convolution without any non-linearity for projection x = inputs prefix = 'stage_{}_block_{}'.format(stage_id, block_id) n_groups = in_channels # for expansion=1, n_groups might be different from pointwise_filters if expansion > 1: # For MobileNet V2, expansion>1 when stage>0 n_groups = int(expansion * in_channels) ## update n_groups x = Conv2d(n_groups, 1, include_bias=False, act='identity', name=prefix + 'expand')(x) x = BN(name=prefix + 'expand_BN', act='identity')(x) # Depthwise convolution x = GroupConv2d(n_groups, n_groups, ksize, stride=stride, act='identity', include_bias=False, name=prefix + 'depthwise')(x) x = BN(name=prefix + 'depthwise_BN', act=activation_fn)(x) # Squeeze-Excitation if 0 < se_ratio <= 1: se_input = x # features to be squeezed x = GlobalAveragePooling2D(name=prefix + "global_avg_pool")(x) # Squeeze channels_se = max(1, int(in_channels * se_ratio)) x = Conv2d(channels_se, 1, include_bias=True, act=activation_fn, name=prefix + 'squeeze')(x) x = Conv2d(n_groups, 1, include_bias=True, act='sigmoid', name=prefix + 'excitation')(x) x = Reshape(name=prefix + 'reshape', width=n_groups, height=1, depth=1)(x) x = Scale(name=prefix + 'scale')([se_input, x]) # x = out*w # Project x = Conv2d(out_channels, 1, include_bias=False, act='identity', name=prefix + 'project')(x) x = BN(name=prefix + 'project_BN', act='identity')(x) # identity activation on narrow tensor # Prepare output for MBConv block if in_channels == out_channels and stride == 1 and (not noskip): # dropout can be added. return Res(name=prefix + 'add_se_residual')([x, inputs]) else: return x
def _shuffle_unit(inputs, in_channels, out_channels, groups, bottleneck_ratio, strides=2, stage=1, block=1): """ create a shuffle unit Parameters ---------- inputs: Input tensor of with `channels_last` data format in_channels: number of input channels out_channels: number of output channels strides: An integer or tuple/list of 2 integers, groups: number of groups per channel bottleneck_ratio: float bottleneck ratio implies the ratio of bottleneck channels to output channels. stage: stage number block: block number """ prefix = 'stage%d/block%d' % (stage, block) # if strides >= 2: # out_channels -= in_channels # default: 1/4 of the output channel of a ShuffleNet Unit bottleneck_channels = int(out_channels * bottleneck_ratio) groups = (1 if stage == 2 and block == 1 else groups) # x = _group_conv(inputs, in_channels, out_channels = bottleneck_channels, # groups = (1 if stage == 2 and block == 1 else groups), # name = '%s/1x1_gconv_1' % prefix) x = GroupConv2d(bottleneck_channels, n_groups=(1 if stage == 2 and block == 1 else groups), act='identity', width=1, height=1, stride=1, include_bias=False, name='%s/1x1_gconv_1' % prefix)(inputs) x = BN(act='relu', name='%s/bn_gconv_1' % prefix)(x) x = ChannelShuffle(n_groups=groups, name='%s/channel_shuffle' % prefix)(x) # depthwise convolutioin x = GroupConv2d(x.shape[-1], n_groups=x.shape[-1], width=3, height=3, include_bias=False, stride=strides, act='identity', name='%s/1x1_dwconv_1' % prefix)(x) x = BN(act=block_act, name='%s/bn_dwconv_1' % prefix)(x) out_channels = out_channels if strides == 1 else out_channels - in_channels x = GroupConv2d(out_channels, n_groups=groups, width=1, height=1, stride=1, act='identity', include_bias=False, name='%s/1x1_gconv_2' % prefix)(x) x = BN(act=block_act, name='%s/bn_gconv_2' % prefix)(x) if strides < 2: ret = Res(act='relu', name='%s/add' % prefix)([x, inputs]) else: avg = Pooling(width=3, height=3, stride=2, pool='mean', name='%s/avg_pool' % prefix)(inputs) ret = Concat(act='relu', name='%s/concat' % prefix)([x, avg]) return ret
def _add_bert_encoding_layer(self, layer_base, src_layers, layer_info): # find input layer(s) input_layers = self._find_layer_def(src_layers, layer_info) hidden_act = self._config['hidden_act'] hidden_dropout_prob = self._config['hidden_dropout_prob'] num_attention_heads = self._config['num_attention_heads'] hidden_size = self._config['hidden_size'] lnorm_eps = self._config['layer_norm_eps'] intermediate_size = self._config['intermediate_size'] # define layer info new_layer_info = collections.OrderedDict() # multi-head attention (BertSelfAttention and part of BertSelfOutput) # NOTE: dropout performed on the Q*K^T matrix AND the output of the final dense layer lname = layer_base + '_mha' new_layer_info['mh_attention'] = dict( name=lname, type=BertCommon['layer_types']['mhattention'], dim=[self._config['hidden_size'], self._config['hidden_size']], ldef=MultiHeadAttention(name=lname, n=hidden_size, n_attn_heads=num_attention_heads, act='auto', dropout=hidden_dropout_prob)(input_layers)) # first residual (BertSelfOutput) lname = layer_base + '_sum1' new_layer_info['sum1'] = dict( name=lname, type=BertCommon['layer_types']['noparms'], dim=None, ldef=Res(name=lname, act='identity')(input_layers + [new_layer_info['mh_attention']['ldef']])) # first layer normalization (BertSelfOutput) lname = layer_base + '_ln1' new_layer_info['layer_norm1'] = dict( name=lname, type=BertCommon['layer_types']['layernorm'], dim=[self._config['hidden_size']], ldef=LayerNormalization(name=lname, act='identity', token_size=self._config['hidden_size'], epsilon=lnorm_eps)( new_layer_info['sum1']['ldef'])) # intermediate fully-connected layer (BertIntermediate) lname = layer_base + '_intm_dense' new_layer_info['intermediate_fc'] = dict( name=lname, type=BertCommon['layer_types']['dense'], dim=[intermediate_size, self._config['hidden_size']], ldef=Dense(name=lname, act=hidden_act, n=intermediate_size)( new_layer_info['layer_norm1']['ldef'])) # final fully-connected layer (BertOutput) lname = layer_base + '_final_dense' new_layer_info['final_fc'] = dict( name=lname, type=BertCommon['layer_types']['dense'], dim=[self._config['hidden_size'], intermediate_size], ldef=Dense(name=lname, act='identity', n=hidden_size, dropout=hidden_dropout_prob)( new_layer_info['intermediate_fc']['ldef'])) # second residual (BertOutput) lname = layer_base + '_sum2' new_layer_info['sum2'] = dict( name=lname, type=BertCommon['layer_types']['noparms'], dim=None, ldef=Res(name=lname, act='identity')([ new_layer_info['layer_norm1']['ldef'], new_layer_info['final_fc']['ldef'] ])) # second layer normalization (BertOutput) lname = layer_base + '_ln2' new_layer_info['layer_norm2'] = dict( name=lname, type=BertCommon['layer_types']['layernorm'], dim=[self._config['hidden_size']], ldef=LayerNormalization(name=lname, act='identity', token_size=self._config['hidden_size'], epsilon=lnorm_eps)( new_layer_info['sum2']['ldef'])) return new_layer_info