def __init__(self, vocab_size, embedding_size, init_scale=0.1): # vocab_size定义了这个skipgram这个模型的词表大小 # embedding_size定义了词向量的维度是多少 # init_scale定义了词向量初始化的范围,一般来说,比较小的初始化范围有助于模型训练 super(SkipGram, self).__init__() self.vocab_size = vocab_size self.embedding_size = embedding_size # 使用paddle.fluid.dygraph提供的Embedding函数,构造一个词向量参数 # 这个参数的大小为:[self.vocab_size, self.embedding_size] # 数据类型为:float32 # 这个参数的名称为:embedding_para # 这个参数的初始化方式为在[-init_scale, init_scale]区间进行均匀采样 self.embedding = Embedding( size=[self.vocab_size, self.embedding_size], dtype='float32', param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-0.5 / embedding_size, high=0.5 / embedding_size))) # 使用paddle.fluid.dygraph提供的Embedding函数,构造另外一个词向量参数 # 这个参数的大小为:[self.vocab_size, self.embedding_size] # 数据类型为:float32 # 这个参数的名称为:embedding_para_out # 这个参数的初始化方式为在[-init_scale, init_scale]区间进行均匀采样 # 跟上面不同的是,这个参数的名称跟上面不同,因此, # embedding_para_out和embedding_para虽然有相同的shape,但是权重不共享 self.embedding_out = Embedding( size=[self.vocab_size, self.embedding_size], dtype='float32', param_attr=fluid.ParamAttr( name='embedding_out_para', initializer=fluid.initializer.UniformInitializer( low=-0.5 / embedding_size, high=0.5 / embedding_size)))
def _init_embedding(self): # the embedding of all words '''self.word_emb = Embedding(size=(self.vocab_size, self.embedding_size), dtype='float32', param_attr=fluid.ParamAttr(name='word_emb_para', initializer=fluid.initializer.UniformInitializer( low=-self.init_scale/self.embedding_size, high=self.init_scale/self.embedding_size)))''' # the embedding of all subwords # padding embedding, this embedding is used to match the embedding of padding label, without updating gradient self.subword_emb = Embedding( size=(self.subvocab_size + 1, self.embedding_size), dtype='float32', padding_idx=self.subvocab_size, param_attr=fluid.ParamAttr( name='subword_emb_para', initializer=fluid.initializer.UniformInitializer( low=-self.init_scale / self.embedding_size, high=self.init_scale / self.embedding_size), trainable=True)) # the embedding of all the words, different from word_emb, this is a hidden parameter that will not be the final # embedding self.word_emb_hidden = Embedding( size=(self.vocab_size, self.embedding_size), dtype='float32', param_attr=fluid.ParamAttr( name='hidden_emb_para', initializer=fluid.initializer.UniformInitializer( low=-self.init_scale / self.embedding_size, high=self.init_scale / self.embedding_size), trainable=True))
def __init__(self): super(DNNPlus, self).__init__() self.name = 'DNNPlus_' + str(cfg.embedding_size) + '_' + str( cfg.dnn_hidden_dims[0]) self.init_value_ = 0.1 self.embedding_w = Embedding( size=[cfg.num_feat + 1, 1], dtype='float32', padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_), regularizer=fluid.regularizer.L1DecayRegularizer(cfg.reg))) self.embedding = Embedding( size=[cfg.num_feat + 1, cfg.embedding_size], dtype='float32', padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_ / math.sqrt(float(cfg.embedding_size))))) self.first_order_act = fluid.layers.sigmoid sizes = [cfg.num_field * cfg.embedding_size ] + cfg.deepfm_layer_sizes + [1] acts = ['relu' for _ in range(len(cfg.deepfm_layer_sizes))] + [None] w_scales = [ self.init_value_ / math.sqrt(float(10)) for _ in range(len(cfg.deepfm_layer_sizes)) ] + [self.init_value_] self.second_order_fc = Linear( cfg.deepfm_layer_sizes[0], 1, act='sigmoid', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=w_scales[0])), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_))) self.add_sublayer('secong_order', self.second_order_fc) self.linears = [] for i in range(len(cfg.deepfm_layer_sizes) + 1): linear = Linear( sizes[i], sizes[i + 1], act=acts[i], param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=w_scales[i])), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_))) self.add_sublayer('linear_%d' % i, linear) self.linears.append(linear)
def __init__(self, name_scope, dict_dim, batch_size, seq_len): super(CNN, self).__init__(name_scope) self.dict_dim = dict_dim self.emb_dim = 128 self.hid_dim = 128 self.fc_hid_dim = 96 self.class_dim = 2 self.win_size = [3, self.hid_dim] self.batch_size = batch_size self.seq_len = seq_len self.embedding = Embedding( self.full_name(), size=[self.dict_dim + 1, self.emb_dim], dtype='float32', is_sparse=False) self._simple_conv_pool_1 = SimpleConvPool( self.full_name(), self.hid_dim, self.win_size, batch_size=self.batch_size) self._fc1 = FC(self.full_name(), size=self.fc_hid_dim, act="softmax") self._fc_prediction = FC(self.full_name(), size=self.class_dim, act="softmax")
def __init__(self, hidden_size, vocab_size, num_steps=20, init_scale=0.1, is_sparse=False, dtype='float32'): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size self.init_scale = init_scale self.num_steps = num_steps self.embedding = Embedding( size=[vocab_size, hidden_size], dtype=dtype, is_sparse=is_sparse, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-init_scale, high=init_scale))) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale))
def __init__(self, dict_dim, emb_dim=128, hid_dim=128, fc_hid_dim=96, class_dim=2, channels=1, win_size=(3, 128)): super(CNN, self).__init__() self.dict_dim = dict_dim self.emb_dim = emb_dim self.hid_dim = hid_dim self.fc_hid_dim = fc_hid_dim self.class_dim = class_dim self.channels = channels self.win_size = win_size self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], dtype='float64', is_sparse=False, padding_idx=0) self._conv2d = Conv2D(num_channels=self.channels, num_filters=self.hid_dim, filter_size=win_size, padding=[1, 0], use_cudnn=True, act=None, dtype="float64") self._fc_1 = Linear(input_dim=self.hid_dim, output_dim=self.fc_hid_dim, dtype="float64") self._fc_2 = Linear(input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax", dtype="float64")
def __init__(self, dict_dim, batch_size, seq_len): super(BiGRU, self).__init__() self.dict_dim = dict_dim self.emb_dim = 128 self.hid_dim = 128 self.fc_hid_dim = 96 self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len self.embedding = Embedding( size=[self.dict_dim + 1, self.emb_dim], dtype='float32', param_attr=fluid.ParamAttr(learning_rate=30), is_sparse=False) h_0 = np.zeros((self.batch_size, self.hid_dim), dtype="float32") h_0 = to_variable(h_0) self._fc1 = Linear(input_dim=self.hid_dim, output_dim=self.hid_dim * 3) self._fc2 = Linear(input_dim=self.hid_dim * 2, output_dim=self.fc_hid_dim, act="tanh") self._fc_prediction = Linear(input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax") self._gru_forward = DynamicGRU(size=self.hid_dim, h_0=h_0, is_reverse=False) self._gru_backward = DynamicGRU(size=self.hid_dim, h_0=h_0, is_reverse=True)
def __init__(self, dict_dim, batch_size, seq_len): super(BiGRU, self).__init__() self.dict_dim = dict_dim self.emb_dim = 128 self.hid_dim = 128 self.fc_hid_dim = 96 self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len self.embedding = Embedding( size=[self.dict_dim + 1, self.emb_dim], dtype='float32', param_attr=fluid.ParamAttr(learning_rate=30), is_sparse=False) h_0 = np.zeros((self.batch_size, self.hid_dim), dtype="float32") h_0 = to_variable(h_0) self._fc1 = Linear(input_dim=self.hid_dim, output_dim=self.hid_dim * 3) self._fc2 = Linear(input_dim=self.hid_dim * 2, output_dim=self.fc_hid_dim, act="tanh") self._fc_prediction = Linear(input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax") self._encoder = BiGRUEncoder(grnn_hidden_dim=self.hid_dim, input_dim=self.hid_dim * 3, h_0=h_0, init_bound=0.1, is_bidirection=True)
def __init__(self): super(DNN, self).__init__() self.name = 'DNN_1024_512_256_dygraph' self.embeddings = [ Embedding( size=[cfg.sparse_feature_dim, cfg.embedding_size], dtype='float32', padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(cfg.embedding_size))))) for _ in range(26) ] feature_size = 13 + 26 * cfg.embedding_size self.block_1 = Linear( feature_size, 1024, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1))) self.block_2_1 = Linear( 1024, 1024, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1))) self.block_2_2 = Linear( 1024, 512, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1))) self.block_2_3 = Linear( 512, 256, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1))) self.block_3 = Linear( 256, 2, act='softmax', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1)))
def __init__(self, dict_dim, batch_size, seq_len): super(CNN, self).__init__() self.dict_dim = dict_dim self.emb_dim = 128 self.hid_dim = 128 self.fc_hid_dim = 96 self.class_dim = 2 self.channels = 1 self.win_size = [3, self.hid_dim] self.batch_size = batch_size self.seq_len = seq_len self.embedding = Embedding( size=[self.dict_dim + 1, self.emb_dim], dtype='float32', is_sparse=False) self._simple_conv_pool_1 = SimpleConvPool( self.channels, self.hid_dim, self.win_size, batch_size=self.batch_size) self._fc1 = Linear( input_dim=self.hid_dim * self.seq_len, output_dim=self.fc_hid_dim, act="softmax") self._fc_prediction = Linear( input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax")
def __init__(self, name_scope, hidden_size, vocab_size, num_layers=2, num_steps=20, init_scale=0.1, dropout=None): super(PtbModel, self).__init__(name_scope) self.hidden_size = hidden_size self.vocab_size = vocab_size self.init_scale = init_scale self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout self.simple_lstm_rnn = SimpleLSTMRNN(self.full_name(), hidden_size, num_steps, num_layers=num_layers, init_scale=init_scale, dropout=dropout) self.embedding = Embedding( self.full_name(), size=[vocab_size, hidden_size], dtype='float32', is_sparse=False, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-init_scale, high=init_scale))) self.out_project = FC(self.full_name(), self.vocab_size, num_flatten_dims=2)
def __init__(self): super(FM, self).__init__() self.init_value_ = 0.1 self.embedding_w = Embedding( size=[cfg.num_feat + 1, 1], dtype='float32', padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_), regularizer=fluid.regularizer.L1DecayRegularizer(cfg.reg))) self.embedding = Embedding( size=[cfg.num_feat + 1, cfg.embedding_size], dtype='float32', padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_ / math.sqrt(float(cfg.embedding_size)))))
def __init__(self, name_scope, embedding_dim, units, vocab_size): super(RNNDecoder, self).__init__(name_scope) self.units = units self.embedding = Embedding(self.full_name(), (vocab_size, embedding_dim)) self.gru = DynamicGRU(self.full_name(), units) self.fc1 = FC(self.full_name(), size=units, num_flatten_dims=2) self.fc2 = FC(self.full_name(), vocab_size) self.attention = BahdanauAttention(self.full_name(), units)
def __init__(self): super(OCRAttention, self).__init__() self.encoder_net = EncoderNet() self.fc = Linear(Config.encoder_size, Config.decoder_size, bias_attr=False, act='relu') self.embedding = Embedding( [Config.num_classes + 2, Config.word_vector_dim], dtype='float32') self.gru_decoder_with_attention = GRUDecoderWithAttention( Config.decoder_size, Config.num_classes)
def __init__(self, scope_name): super(OCRAttention, self).__init__(scope_name) self.encoder_net = EncoderNet(self.full_name()) self.fc = FC(self.full_name(), size=Config.decoder_size, bias_attr=False, act='relu') self.embedding = Embedding( self.full_name(), [Config.num_classes + 2, Config.word_vector_dim], dtype='float32') self.gru_decoder_with_attention = GRUDecoderWithAttention( self.full_name(), Config.decoder_size, Config.num_classes)
def __init__(self, hidden_size, vocab_size, is_sparse=False): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size self.embedding = Embedding(size=[self.vocab_size, self.hidden_size], dtype='float32', is_sparse=is_sparse) self.lin_a = paddle.nn.Linear(self.hidden_size, self.vocab_size) self.lin_b = paddle.nn.Linear(self.vocab_size, 1) self.unused_net = paddle.nn.Linear(5, 3) self.phony = self.create_parameter(shape=[1], dtype="float32")
def __init__(self, batch_size, num_classes, encoder_size, decoder_size, word_vector_dim): super(OCRAttention, self).__init__() self.encoder_net = EncoderNet(batch_size, decoder_size) self.fc = Linear(input_dim=encoder_size, output_dim=decoder_size, bias_attr=False, act='relu') self.embedding = Embedding([num_classes + 2, word_vector_dim], dtype='float32') self.gru_decoder_with_attention = GRUDecoderWithAttention( encoder_size, decoder_size, num_classes) self.batch_size = batch_size
def __init__(self, name_scope, vocab_size, embedding_size, init_scale=0.1): super(SkipGram, self).__init__(name_scope) self.vocab_size = vocab_size self.embedding_size = embedding_size self.embedding = Embedding( size=[self.vocab_size, self.embedding_size], dtype='float32', param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-0.5 / self.embedding_size, high=0.5 / self.embedding_size))) self.embedding_out = Embedding( size=[self.vocab_size, self.embedding_size], dtype='float32', param_attr=fluid.ParamAttr( name='embedding_out_para', initializer=fluid.initializer.UniformInitializer( low=-0.5 / self.embedding_size, high=0.5 / self.embedding_size)))
def __init__(self): super(DRNN, self).__init__() self.name = 'DRNN_' + str(cfg.embedding_size) + '_' + str( cfg.drnn_hidden_dim) + '_' + str(cfg.drnn_hidden_layer) self.init_value_ = 0.1 self.embedding = Embedding( size=[cfg.num_feat + 1, cfg.embedding_size], dtype='float32', padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_ / math.sqrt(float(cfg.embedding_size))))) self.hidden_1 = Linear( cfg.num_field * cfg.embedding_size, cfg.drnn_hidden_dim, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_ / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_))) self.add_sublayer('hidden_1', self.hidden_1) self.hidden_2 = Linear( cfg.drnn_hidden_dim, cfg.drnn_hidden_dim, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_ / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_))) self.add_sublayer('hidden_2', self.hidden_2) self.hidden_3 = Linear( cfg.drnn_hidden_dim, 1, act='sigmoid', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_ / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=self.init_value_))) self.add_sublayer('hidden_3', self.hidden_3)
def __init__(self): super(DRNN, self).__init__() self.name = 'DRNN_' + str(cfg.drnn_hidden_dim) + '_' + str( cfg.drnn_hidden_layer) self.embeddings = [ Embedding( size=[cfg.sparse_feature_dim, cfg.embedding_size], dtype='float32', padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(cfg.embedding_size))))) for _ in range(26) ] feature_size = 13 + 26 * cfg.embedding_size self.block_1 = Linear( feature_size, cfg.drnn_hidden_dim, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1))) self.block_2 = Linear( cfg.drnn_hidden_dim, cfg.drnn_hidden_dim, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1))) self.block_3 = Linear( cfg.drnn_hidden_dim, 2, act='softmax', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1 / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=0.1)))
def __init__(self, hidden_size, vocab_size, class_num=2, num_layers=1, num_steps=128, init_scale=0.1, dropout=None): super(SentimentClassifierwithNormalDis, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size self.class_num = class_num self.init_scale = init_scale self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout self.simple_lstm_rnn = SimpleLSTMRNNwithNormalDis( hidden_size, num_steps, num_layers=num_layers, init_scale=init_scale, dropout=dropout) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', is_sparse=False, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-init_scale, high=init_scale))) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.class_num], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale)) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.class_num], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale))
def __init__(self, dict_dim, batch_size, seq_len): super(BOW, self).__init__() self.dict_dim = dict_dim self.emb_dim = 128 self.hid_dim = 128 self.fc_hid_dim = 96 self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len self.embedding = Embedding( size=[self.dict_dim + 1, self.emb_dim], dtype='float32', is_sparse=False) self._fc1 = Linear(input_dim = self.hid_dim, output_dim=self.hid_dim, act="tanh") self._fc2 = Linear(input_dim = self.hid_dim, output_dim=self.fc_hid_dim, act="tanh") self._fc_prediction = Linear(input_dim = self.fc_hid_dim, output_dim = self.class_dim, act="softmax")
def __init__(self, name_scope, hidden_size, vocab_size, num_layers=2, num_steps=20, init_scale=0.1, dropout=None): #super(PtbModel, self).__init__(name_scope) super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size self.init_scale = init_scale self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout self.simple_gru_rnn = SimpleGRURNN( #self.full_name(), hidden_size, num_steps, num_layers=num_layers, init_scale=init_scale, dropout=dropout) self.embedding = Embedding( #self.full_name(), size=[vocab_size, hidden_size], dtype='float32', is_sparse=False, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-init_scale, high=init_scale))) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale)) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale))
def __init__(self, args, vocab_size, num_labels, length=None): super(lex_net, self).__init__() """ define the lexical analysis network structure word: stores the input of the model for_infer: a boolean value, indicating if the model to be created is for training or predicting. return: for infer: return the prediction otherwise: return the prediction """ self.word_emb_dim = args.word_emb_dim self.vocab_size = vocab_size self.num_labels = num_labels self.grnn_hidden_dim = args.grnn_hidden_dim self.emb_lr = args.emb_learning_rate if 'emb_learning_rate' in dir( args) else 1.0 self.crf_lr = args.emb_learning_rate if 'crf_learning_rate' in dir( args) else 1.0 self.bigru_num = args.bigru_num self.init_bound = 0.1 #self.IS_SPARSE = True self.word_embedding = Embedding( size=[self.vocab_size, self.word_emb_dim], dtype='float32', #is_sparse=self.IS_SPARSE, param_attr=fluid.ParamAttr(learning_rate=self.emb_lr, name="word_emb", initializer=fluid.initializer.Uniform( low=-self.init_bound, high=self.init_bound))) h_0 = np.zeros((args.batch_size, self.grnn_hidden_dim), dtype="float32") h_0 = to_variable(h_0) self.bigru_units = [] for i in range(self.bigru_num): if i == 0: self.bigru_units.append( self.add_sublayer( "bigru_units%d" % i, BiGRU(self.grnn_hidden_dim, self.grnn_hidden_dim, self.init_bound, h_0=h_0))) else: self.bigru_units.append( self.add_sublayer( "bigru_units%d" % i, BiGRU(self.grnn_hidden_dim * 2, self.grnn_hidden_dim, self.init_bound, h_0=h_0))) self.fc = Linear(input_dim=self.grnn_hidden_dim * 2, output_dim=self.num_labels, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Uniform( low=-self.init_bound, high=self.init_bound), regularizer=fluid.regularizer.L2DecayRegularizer( regularization_coeff=1e-4))) #, #num_flatten_dims=2) self.linear_chain_crf = Linear_chain_crf(param_attr=fluid.ParamAttr( name='linear_chain_crfw', learning_rate=self.crf_lr), size=self.num_labels) self.crf_decoding = Crf_decoding(param_attr=fluid.ParamAttr( name='crfw', learning_rate=self.crf_lr), size=self.num_labels)
def __init__(self, hidden_size, src_vocab_size, tar_vocab_size, batch_size, num_layers=1, init_scale=0.1, dropout=None, beam_size=1, beam_start_token=1, beam_end_token=2, beam_max_step_num=100, mode='train'): super(AttentionModel, self).__init__() self.hidden_size = hidden_size self.src_vocab_size = src_vocab_size self.tar_vocab_size = tar_vocab_size self.batch_size = batch_size self.num_layers = num_layers self.init_scale = init_scale self.dropout = dropout self.beam_size = beam_size self.beam_start_token = beam_start_token self.beam_end_token = beam_end_token self.beam_max_step_num = beam_max_step_num self.mode = mode self.kinf = 1e9 param_attr = ParamAttr(initializer=uniform_initializer(self.init_scale)) bias_attr = ParamAttr(initializer=zero_constant) forget_bias = 1.0 self.src_embeder = Embedding( size=[self.src_vocab_size, self.hidden_size], param_attr=fluid.ParamAttr( name='source_embedding', initializer=uniform_initializer(init_scale))) self.tar_embeder = Embedding( size=[self.tar_vocab_size, self.hidden_size], is_sparse=False, param_attr=fluid.ParamAttr( name='target_embedding', initializer=uniform_initializer(init_scale))) self.enc_units = [] for i in range(num_layers): self.enc_units.append( self.add_sublayer( "enc_units_%d" % i, BasicLSTMUnit( hidden_size=self.hidden_size, input_size=self.hidden_size, param_attr=param_attr, bias_attr=bias_attr, forget_bias=forget_bias))) self.dec_units = [] for i in range(num_layers): if i == 0: self.dec_units.append( self.add_sublayer( "dec_units_%d" % i, BasicLSTMUnit( hidden_size=self.hidden_size, input_size=self.hidden_size * 2, param_attr=param_attr, bias_attr=bias_attr, forget_bias=forget_bias))) else: self.dec_units.append( self.add_sublayer( "dec_units_%d" % i, BasicLSTMUnit( hidden_size=self.hidden_size, input_size=self.hidden_size, param_attr=param_attr, bias_attr=bias_attr, forget_bias=forget_bias))) self.fc = fluid.dygraph.nn.Linear( self.hidden_size, self.tar_vocab_size, param_attr=param_attr, bias_attr=False) self.attn_fc = fluid.dygraph.nn.Linear( self.hidden_size, self.hidden_size, param_attr=param_attr, bias_attr=False) self.concat_fc = fluid.dygraph.nn.Linear( 2 * self.hidden_size, self.hidden_size, param_attr=param_attr, bias_attr=False)
def __init__(self, hidden_size, vocab_size, class_num=2, num_layers=1, num_steps=128, init_scale=0.1, dropout=None): # 这个模型的参数分别为: # 1. hidden_size,表示embedding-size,hidden和cell向量的维度 # 2. vocab_size,模型可以考虑的词表大小 # 3. class_num,情感类型个数,可以是2分类,也可以是多分类 # 4. num_steps,表示这个情感分析模型最大可以考虑的句子长度 # 5. init_scale,表示网络内部的参数的初始化范围, # 长短时记忆网络内部用了很多tanh,sigmoid等激活函数,这些函数对数值精度非常敏感, # 因此我们一般只使用比较小的初始化范围,以保证效果 super(SentimentClassifier, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size self.class_num = class_num self.init_scale = init_scale self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout # 声明一个LSTM模型,用来把一个句子抽象城一个向量 self.simple_lstm_rnn = SimpleLSTMRNN( hidden_size, num_steps, num_layers=num_layers, init_scale=init_scale, dropout=dropout) # 声明一个embedding层,用来把句子中的每个词转换为向量 self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', is_sparse=False, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-init_scale, high=init_scale))) # 在得到一个句子的向量表示后,我们需要根据这个向量表示对这个句子进行分类 # 一般来说,我们可以把这个句子的向量表示, # 乘以一个大小为[self.hidden_size, self.class_num]的W参数 # 并加上一个大小为[self.class_num]的b参数 # 通过这种手段达到把句子向量映射到分类结果的目标 # 我们需要声明最终在使用句子向量映射到具体情感类别过程中所需要使用的参数 # 这个参数的大小一般是[self.hidden_size, self.class_num] self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.class_num], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale)) # 同样的,我们需要声明最终分类过程中的b参数 # 这个参数的大小一般是[self.class_num] self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.class_num], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale))
def __init__(self, Data, embedding_weight, gru_steps=10, gru_num_layers=1, init_scale=0.1): super(Model_2_dnngru, self).__init__() self.init_scale = init_scale self.Data = Data USR_ID_SIZE = self.Data.user_id_size self.usr_id_emb = Embedding([USR_ID_SIZE, 32]) self.usr_fc = Linear(32, 32) USR_GENDER_SIZE = self.Data.user_gender_size + 1 self.usr_gender_emb = Embedding([USR_GENDER_SIZE, 4]) self.usr_gender_fc = Linear(4, 4) USR_AGE_LEV_SIZE = self.Data.user_age_level_size + 1 self.usr_age_emb = Embedding([USR_AGE_LEV_SIZE, 16]) self.usr_age_fc = Linear(16, 16) USR_CITY_LEV_SIZE = self.Data.user_city_level_size + 1 self.usr_city_emb = Embedding([USR_CITY_LEV_SIZE, 16]) self.usr_city_fc = Linear(16, 16) ITM_ID_SIZE = self.Data.item_id_size item_emb_weight = fluid.ParamAttr( learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( embedding_weight), trainable=False) self.itm_id_emb = Embedding([ITM_ID_SIZE, 200], param_attr=item_emb_weight) self.click_fc = Linear(1, 16) # (32+4+16+16)+(200+128+128)+16=540 # self.all_combined = Linear(540, 512, act='tanh') # GRU,前向传播时,实际预测时取最后一步的结果,构建序列时0填充在实际元素之前 self.gru_steps = gru_steps self.gru_num_layers = gru_num_layers self.gru_hidden_size = 540 self.simple_gru_rnn = SimpleGRURNN(hidden_size=self.gru_hidden_size, num_steps=self.gru_steps, num_layers=self.gru_num_layers, init_scale=0.1, dropout=None) # 最后映射到每一个维度概率的参数 self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.gru_hidden_size, ITM_ID_SIZE], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale)) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[ITM_ID_SIZE], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale)) # self.gru_fc = Linear(self.gru_hidden_size, 512, act='sigmoid') self.gru_fc_out = Linear(self.gru_hidden_size, ITM_ID_SIZE)