def __init__(self, arg):
super(denRNNEncoder, self).__init__()
logger.info('Building RNN DenseEncoder...')
if type(arg) is not dict:
arg = vars(arg)
self.input_fields = arg['denEncoder']['input_fields']
rnnunit_map = {'rnn': nn.RNN, 'lstm': nn.LSTM, 'gru': nn.GRU}
tmp_rnn_droprate = arg['denEncoder'][
'droprate'] if arg['denEncoder']['rnn_layer'] > 1 else 0
self.rnn = rnnunit_map[arg['denEncoder']['rnn_unit']](
arg['denEncoder']['rnn_input'],
arg['denEncoder']['rnn_hidden'],
arg['denEncoder']['rnn_layer'],
dropout=tmp_rnn_droprate,
bidirectional=True,
batch_first=True)
# self.dropout = nn.Dropout(p = arg['denEncoder']['droprate'])
self.dropout = VariationalDropout(p=arg['denEncoder']['droprate'])
logger.info('RNN DenseEncoder has been built successfully.')
if 'emb_project_from' in arg[
'denEncoder'] and arg['denEncoder']['emb_project_from'] > 0:
self.project = nn.Linear(arg['denEncoder']['emb_project_from'],
arg['denEncoder']['rnn_input'])
# self.activate = nn.ReLU()
init_linear(self.project)
else:
self.project = None
self.word_dropout = WordDropout(arg['denEncoder']['word_droprate'])
def __init__(self, arg):
super(spCRFDecoder, self).__init__()
logger.info('Building CRF spDecoder')
if type(arg) is not dict:
arg = vars(arg)
try:
logger.info('Loading label dictionary from: {}'.format(arg['spDecoder']['label_dict']))
with open(arg['spDecoder']['label_dict'], 'r') as fin:
self.label_dict = json.load(fin)
except FileNotFoundError as err:
logger.error('File not exist: {}'.format(arg['spDecoder']['label_dict']))
raise
self.sof = len(self.label_dict)
self.label_dict['<sof>'] = len(self.label_dict)
self.eof = len(self.label_dict)
self.label_dict['<eof>'] = len(self.label_dict)
self.reverse_label_dict = {v: k for k, v in self.label_dict.items()}
self.tagset_size = len(self.label_dict)
self.hidden2tag = nn.Linear(arg['spDecoder']['input_dim'], self.tagset_size)
self.transitions = nn.Parameter(torch.randn(self.tagset_size, self.tagset_size))
# self.transitions.detach()[self.sof, :] = -10000
# self.transitions.detach()[:, self.eof] = -10000
# self.transitions.detach()[self.eof, :] = -10000
# self.transitions.detach()[:, self.sof] = -10000
init_linear(self.hidden2tag)
self.transitions.data.zero_()
logger.info('CRF spDecoder has been built successfully.')
# self.dropout = VariationalDropout(p= arg['spDecoder']['droprate'])
self.dropout = nn.Dropout(p= arg['spDecoder']['droprate'])
def build_pipeline(self, arg):
embedding_list = list()
with open(arg['embed'], 'r') as fin:
logger.info('Loading embedding from: {}'.format(arg['embed']))
line = fin.readline()
num_emb, emb_dim = [int(tup) for tup in line.split()]
for line in fin:
if '\t' == arg['embed_seperator']:
line = line.rstrip().split('\t')[1].split()
else:
assert ' ' == arg['embed_seperator']
line = line.rstrip().split()[1:]
embedding_list.append([float(tup) for tup in line])
self.embed = nn.Embedding(num_emb, emb_dim).cpu()
self.embed.weight = nn.Parameter(torch.FloatTensor(embedding_list))
self.embed.weight.requires_grad = False
self.eof_embedding = nn.Parameter(torch.FloatTensor(1, emb_dim))
self.space_embedding = nn.Parameter(torch.FloatTensor(1, emb_dim))
bias = np.sqrt(3.0 / emb_dim)
nn.init.uniform_(self.eof_embedding.data, -bias, bias)
nn.init.uniform_(self.space_embedding.data, -bias, bias)
self.word_dropout = WordDropout(arg['word_droprate'])
if 'emb_project_to' in arg and arg['emb_project_to'] > 0:
self.project = nn.Linear(emb_dim, arg['emb_project_to'])
self.dropout = nn.Dropout(arg['droprate'])
# self.activate = nn.ReLU()
init_linear(self.project)
else:
self.project = None
logger.info(
'Embedding loading completed, {} words are imported'.format(
num_emb))
def build_pipeline(self, arg):
logger.info('Building Char-level Representation models...')
f_rnn = BasicRNN(arg['char_layer_num'], arg['char_unit'],
arg['char_emb_dim'], arg['char_hid_dim'],
arg['droprate'])
self.f_lm = LM(f_rnn, arg['c_num'], arg['c_dim_char'], arg['t_num'],
arg['t_dim_char'], arg['droprate'])
b_rnn = BasicRNN(arg['char_layer_num'], arg['char_unit'],
arg['char_emb_dim'], arg['char_hid_dim'],
arg['droprate'])
self.b_lm = LM(b_rnn, arg['c_num'], arg['c_dim_char'], arg['t_num'],
arg['t_dim_char'], arg['droprate'])
try:
with open(arg['lm_dict'], 'r') as fin:
tmp_dict = json.load(fin)
self.lm_char_dict = tmp_dict['lm_char_dict']
self.lm_type_dict = tmp_dict['lm_type_dict']
except FileNotFoundError as err:
logger.error('File not exist: {}'.format(arg['lm_dict']))
raise
self.word_dropout = WordDropout(arg['word_droprate'])
if 'emb_project_to' in arg and arg['emb_project_to'] > 0:
self.project = nn.Linear(arg['char_hid_dim'] * 2,
arg['emb_project_to'])
self.dropout = nn.Dropout(arg['droprate'])
# self.activate = nn.ReLU()
init_linear(self.project)
else:
self.project = None
logger.info('Char-level Representation models building completed.')
def build_pipeline(self, arg):
logger.info('Building language models...')
f_rnn = BasicRNN(arg['frnn_layer_num'], arg['frnn_unit'],
arg['frnn_emb_dim'], arg['frnn_hid_dim'],
arg['lm_droprate'])
self.f_lm = LM(f_rnn, arg['c_num'], arg['c_dim_lm'], arg['t_num'],
arg['t_dim_lm'], arg['lm_droprate'])
b_rnn = BasicRNN(arg['brnn_layer_num'], arg['brnn_unit'],
arg['brnn_emb_dim'], arg['brnn_hid_dim'],
arg['lm_droprate'])
self.b_lm = LM(b_rnn, arg['c_num'], arg['c_dim_lm'], arg['t_num'],
arg['t_dim_lm'], arg['lm_droprate'])
logger.info('Loading forward language model from: {}...'.format(
arg['flm_weight']))
try:
flm_weight = bw.restore_checkpoint(arg['flm_weight'])['model']
self.f_lm.load_state_dict(flm_weight, False)
except FileNotFoundError as err:
logger.error('File not exist: {}'.format(arg['flm_weight']))
raise
logger.info('Loading backward language model from: {}...'.format(
arg['blm_weight']))
try:
blm_weight = bw.restore_checkpoint(arg['blm_weight'])['model']
self.b_lm.load_state_dict(blm_weight, False)
except FileNotFoundError as err:
logger.error('File not exist: {}'.format(arg['blm_weight']))
raise
for param in self.f_lm.parameters():
param.requires_grad = False
for param in self.b_lm.parameters():
param.requires_grad = False
try:
with open(arg['lm_dict'], 'r') as fin:
tmp_dict = json.load(fin)
self.lm_char_dict = tmp_dict['lm_char_dict']
self.lm_type_dict = tmp_dict['lm_type_dict']
except FileNotFoundError as err:
logger.error('File not exist: {}'.format(arg['lm_dict']))
raise
if 'emb_project_to' in arg and arg['emb_project_to'] > 0 and (
arg['emb_project_to'] !=
arg['frnn_hid_dim'] + arg['brnn_hid_dim']):
self.project = nn.Linear(arg['frnn_hid_dim'] + arg['brnn_hid_dim'],
arg['emb_project_to'])
self.dropout = nn.Dropout(arg['droprate'])
# self.activate = nn.ReLU()
init_linear(self.project)
else:
self.project = None
self.word_dropout = WordDropout(arg['word_droprate'])
logger.info('Language models building completed.')