def __init__(self, model, state_space, action_space):
def build_encoder():
if model.encoder.type == "fc":
return FCEncoder(state_space, model.encoder.out_features)
elif model.encoder.type == "conv":
return ConvEncoder(
state_space, model.encoder.base_channels, model.encoder.out_features
)
elif model.encoder.type == "gridworld":
return GridWorldEncoder(
state_space, model.encoder.base_channels, model.encoder.out_features
)
else:
raise AssertionError("invalid type {}".format(model.encoder.type))
def build_policy():
if isinstance(action_space, gym.spaces.Discrete):
return PolicyCategorical(model.encoder.out_features, action_space)
elif isinstance(action_space, gym.spaces.Box):
return PolicyBeta(model.encoder.out_features, action_space)
# return PolicyNormal(model.encoder.out_features, action_space)
else:
raise AssertionError("invalid action_space {}".format(action_space))
def build_value_function():
return ValueFunction(model.encoder.out_features)
super().__init__()
self.encoder = build_encoder()
self.rnn = RNN(model.rnn.type, model.encoder.out_features, model.encoder.out_features)
self.policy = build_policy()
self.value_function = build_value_function()
self.apply(self.weight_init)
def __init__(self, dataset, config):
super(TextCRAN, self).__init__(dataset, config)
self.doc_embedding_type = config.TextCRAN.doc_embedding_type
self.kernel_sizes = config.TextCRAN.kernel_sizes
self.convs = torch.nn.ModuleList()
for kernel_size in self.kernel_sizes:
self.convs.append(
torch.nn.Conv1d(config.embedding.dimension,
config.TextCRAN.num_kernels,
kernel_size,
padding=kernel_size - 1))
self.top_k = self.config.TextCRAN.top_k_max_pooling
hidden_size = len(config.TextCRAN.kernel_sizes) * \
config.TextCRAN.num_kernels * self.top_k
self.rnn = RNN(config.embedding.dimension,
config.TextCRAN.hidden_dimension,
num_layers=config.TextCRAN.num_layers,
batch_first=True,
bidirectional=config.TextCRAN.bidirectional,
rnn_type=config.TextCRAN.rnn_type)
hidden_dimension = config.TextCRAN.hidden_dimension
if config.TextCRAN.bidirectional:
hidden_dimension *= 2
self.sum_attention = SumAttention(
config.TextCRAN.attention_input_dimension,
config.TextCRAN.attention_dimension, config.device)
self.linear = torch.nn.Linear(
config.TextCRAN.attention_input_dimension, len(dataset.label_map))
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def __init__(self, dataset, config):
super(TextRNN, self).__init__(dataset, config)
self.pad = dataset.token_map[dataset.VOCAB_PADDING]
seq_max_len = config.feature.max_token_len
self.position_enc = PositionEmbedding(seq_max_len,
config.embedding.dimension,
self.pad)
self.doc_embedding_type = config.TextRNN.doc_embedding_type
self.rnn = RNN(config.embedding.dimension,
config.TextRNN.hidden_dimension,
num_layers=config.TextRNN.num_layers,
batch_first=True,
bidirectional=config.TextRNN.bidirectional,
rnn_type=config.TextRNN.rnn_type)
hidden_dimension = config.TextRNN.hidden_dimension
if config.TextRNN.bidirectional:
hidden_dimension *= 2
self.sum_attention = SumAttention(hidden_dimension,
config.TextRNN.attention_dimension,
config.device)
self.linear = torch.nn.Linear(hidden_dimension, len(dataset.label_map))
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def __init__(self, dataset, config):
assert config.label_embedding.dimension == config.ZSJLRNN.gcn_in_features, \
"label embedding dimension should be same as gcn input feature dimension"
assert len(config.data.label_relation_files) >= 2, \
"this model should utilize at least 2 different graphs' adjacency"
super(ZSJLRNN, self).__init__(dataset, config)
self.rnn = RNN(config.embedding.dimension,
config.ZSJLRNN.hidden_dimension,
num_layers=config.ZSJLRNN.num_layers,
batch_first=True,
bidirectional=config.ZSJLRNN.bidirectional,
rnn_type=config.ZSJLRNN.rnn_type)
self.label_wise_attention = LabelWiseAttention(
feat_dim=config.ZSJLRNN.hidden_dimension * 2 if
config.ZSJLRNN.bidirectional else config.ZSJLRNN.hidden_dimension,
label_emb_dim=config.label_embedding.dimension,
store_attention_score=config.ZSJLRNN.store_attention_score)
self.multi_gcn = torch.nn.ModuleList([
MultiGraphConvolution(
n_adj=len(config.data.label_relation_files),
in_features=config.ZSJLRNN.gcn_in_features,
out_features=config.ZSJLRNN.gcn_hidden_features,
bias=True,
act=torch.relu_,
featureless=False,
dropout=config.ZSJLRNN.gcn_dropout),
MultiGraphConvolution(
n_adj=len(config.data.label_relation_files),
in_features=config.ZSJLRNN.gcn_hidden_features,
out_features=config.ZSJLRNN.gcn_out_features,
bias=True,
act=torch.relu_,
featureless=False,
dropout=config.ZSJLRNN.gcn_dropout)
])
self.multi_gcn_fuse = Fusion(config)
if config.fusion.fusion_type == FusionType.CONCATENATION:
out_tmp = config.ZSJLRNN.gcn_in_features + config.fusion.out_features
elif config.fusion.fusion_type == FusionType.ATTACH:
out_tmp = config.ZSJLRNN.gcn_in_features + \
config.ZSJLRNN.gcn_out_features * len(config.data.label_relation_files)
else:
raise NotImplementedError
self.doc_out_transform = torch.nn.Sequential(
torch.nn.Linear(in_features=config.ZSJLRNN.hidden_dimension *
2 if config.ZSJLRNN.bidirectional else
config.ZSJLRNN.hidden_dimension,
out_features=out_tmp), torch.nn.ReLU())
def __init__(self, dataset, config):
super(TextRNN, self).__init__(dataset, config)
self.rnn = RNN(
config.embedding.dimension, config.TextRNN.hidden_dimension,
num_layers=config.TextRNN.num_layers, batch_first=True,
bidirectional=config.TextRNN.bidirectional,
rnn_type=config.TextRNN.rnn_type)
hidden_dimension = config.TextRNN.hidden_dimension
if config.TextRNN.bidirectional:
hidden_dimension *= 2
self.sum_attention = SumAttention(hidden_dimension,
config.TextRNN.attention_dimension)
self.linear = torch.nn.Linear(hidden_dimension, len(dataset.label_map))
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def __init__(self, hidden_dim, input_dim, future):
super().__init__()
self.hidden_dim = hidden_dim
self.input_dim = input_dim
self.rnn = RNN(TimeLSICell(input_dim, hidden_dim, 16))
self.hidden2out = nn.Linear(hidden_dim, future)
def __init__(self, hidden_dim, input_dim, future):
super().__init__()
self.hidden_dim = hidden_dim
self.input_dim = input_dim
self.ortho_rnn = RNN(OrthogonalCell(input_dim, hidden_dim))
self.hidden2out = nn.Linear(hidden_dim, future)
class Model(nn.Module):
def __init__(self, model, state_space, action_space):
def build_encoder():
if model.encoder.type == "fc":
return FCEncoder(state_space, model.encoder.out_features)
elif model.encoder.type == "conv":
return ConvEncoder(
state_space, model.encoder.base_channels, model.encoder.out_features
)
elif model.encoder.type == "gridworld":
return GridWorldEncoder(
state_space, model.encoder.base_channels, model.encoder.out_features
)
else:
raise AssertionError("invalid type {}".format(model.encoder.type))
def build_policy():
if isinstance(action_space, gym.spaces.Discrete):
return PolicyCategorical(model.encoder.out_features, action_space)
elif isinstance(action_space, gym.spaces.Box):
return PolicyBeta(model.encoder.out_features, action_space)
# return PolicyNormal(model.encoder.out_features, action_space)
else:
raise AssertionError("invalid action_space {}".format(action_space))
def build_value_function():
return ValueFunction(model.encoder.out_features)
super().__init__()
self.encoder = build_encoder()
self.rnn = RNN(model.rnn.type, model.encoder.out_features, model.encoder.out_features)
self.policy = build_policy()
self.value_function = build_value_function()
self.apply(self.weight_init)
def forward(self, input, h, d):
input = self.encoder(input)
input, h = self.rnn(input, h, d)
dist = self.policy(input)
value = self.value_function(input)
return dist, value, h
def zero_state(self, batch_size):
return self.rnn.zero_state(batch_size)
@staticmethod
def weight_init(m):
if isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.Conv2d):
nn.init.normal_(m.weight, 0, 0.01)
if m.bias is not None:
nn.init.constant_(m.bias, 0)
def __init__(self, dataset, config):
assert config.label_embedding.dimension == config.ZAGRNN.gcn_in_features, \
"label embedding dimension should be same as gcn input feature dimension"
super(ZAGRNN, self).__init__(dataset, config)
self.rnn = RNN(
config.embedding.dimension, config.ZAGRNN.hidden_dimension,
num_layers=config.ZAGRNN.num_layers, batch_first=True,
bidirectional=config.ZAGRNN.bidirectional,
rnn_type=config.ZAGRNN.rnn_type)
self.label_wise_attention = LabelWiseAttention(
feat_dim=config.ZAGRNN.hidden_dimension*2 if config.ZAGRNN.bidirectional else config.ZAGRNN.hidden_dimension,
label_emb_dim=config.label_embedding.dimension,
store_attention_score=config.ZAGRNN.store_attention_score)
if config.ZAGRNN.use_gcn:
self.gcn = torch.nn.ModuleList([
GraphConvolution(
in_features=config.ZAGRNN.gcn_in_features,
out_features=config.ZAGRNN.gcn_hidden_features,
bias=True,
act=torch.relu_,
featureless=False,
dropout=config.ZAGRNN.gcn_dropout),
GraphConvolution(
in_features=config.ZAGRNN.gcn_hidden_features,
out_features=config.ZAGRNN.gcn_out_features,
bias=True,
act=torch.relu_,
featureless=False,
dropout=config.ZAGRNN.gcn_dropout)
])
self.doc_out_transform = torch.nn.Sequential(
torch.nn.Linear(
in_features=config.ZAGRNN.hidden_dimension*2 if config.ZAGRNN.bidirectional else config.ZAGRNN.hidden_dimension,
out_features=config.ZAGRNN.gcn_in_features + config.ZAGRNN.gcn_out_features
),
torch.nn.ReLU()
)
def main(random_state=100):
data_type = 'payload' # header, header_payload
X_train, y_train, X_test, y_test = load_flow_data(
random_state=random_state, data_type=data_type)
in_dim = len(X_train[0][0])
rnn = RNN(n_epochs=100,
in_dim=in_dim,
out_dim=10,
n_layers=1,
lr=1e-3,
bias=False,
random_state=random_state)
rnn.train(X_train=X_train,
y_train=y_train,
X_val=X_test,
y_val=y_test,
split=True)
#计算test运行时间
begin_time = time()
rnn.test(X_test=X_test, y_test=y_test, split=True)
end_time = time()
run_time = end_time - begin_time
print("test time：", run_time)
def __init__(self, dataset, config):
super(HMCN, self).__init__(dataset, config)
self.hierarchical_depth = config.HMCN.hierarchical_depth
self.hierarchical_class = dataset.hierarchy_classes
print("hierarchy_classes: {}".format(self.hierarchical_class))
self.global2local = config.HMCN.global2local
self.rnn = RNN(config.embedding.dimension,
config.TextRNN.hidden_dimension,
num_layers=config.TextRNN.num_layers,
batch_first=True,
bidirectional=config.TextRNN.bidirectional,
rnn_type=config.TextRNN.rnn_type)
hidden_dimension = config.TextRNN.hidden_dimension
if config.TextRNN.bidirectional:
hidden_dimension *= 2
self.local_layers = torch.nn.ModuleList()
self.global_layers = torch.nn.ModuleList()
for i in range(1, len(self.hierarchical_depth)):
self.global_layers.append(
torch.nn.Sequential(
torch.nn.Linear(
hidden_dimension + self.hierarchical_depth[i - 1],
self.hierarchical_depth[i]), torch.nn.ReLU(),
torch.nn.BatchNorm1d(self.hierarchical_depth[i]),
torch.nn.Dropout(p=0.5)))
self.local_layers.append(
torch.nn.Sequential(
torch.nn.Linear(self.hierarchical_depth[i],
self.global2local[i]), torch.nn.ReLU(),
torch.nn.BatchNorm1d(self.global2local[i]),
torch.nn.Linear(self.global2local[i],
self.hierarchical_class[i - 1])))
self.global_layers.apply(self._init_weight)
self.local_layers.apply(self._init_weight)
self.linear = torch.nn.Linear(self.hierarchical_depth[-1],
len(dataset.label_map))
self.linear.apply(self._init_weight)
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def _build_model(time_step, fearure_dim, modelName):
i = Input(shape=(time_step, fearure_dim))
if modelName == 'FEMT_LSTM':
m = RNN_SEPARATE_2(time_step, fearure_dim)(i)
else:
m = RNN(time_step, fearure_dim)(i)
m = Dense(1, activation='sigmoid')(m)
model = Model(inputs=[i], outputs=[m])
model.summary()
# model.compile(optimizer=optimizers.Adam(lr=0.001, clipvalue=15), loss=focal_loss(alpha=0.75, gamma=0)) #alpha=0.75, gamma=0 Adam(lr=0.0001)
model.compile(optimizer=optimizers.Adam(lr=0.001, clipvalue=15),
loss=myloss(alpha=0.75,
gamma=0)) #alpha=0.75, gamma=0 Adam(lr=0.0001)
return model
def __init__(self, dataset, config):
super(DRNN, self).__init__(dataset, config)
self.rnn_type = config.DRNN.rnn_type
self.forward_rnn = RNN(config.embedding.dimension,
config.DRNN.hidden_dimension,
batch_first=True,
rnn_type=config.DRNN.rnn_type)
if config.DRNN.bidirectional:
self.backward_rnn = RNN(config.embedding.dimension,
config.DRNN.hidden_dimension,
batch_first=True,
rnn_type=config.DRNN.rnn_type)
self.window_size = config.DRNN.window_size
self.dropout = torch.nn.Dropout(p=config.DRNN.cell_hidden_dropout)
self.hidden_dimension = config.DRNN.hidden_dimension
if config.DRNN.bidirectional:
self.hidden_dimension *= 2
self.batch_norm = torch.nn.BatchNorm1d(self.hidden_dimension)
self.mlp = torch.nn.Linear(self.hidden_dimension,
self.hidden_dimension)
self.linear = torch.nn.Linear(self.hidden_dimension,
len(dataset.label_map))
def main(random_state=100):
data_type = 'payload' # header, header_payload
X_train, y_train, X_test, y_test = load_flow_data(random_state=random_state, data_type=data_type)
in_dim = len(X_train[0][0])
rnn = RNN(n_epochs=100, in_dim=in_dim, out_dim=10, n_layers=1, lr=1e-3, bias=False, random_state=random_state)
rnn.train(X_train=X_train, y_train=y_train, X_val=X_test, y_val=y_test, split=True)
rnn.test(X_test=X_test, y_test=y_test, split=True)
def __init__(self, config):
super().__init__()
self.label_emb = config.HybridRCNN.label_emb
# 定义rnn,使用GRU模式
self.rnn = RNN(config.embedding.dimension,
config.HybridRCNN.hidden_dimension,
num_layers=config.HybridRCNN.num_layers,
batch_first=True,
bidirectional=config.HybridRCNN.bidirectional,
rnn_type=config.HybridRCNN.rnn_type)
hidden_dimension = config.HybridRCNN.hidden_dimension
if config.HybridRCNN.bidirectional:
hidden_dimension *= 2
# 定义 self-attention
self.sum_attention = LabelSumAttention(
hidden_dimension, config.HybridRCNN.attention_dimension,
config.HybridRCNN.num_labels, config.device)
for line in lines:
temp = line.strip('\n').strip('\r').strip(' ')
charList[temp] = ii
ii += 1
max_features = ii
x_data_sum = []
#
with open(trainDataPath, 'r') as trainFile:
lines = trainFile.read().split('\n')
for line in lines:
if line.strip('\n').strip('\r').strip(' ') == '':
continue
x_data = []
x = line.strip('\n').strip('\r').strip(' ').split(',')[0]
for char in x:
try:
x_data.append(charList[char])
except:
print('unexpected char' + ' : ' + char)
print(line)
x_data.append(0)
x_data_sum.append(x_data)
x_data_sum = np.array(x_data_sum)
rnn_binary = RNN()
rnn_binary.predict(x_data_sum, resultPath, modelPath=modelPath)
class HMCN(Classifier):
""" Implement HMCN(Hierarchical Multi-Label Classification Networks)
Reference: "Hierarchical Multi-Label Classification Networks"
"""
def __init__(self, dataset, config):
super(HMCN, self).__init__(dataset, config)
self.hierarchical_depth = config.HMCN.hierarchical_depth
self.hierarchical_class = dataset.hierarchy_classes
self.global2local = config.HMCN.global2local
self.rnn = RNN(
config.embedding.dimension, config.TextRNN.hidden_dimension,
num_layers=config.TextRNN.num_layers, batch_first=True,
bidirectional=config.TextRNN.bidirectional,
rnn_type=config.TextRNN.rnn_type)
hidden_dimension = config.TextRNN.hidden_dimension
if config.TextRNN.bidirectional:
hidden_dimension *= 2
self.local_layers = torch.nn.ModuleList()
self.global_layers = torch.nn.ModuleList()
for i in range(1, len(self.hierarchical_depth)):
self.global_layers.append(
torch.nn.Sequential(
torch.nn.Linear(hidden_dimension + self.hierarchical_depth[i-1], self.hierarchical_depth[i]),
torch.nn.ReLU(),
torch.nn.BatchNorm1d(self.hierarchical_depth[i]),
torch.nn.Dropout(p=0.5)
))
self.local_layers.append(
torch.nn.Sequential(
torch.nn.Linear(self.hierarchical_depth[i], self.global2local[i]),
torch.nn.ReLU(),
torch.nn.BatchNorm1d(self.global2local[i]),
torch.nn.Linear(self.global2local[i], self.hierarchical_class[i-1])
))
self.global_layers.apply(self._init_weight)
self.local_layers.apply(self._init_weight)
self.linear = torch.nn.Linear(self.hierarchical_depth[-1], len(dataset.label_map))
self.linear.apply(self._init_weight)
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def _init_weight(self, m):
if isinstance(m, torch.nn.Linear):
torch.nn.init.normal_(m.weight, std=0.1)
def get_parameter_optimizer_dict(self):
params = super(HMCN, self).get_parameter_optimizer_dict()
params.append({'params': self.rnn.parameters()})
params.append({'params': self.local_layers.parameters()})
params.append({'params': self.global_layers.parameters()})
params.append({'params': self.linear.parameters()})
return params
def update_lr(self, optimizer, epoch):
""" Update lr
"""
if epoch > self.config.train.num_epochs_static_embedding:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = self.config.optimizer.learning_rate
else:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = 0
def forward(self, batch):
if self.config.feature.feature_names[0] == "token":
embedding = self.token_embedding(
batch[cDataset.DOC_TOKEN].to(self.config.device))
length = batch[cDataset.DOC_TOKEN_LEN].to(self.config.device)
else:
embedding = self.char_embedding(
batch[cDataset.DOC_TOKEN].to(self.config.device))
length = batch[cDataset.DOC_CHAR_LEN].to(self.config.device)
output, last_hidden = self.rnn(embedding, length)
doc_embedding = torch.sum(output, 1) / length.unsqueeze(1)
local_layer_outputs = []
global_layer_activation = doc_embedding
batch_size = doc_embedding.size()[0]
for i, (local_layer, global_layer) in enumerate(zip(self.local_layers, self.global_layers)):
local_layer_activation = global_layer(global_layer_activation)
local_layer_outputs.append(local_layer(local_layer_activation))
if i < len(self.global_layers)-1:
global_layer_activation = torch.cat((local_layer_activation, doc_embedding), 1)
else:
global_layer_activation = local_layer_activation
global_layer_output = self.linear(global_layer_activation)
local_layer_output = torch.cat(local_layer_outputs, 1)
return global_layer_output, local_layer_output, 0.5 * global_layer_output + 0.5 * local_layer_output
def __init__(
self,
input_size,
output_size,
output_len=0,
cell='lstm',
cell_args={},
output_hiddens=[],
embed_args=None,
preprocess=None,
ff=False,
dropout=0.0,
split=0,
):
super(Model, self).__init__()
# Save arguments needed for forward pass
self.input_size = input_size
self.output_size = output_size
self.output_len = output_len
assert output_len >= 0, f"output_len {output_len} should be 0 to return just the state or >0 to return the last output tokens"
self.dropout = dropout
self.split = split
cell_args['input_size'] = input_size
if embed_args is not None:
self.embed_dim = embed_args['embed_dim']
self.embedding = nn.Embedding(input_size, self.embed_dim)
cell_args['input_size'] = self.embed_dim
### Handle optional Hippo preprocessing
self.preprocess = preprocess
if self.preprocess is not None:
assert isinstance(self.preprocess, dict)
assert 'order' in self.preprocess
assert 'measure' in self.preprocess
self.hippo = VariableMemoryProjection(**self.preprocess)
cell_args['input_size'] *= (
self.preprocess['order'] + 1
) # will append this output to original channels
### Construct main RNN
if ff: # feedforward model
cell_args['input_size'] = input_size
self.rnn = QRNN(**cell_args)
else:
# Initialize proper cell type
if cell == 'lstm':
self.rnn = LSTMWrapper(**cell_args, dropout=self.dropout)
else:
if cell in CellBase.registry:
cell_ctor = CellBase.registry[cell]
elif cell == 'orthogonal':
cell_ctor = OrthogonalCell
else:
assert False, f"cell {cell} not supported"
self.rnn = RNN(cell_ctor(**cell_args), dropout=self.dropout)
if self.split > 0:
self.initial_rnn = RNN(cell_ctor(**cell_args),
dropout=self.dropout)
### Construct output head
sizes = [self.rnn.output_size()] + output_hiddens + [output_size]
self.output_mlp = nn.Sequential(
*
[nn.Linear(sizes[i], sizes[i + 1]) for i in range(len(sizes) - 1)])
class Model(nn.Module):
def __init__(
self,
input_size,
output_size,
output_len=0,
cell='lstm',
cell_args={},
output_hiddens=[],
embed_args=None,
preprocess=None,
ff=False,
dropout=0.0,
split=0,
):
super(Model, self).__init__()
# Save arguments needed for forward pass
self.input_size = input_size
self.output_size = output_size
self.output_len = output_len
assert output_len >= 0, f"output_len {output_len} should be 0 to return just the state or >0 to return the last output tokens"
self.dropout = dropout
self.split = split
cell_args['input_size'] = input_size
if embed_args is not None:
self.embed_dim = embed_args['embed_dim']
self.embedding = nn.Embedding(input_size, self.embed_dim)
cell_args['input_size'] = self.embed_dim
### Handle optional Hippo preprocessing
self.preprocess = preprocess
if self.preprocess is not None:
assert isinstance(self.preprocess, dict)
assert 'order' in self.preprocess
assert 'measure' in self.preprocess
self.hippo = VariableMemoryProjection(**self.preprocess)
cell_args['input_size'] *= (
self.preprocess['order'] + 1
) # will append this output to original channels
### Construct main RNN
if ff: # feedforward model
cell_args['input_size'] = input_size
self.rnn = QRNN(**cell_args)
else:
# Initialize proper cell type
if cell == 'lstm':
self.rnn = LSTMWrapper(**cell_args, dropout=self.dropout)
else:
if cell in CellBase.registry:
cell_ctor = CellBase.registry[cell]
elif cell == 'orthogonal':
cell_ctor = OrthogonalCell
else:
assert False, f"cell {cell} not supported"
self.rnn = RNN(cell_ctor(**cell_args), dropout=self.dropout)
if self.split > 0:
self.initial_rnn = RNN(cell_ctor(**cell_args),
dropout=self.dropout)
### Construct output head
sizes = [self.rnn.output_size()] + output_hiddens + [output_size]
self.output_mlp = nn.Sequential(
*
[nn.Linear(sizes[i], sizes[i + 1]) for i in range(len(sizes) - 1)])
# @profile
def forward(self, inputs, len_batch=None):
B, L, C = inputs.shape
inputs = inputs.transpose(
0, 1) # .unsqueeze(-1) # (seq_length, batch, channels)
# Apply Hippo preprocessing if necessary
if self.preprocess is not None:
p = self.hippo(inputs)
p = p.reshape(L, B, self.input_size * self.preprocess['order'])
inputs = torch.cat([inputs, p], dim=-1)
# Handle embedding
if hasattr(self, 'embedding'):
inputs = self.embedding(inputs)
if len_batch is not None:
inputs = nn.utils.rnn.pack_padded_sequence(inputs,
len_batch,
enforce_sorted=False)
# Option to have separate RNN for head of sequence, mostly for debugging gradients etc
if self.split > 0:
initial_inputs, inputs = inputs[:self.split], inputs[self.split:]
_, initial_state = self.initial_rnn(initial_inputs,
return_output=False)
else:
initial_state = None
# Apply main RNN
if self.output_len > 0:
outputs, _ = self.rnn(inputs,
init_state=initial_state,
return_output=True)
# get last output tokens
outputs = outputs[-self.output_len:, :, :]
outputs = outputs.transpose(0, 1)
return self.output_mlp(outputs)
else:
_, state = self.rnn(inputs,
init_state=initial_state,
return_output=False)
state = self.rnn.output(state)
return self.output_mlp(state)
class DRNN(Classifier):
def __init__(self, dataset, config):
super(DRNN, self).__init__(dataset, config)
self.rnn_type = config.DRNN.rnn_type
self.forward_rnn = RNN(config.embedding.dimension,
config.DRNN.hidden_dimension,
batch_first=True,
rnn_type=config.DRNN.rnn_type)
if config.DRNN.bidirectional:
self.backward_rnn = RNN(config.embedding.dimension,
config.DRNN.hidden_dimension,
batch_first=True,
rnn_type=config.DRNN.rnn_type)
self.window_size = config.DRNN.window_size
self.dropout = torch.nn.Dropout(p=config.DRNN.cell_hidden_dropout)
self.hidden_dimension = config.DRNN.hidden_dimension
if config.DRNN.bidirectional:
self.hidden_dimension *= 2
self.batch_norm = torch.nn.BatchNorm1d(self.hidden_dimension)
self.mlp = torch.nn.Linear(self.hidden_dimension,
self.hidden_dimension)
self.linear = torch.nn.Linear(self.hidden_dimension,
len(dataset.label_map))
def get_parameter_optimizer_dict(self):
params = super(DRNN, self).get_parameter_optimizer_dict()
params.append({'params': self.forward_rnn.parameters()})
if self.config.DRNN.bidirectional:
params.append({'params': self.backward_rnn.parameters()})
params.append({'params': self.batch_norm.parameters()})
params.append({'params': self.mlp.parameters()})
params.append({'params': self.linear.parameters()})
return params
def forward(self, batch):
front_pad_embedding, _, mask = self.get_embedding(
batch, [self.window_size - 1, 0], cDataset.VOCAB_PADDING_LEARNABLE)
if self.config.DRNN.bidirectional:
tail_pad_embedding, _, _ = self.get_embedding(
batch, [0, self.window_size - 1],
cDataset.VOCAB_PADDING_LEARNABLE)
batch_size = front_pad_embedding.size(0)
mask = mask.unsqueeze(2)
front_slice_embedding_list = \
[front_pad_embedding[:, i:i + self.window_size, :] for i in
range(front_pad_embedding.size(1) - self.window_size + 1)]
front_slice_embedding = torch.cat(front_slice_embedding_list, dim=0)
state = None
for i in range(front_slice_embedding.size(1)):
_, state = self.forward_rnn(front_slice_embedding[:, i:i + 1, :],
init_state=state,
ori_state=True)
if self.rnn_type == RNNType.LSTM:
state[0] = self.dropout(state[0])
else:
state = self.dropout(state)
front_state = state[0] if self.rnn_type == RNNType.LSTM else state
front_state = front_state.transpose(0, 1)
front_hidden = torch.cat(front_state.split(batch_size, dim=0), dim=1)
front_hidden = front_hidden * mask
hidden = front_hidden
if self.config.DRNN.bidirectional:
tail_slice_embedding_list = list()
for i in range(tail_pad_embedding.size(1) - self.window_size + 1):
slice_embedding = \
tail_pad_embedding[:, i:i + self.window_size, :]
tail_slice_embedding_list.append(slice_embedding)
tail_slice_embedding = torch.cat(tail_slice_embedding_list, dim=0)
state = None
for i in range(tail_slice_embedding.size(1), 0, -1):
_, state = self.backward_rnn(tail_slice_embedding[:,
i - 1:i, :],
init_state=state,
ori_state=True)
if i != tail_slice_embedding.size(1) - 1:
if self.rnn_type == RNNType.LSTM:
state[0] = self.dropout(state[0])
else:
state = self.dropout(state)
tail_state = state[0] if self.rnn_type == RNNType.LSTM else state
tail_state = tail_state.transpose(0, 1)
tail_hidden = torch.cat(tail_state.split(batch_size, dim=0), dim=1)
tail_hidden = tail_hidden * mask
hidden = torch.cat([hidden, tail_hidden], dim=2)
hidden = hidden.transpose(1, 2).contiguous()
batch_normed = self.batch_norm(hidden).transpose(1, 2)
batch_normed = batch_normed * mask
mlp_hidden = self.mlp(batch_normed)
mlp_hidden = mlp_hidden * mask
neg_mask = (mask - 1) * 65500.0
mlp_hidden = mlp_hidden + neg_mask
max_pooling = torch.nn.functional.max_pool1d(
mlp_hidden.transpose(1, 2), mlp_hidden.size(1)).squeeze()
return self.linear(self.dropout(max_pooling))
class TextRNN(Classifier):
"""Implement TextRNN, contains LSTM，BiLSTM，GRU，BiGRU
Reference: "Effective LSTMs for Target-Dependent Sentiment Classification"
"Bidirectional LSTM-CRF Models for Sequence Tagging"
"Generative and discriminative text classification
with recurrent neural networks"
"""
def __init__(self, dataset, config):
super(TextRNN, self).__init__(dataset, config)
self.rnn = RNN(
config.embedding.dimension, config.TextRNN.hidden_dimension,
num_layers=config.TextRNN.num_layers, batch_first=True,
bidirectional=config.TextRNN.bidirectional,
rnn_type=config.TextRNN.rnn_type)
hidden_dimension = config.TextRNN.hidden_dimension
if config.TextRNN.bidirectional:
hidden_dimension *= 2
self.sum_attention = SumAttention(hidden_dimension,
config.TextRNN.attention_dimension)
self.linear = torch.nn.Linear(hidden_dimension, len(dataset.label_map))
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def get_parameter_optimizer_dict(self):
params = super(TextRNN, self).get_parameter_optimizer_dict()
params.append({'params': self.rnn.parameters()})
params.append({'params': self.linear.parameters()})
return params
def update_lr(self, optimizer, epoch):
if epoch > self.config.train.num_epochs_static_embedding:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = self.config.optimizer.learning_rate
else:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = 0.0
def forward(self, batch):
if self.config.feature.feature_names[0] == "token":
embedding = self.token_embedding(
batch[cDataset.DOC_TOKEN].to(self.config.device))
length = batch[cDataset.DOC_TOKEN_LEN].to(self.config.device)
else:
embedding = self.char_embedding(
batch[cDataset.DOC_CHAR].to(self.config.device))
length = batch[cDataset.DOC_CHAR_LEN].to(self.config.device)
output, last_hidden = self.rnn(embedding, length)
doc_embedding_type = self.config.TextRNN.doc_embedding_type
if doc_embedding_type == DocEmbeddingType.AVG:
doc_embedding = torch.sum(output, 1) / length.unsqueeze(1)
elif doc_embedding_type == DocEmbeddingType.ATTENTION:
doc_embedding = self.sum_attention(output)
elif doc_embedding_type == DocEmbeddingType.LAST_HIDDEN:
doc_embedding = last_hidden
else:
raise TypeError(
"Unsupported rnn init type: %s. Supported rnn type is: %s" % (
doc_embedding_type, DocEmbeddingType.str()))
return self.dropout(self.linear(doc_embedding))
lines = confFile.read().split('\n')
ii = 0
for line in lines:
temp = line.strip('\n').strip('\r').strip(' ')
charList[temp] = ii
ii += 1
return charList
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--domain', required=True)
io_args = parser.parse_args()
domain = io_args.domain
charList = getCharList(charListPath)
print(charList)
x_data = []
for char in domain:
try:
x_data.append(charList[char])
except:
print('unexpected char' + ' : ' + char)
x_data.append(0)
print(x_data)
rnn_binary_model = RNN()
rnn_binary_model.predict_p([x_data], modelPath=modelPath)
class TextCRVariant(Classifier):
def __init__(self, dataset, config):
super(TextCRVariant, self).__init__(dataset, config)
self.label_semantic_emb = config.TextCRVariant.label_semantic_emb
self.doc_embedding_type = config.TextCRVariant.doc_embedding_type
self.kernel_sizes = config.TextCRVariant.kernel_sizes
self.convs = torch.nn.ModuleList()
for kernel_size in self.kernel_sizes:
self.convs.append(
torch.nn.Conv1d(384,
config.TextCRVariant.num_kernels,
kernel_size,
padding=kernel_size - 1))
self.top_k = self.config.TextCRVariant.top_k_max_pooling
hidden_size = len(config.TextCRVariant.kernel_sizes) * \
config.TextCRVariant.num_kernels * self.top_k
self.rnn3 = torch.nn.GRU(256,
256,
num_layers=config.TextCRVariant.num_layers,
batch_first=True)
self.rnn1 = RNN(config.embedding.dimension,
config.TextCRVariant.hidden_dimension,
num_layers=config.TextCRVariant.num_layers,
batch_first=True,
bidirectional=config.TextCRVariant.bidirectional,
rnn_type=config.TextCRVariant.rnn_type)
self.rnn2 = RNN(384,
config.TextCRVariant.hidden_dimension,
num_layers=config.TextCRVariant.num_layers,
batch_first=True,
bidirectional=config.TextCRVariant.bidirectional,
rnn_type=config.TextCRVariant.rnn_type)
hidden_dimension = config.TextCRVariant.hidden_dimension
if config.TextCRVariant.bidirectional:
hidden_dimension *= 2
self.sum_attention = SumAttention(
hidden_dimension, config.TextCRVariant.attention_dimension,
config.device)
self.linear = torch.nn.Linear(hidden_size, len(dataset.label_map))
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def get_parameter_optimizer_dict(self):
params = list()
params.append({'params': self.token_embedding.parameters()})
params.append({'params': self.char_embedding.parameters()})
params.append({'params': self.convs.parameters()})
params.append({'params': self.rnn1.parameters()})
params.append({'params': self.rnn2.parameters()})
params.append({'params': self.linear.parameters()})
return params
def update_lr(self, optimizer, epoch):
"""Update lr
"""
if epoch > self.config.train.num_epochs_static_embedding:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = self.config.optimizer.learning_rate
else:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = 0
def forward(self, batch):
if self.config.feature.feature_names[0] == "token":
embedding = self.token_embedding(batch[cDataset.DOC_TOKEN].to(
self.config.device))
seq_length = batch[cDataset.DOC_TOKEN_LEN].to(self.config.device)
else:
embedding = self.char_embedding(batch[cDataset.DOC_CHAR].to(
self.config.device))
seq_length = batch[cDataset.DOC_CHAR_LEN].to(self.config.device)
label_semantic_emb = self.label_semantic_emb.cuda()
label_semantic_emb = label_semantic_emb.expand(
(embedding.size(0), label_semantic_emb.size(0),
label_semantic_emb.size(1))) # [batch,L,256]
output1, last_hidden = self.rnn3(label_semantic_emb) # [batch,n,256]
last_hidden = last_hidden.squeeze(dim=0)
last_hidden = last_hidden.expand(
(embedding.size(1), last_hidden.size(0),
last_hidden.size(1))) # [batch,n,256]
last_hidden = last_hidden.transpose(0, 1)
doc_embedding, _ = self.rnn1(embedding, seq_length) #[batch,n,256]
input = torch.cat((doc_embedding, last_hidden), 2) ##[batch,512]
doc_embedding = input.transpose(1, 2)
pooled_outputs = []
for _, conv in enumerate(self.convs):
convolution = torch.nn.functional.relu(conv(doc_embedding))
pooled = torch.topk(convolution,
self.top_k)[0].view(convolution.size(0), -1)
pooled_outputs.append(pooled)
doc_embedding = torch.cat(pooled_outputs, 1)
out = self.dropout(self.linear(doc_embedding))
return out
class TextGRA(Classifier):
def __init__(self, dataset, config):
super(TextGRA, self).__init__(dataset, config)
self.doc_embedding_type = config.TextGRA.doc_embedding_type
self.kernel_sizes = config.TextGRA.kernel_sizes
self.convs = torch.nn.ModuleList()
for kernel_size in self.kernel_sizes:
self.convs.append(
torch.nn.Conv1d(config.embedding.dimension,
config.TextGRA.num_kernels,
kernel_size,
padding=kernel_size - 1))
self.top_k = self.config.TextGRA.top_k_max_pooling
hidden_size = len(config.TextGRA.kernel_sizes) * \
config.TextGRA.num_kernels * self.top_k
self.rnn = RNN(config.embedding.dimension,
config.TextGRA.hidden_dimension,
num_layers=config.TextGRA.num_layers,
batch_first=True,
bidirectional=config.TextGRA.bidirectional,
rnn_type=config.TextGRA.rnn_type)
self.rnn2 = RNN(600,
config.TextGRA.hidden_dimension,
num_layers=config.TextGRA.num_layers,
batch_first=True,
bidirectional=config.TextGRA.bidirectional,
rnn_type=config.TextGRA.rnn_type)
hidden_dimension = config.TextGRA.hidden_dimension
if config.TextGRA.bidirectional:
hidden_dimension *= 2
self.linear_u = torch.nn.Linear(300, 300)
self.linear = torch.nn.Linear(hidden_dimension, len(dataset.label_map))
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def get_parameter_optimizer_dict(self):
params = list()
params.append({'params': self.token_embedding.parameters()})
params.append({'params': self.char_embedding.parameters()})
params.append({'params': self.convs.parameters()})
params.append({'params': self.rnn.parameters()})
params.append({'params': self.rnn2.parameters()})
params.append({'params': self.linear_u.parameters()})
params.append({'params': self.linear.parameters()})
return params
def update_lr(self, optimizer, epoch):
"""Update lr
"""
if epoch > self.config.train.num_epochs_static_embedding:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = self.config.optimizer.learning_rate
else:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = 0
def forward(self, batch):
if self.config.feature.feature_names[0] == "token":
embedding = self.token_embedding(batch[cDataset.DOC_TOKEN].to(
self.config.device))
seq_length = batch[cDataset.DOC_TOKEN_LEN].to(self.config.device)
else:
embedding = self.char_embedding(batch[cDataset.DOC_CHAR].to(
self.config.device))
seq_length = batch[cDataset.DOC_CHAR_LEN].to(self.config.device)
# RNN sentence vector
S, output1 = self.rnn(embedding, seq_length) #[batch,n, 300]
embedding2 = embedding.transpose(1, 2)
# CNN parase vector
pooled_outputs = []
for i, conv in enumerate(self.convs):
#convolution = torch.nn.ReLU(conv(embedding))
convolution = torch.nn.functional.relu(
conv(embedding2)) #[batch,100,n]
pooled = torch.topk(convolution,
self.top_k)[0].view(convolution.size(0), -1)
pooled_outputs.append(pooled)
P = torch.cat(pooled_outputs, 1) ##[batch,300]
P = P.expand(embedding.size(1), P.size(0), P.size(1)) #[batch,n,300]
# attention scoring
attention_score = self.linear_u(
torch.tanh(torch.mul(S, P.transpose(0, 1)))) #[batch,n,300]
# attention gate
A, output2 = self.rnn(attention_score, seq_length) # [batch,n, 450]
# attention parase vector
C = torch.mul(A, P.transpose(0, 1)) #[batch,n,450]
# combine [x,c] input into RNN
input = torch.cat((C, embedding), 2) ##[batch,n,600]
output, last_hidden = self.rnn2(input, seq_length) #[batch,n,450]
# average output
if self.doc_embedding_type == DocEmbeddingType.AVG:
doc_embedding = torch.sum(output, 1) / seq_length.unsqueeze(1)
out = self.dropout(self.linear(doc_embedding))
return out
max_features = ii
x_data_sum = []
x_names = []
#
with open(trainDataPath, 'r') as trainFile:
lines = trainFile.read().split('\n')
for line in lines:
if line.strip('\n').strip('\r').strip(' ') == '':
continue
x_data = []
x = line.strip('\n').strip('\r').strip(' ').split(',')[0]
x_names.append(x)
for char in x:
try:
x_data.append(charList[char])
except:
print('unexpected char' + ' : ' + char)
print(line)
x_data.append(0)
x_data_sum.append(x_data)
x_data_sum = np.array(x_data_sum)
rnn_binary = RNN()
rnn_binary.predict_dense_out(x_data_sum,
x_names,
resultPath,
modelPath=modelPath)
class TextCRAN(Classifier):
def __init__(self, dataset, config):
super(TextCRAN, self).__init__(dataset, config)
self.doc_embedding_type = config.TextCRAN.doc_embedding_type
self.kernel_sizes = config.TextCRAN.kernel_sizes
self.convs = torch.nn.ModuleList()
for kernel_size in self.kernel_sizes:
self.convs.append(
torch.nn.Conv1d(config.embedding.dimension,
config.TextCRAN.num_kernels,
kernel_size,
padding=kernel_size - 1))
self.top_k = self.config.TextCRAN.top_k_max_pooling
hidden_size = len(config.TextCRAN.kernel_sizes) * \
config.TextCRAN.num_kernels * self.top_k
self.rnn = RNN(config.embedding.dimension,
config.TextCRAN.hidden_dimension,
num_layers=config.TextCRAN.num_layers,
batch_first=True,
bidirectional=config.TextCRAN.bidirectional,
rnn_type=config.TextCRAN.rnn_type)
hidden_dimension = config.TextCRAN.hidden_dimension
if config.TextCRAN.bidirectional:
hidden_dimension *= 2
self.sum_attention = SumAttention(
config.TextCRAN.attention_input_dimension,
config.TextCRAN.attention_dimension, config.device)
self.linear = torch.nn.Linear(
config.TextCRAN.attention_input_dimension, len(dataset.label_map))
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def get_parameter_optimizer_dict(self):
params = list()
params.append({'params': self.token_embedding.parameters()})
params.append({'params': self.char_embedding.parameters()})
params.append({'params': self.convs.parameters()})
params.append({'params': self.rnn.parameters()})
params.append({'params': self.linear.parameters()})
return params
def update_lr(self, optimizer, epoch):
"""Update lr
"""
if epoch > self.config.train.num_epochs_static_embedding:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = self.config.optimizer.learning_rate
else:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = 0
def forward(self, batch):
if self.config.feature.feature_names[0] == "token":
embedding = self.token_embedding(batch[cDataset.DOC_TOKEN].to(
self.config.device))
seq_length = batch[cDataset.DOC_TOKEN_LEN].to(self.config.device)
else:
embedding = self.char_embedding(batch[cDataset.DOC_CHAR].to(
self.config.device))
seq_length = batch[cDataset.DOC_CHAR_LEN].to(self.config.device)
# CNN layer
embedding2 = embedding.transpose(1, 2)
pooled_outputs = []
for i, conv in enumerate(self.convs):
#convolution = torch.nn.ReLU(conv(embedding))
convolution = torch.nn.functional.relu(
conv(embedding2)) #[batch,100,n]
pooled = torch.topk(convolution,
self.top_k)[0].view(convolution.size(0), -1)
pooled_outputs.append(pooled)
cnn = torch.cat(pooled_outputs, 1) ##[batch,300]
cnn = cnn.expand(embedding.size(1), cnn.size(0),
cnn.size(1)) # [n,batch,300]
cnn = cnn.transpose(0, 1) # [batch,n,300]
# RNN layer
rnn, output1 = self.rnn(embedding, seq_length) #[batch,n, 300]
output = torch.cat((cnn, rnn), 2) ##[batch,n,600]
if self.doc_embedding_type == DocEmbeddingType.ATTENTION:
doc_embedding = self.sum_attention(output)
out = self.dropout(self.linear(doc_embedding))
return out
import sys sys.path.insert(1, '/Users/jingyuan/Desktop/dga/dga_detection_rnn') from model.rnn import RNN model_path = "model_export/rnn_binary_model_0317_rnn32.h5" export_path = "model_export/rnn_binary_model_0317_rnn32.bin" export_tt_path = "model_export/rnn_binary_model_0317_rnn32_float.txt" export_binary_txt_path = "model_export/rnn_binary_model_0317_rnn32_fixed.txt" model = RNN() model.save_model_txt(export_tt_path, modelPath=model_path) model.save_model_bin(export_path, modelPath=model_path) model.save_model_txt_binary(export_binary_txt_path, modelPath=model_path)
class TextRCNN(Classifier):
"""TextRNN + TextCNN
"""
def __init__(self, dataset, config):
super(TextRCNN, self).__init__(dataset, config)
self.rnn = RNN(config.embedding.dimension,
config.TextRCNN.hidden_dimension,
num_layers=config.TextRCNN.num_layers,
batch_first=True,
bidirectional=config.TextRCNN.bidirectional,
rnn_type=config.TextRCNN.rnn_type)
hidden_dimension = config.TextRCNN.hidden_dimension
if config.TextRCNN.bidirectional:
hidden_dimension *= 2
self.kernel_sizes = config.TextRCNN.kernel_sizes
self.convs = torch.nn.ModuleList()
for kernel_size in self.kernel_sizes:
self.convs.append(
torch.nn.Conv1d(hidden_dimension,
config.TextRCNN.num_kernels,
kernel_size,
padding=kernel_size - 1))
self.top_k = self.config.TextRCNN.top_k_max_pooling
hidden_size = len(config.TextRCNN.kernel_sizes) * \
config.TextRCNN.num_kernels * self.top_k
self.linear = torch.nn.Linear(hidden_size, len(dataset.label_map))
self.dropout = torch.nn.Dropout(p=config.train.hidden_layer_dropout)
def get_parameter_optimizer_dict(self):
params = list()
params.append({'params': self.token_embedding.parameters()})
params.append({'params': self.char_embedding.parameters()})
params.append({'params': self.rnn.parameters()})
params.append({'params': self.convs.parameters()})
params.append({'params': self.linear.parameters()})
return params
def update_lr(self, optimizer, epoch):
"""
"""
if epoch > self.config.train.num_epochs_static_embedding:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = self.config.optimizer.learning_rate
else:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = 0
def forward(self, batch):
if self.config.feature.feature_names[0] == "token":
embedding = self.token_embedding(batch[cDataset.DOC_TOKEN].to(
self.config.device))
seq_length = batch[cDataset.DOC_TOKEN_LEN].to(self.config.device)
else:
embedding = self.char_embedding(batch[cDataset.DOC_CHAR].to(
self.config.device))
seq_length = batch[cDataset.DOC_CHAR_LEN].to(self.config.device)
embedding = self.token_similarity_attention(embedding)
output, _ = self.rnn(embedding, seq_length)
doc_embedding = output.transpose(1, 2)
pooled_outputs = []
for _, conv in enumerate(self.convs):
convolution = F.relu(conv(doc_embedding))
pooled = torch.topk(convolution,
self.top_k)[0].view(convolution.size(0), -1)
pooled_outputs.append(pooled)
doc_embedding = torch.cat(pooled_outputs, 1)
return self.dropout(self.linear(doc_embedding))
def token_similarity_attention(self, output):
# output: (batch, sentence length, embedding dim)
symptom_id_list = [
6, 134, 15, 78, 2616, 257, 402, 281, 14848, 71, 82, 96, 352, 60,
227, 204, 178, 175, 233, 192, 416, 91, 232, 317, 17513, 628, 1047
]
symptom_embedding = self.token_embedding(
torch.LongTensor(symptom_id_list).cuda())
# symptom_embedding: torch.tensor(symptom_num, embedding dim)
batch_symptom_embedding = torch.cat(
[symptom_embedding.view(1, symptom_embedding.shape[0], -1)] *
output.shape[0],
dim=0)
similarity = torch.sigmoid(
torch.bmm(
torch.nn.functional.normalize(output, dim=2),
torch.nn.functional.normalize(batch_symptom_embedding.permute(
0, 2, 1),
dim=2)))
#similarity = torch.bmm(torch.nn.functional.normalize(output, dim=2), torch.nn.functional.normalize(batch_symptom_embedding.permute(0, 2, 1), dim=2))
#similarity = torch.sigmoid(torch.max(similarity, dim=2)[0])
similarity = torch.max(similarity, dim=2)[0]
#similarity = torch.sigmoid(torch.sum(similarity, dim=2))
# similarity: torch.tensor(batch, sentence_len)
similarity = torch.cat([similarity.view(similarity.shape[0], -1, 1)] *
output.shape[2],
dim=2)
# similarity: torch.tensor(batch, batch, sentence_len, embedding dim)
#sentence_embedding = torch.sum(torch.mul(similarity, output), dim=1)
# sentence_embedding: (batch, embedding)
sentence_embedding = torch.mul(similarity, output)
# sentence_embedding: (batch, sentence len, embedding)
return sentence_embedding
y_data_sum = []
#
with open(trainDataPath, 'r') as trainFile:
lines = trainFile.read().split('\n')
for line in lines:
if line.strip('\n').strip('\r').strip(' ') == '':
continue
x_data = []
x = line.strip('\n').strip('\r').strip(' ').split(',')[0]
y = int(line.strip('\n').strip('\r').strip(' ').split(',')[1])
for char in x:
try:
x_data.append(charList[char])
except:
print('unexpected char' + ' : ' + char)
print(line)
x_data.append(0)
x_data_sum.append(x_data)
y_data_sum.append(y)
x_data_sum = np.array(x_data_sum)
y_data_sum = np.array(y_data_sum)
rnn_binary = RNN()
rnn_binary.fit(x_data_sum, y_data_sum, modelPath)
endtime = datetime.datetime.now()
print('=== starttime : ', starttime)
print('=== endtime : ', endtime)
class ZAGRNN(Classifier):
def __init__(self, dataset, config):
assert config.label_embedding.dimension == config.ZAGRNN.gcn_in_features, \
"label embedding dimension should be same as gcn input feature dimension"
super(ZAGRNN, self).__init__(dataset, config)
self.rnn = RNN(
config.embedding.dimension, config.ZAGRNN.hidden_dimension,
num_layers=config.ZAGRNN.num_layers, batch_first=True,
bidirectional=config.ZAGRNN.bidirectional,
rnn_type=config.ZAGRNN.rnn_type)
self.label_wise_attention = LabelWiseAttention(
feat_dim=config.ZAGRNN.hidden_dimension*2 if config.ZAGRNN.bidirectional else config.ZAGRNN.hidden_dimension,
label_emb_dim=config.label_embedding.dimension,
store_attention_score=config.ZAGRNN.store_attention_score)
if config.ZAGRNN.use_gcn:
self.gcn = torch.nn.ModuleList([
GraphConvolution(
in_features=config.ZAGRNN.gcn_in_features,
out_features=config.ZAGRNN.gcn_hidden_features,
bias=True,
act=torch.relu_,
featureless=False,
dropout=config.ZAGRNN.gcn_dropout),
GraphConvolution(
in_features=config.ZAGRNN.gcn_hidden_features,
out_features=config.ZAGRNN.gcn_out_features,
bias=True,
act=torch.relu_,
featureless=False,
dropout=config.ZAGRNN.gcn_dropout)
])
self.doc_out_transform = torch.nn.Sequential(
torch.nn.Linear(
in_features=config.ZAGRNN.hidden_dimension*2 if config.ZAGRNN.bidirectional else config.ZAGRNN.hidden_dimension,
out_features=config.ZAGRNN.gcn_in_features + config.ZAGRNN.gcn_out_features
),
torch.nn.ReLU()
)
def get_parameter_optimizer_dict(self):
params = list()
params.append({'params': self.token_embedding.parameters()})
params.append({'params': self.rnn.parameters()})
params.append({'params': self.label_wise_attention.parameters()})
if self.config.ZAGRNN.use_gcn:
params.append({'params': self.gcn.parameters()})
params.append({'params': self.doc_out_transform.parameters()})
return params
def update_lr(self, optimizer, epoch):
if epoch > self.config.train.num_epochs_static_embedding:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = self.config.optimizer.learning_rate
else:
for param_group in optimizer.param_groups[:2]:
param_group["lr"] = 0
def forward(self, batch):
if self.config.feature.feature_names[0] == "token":
embedding = self.token_embedding(
batch[cDataset.DOC_TOKEN].to(self.config.device))
length = batch[cDataset.DOC_TOKEN_LEN].to(self.config.device)
else:
raise NotImplementedError
doc_embedding, _ = self.rnn(embedding, length)
label_repr = self.label_embedding(
batch[cDataset.DOC_LABEL_ID].to(self.config.device))
attentive_doc_embedding = self.label_wise_attention(doc_embedding, label_repr)
if self.config.ZAGRNN.use_gcn:
label_repr_gcn = label_repr
for gcn_layer in self.gcn:
label_repr_gcn = gcn_layer(label_repr_gcn, batch[cDataset.DOC_LABEL_RELATION].to(self.config.device))
label_repr = torch.cat((label_repr, label_repr_gcn), dim=1)
return torch.sum(self.doc_out_transform(attentive_doc_embedding) * label_repr, dim=-1)
return torch.sum(attentive_doc_embedding * label_repr, dim=-1)