def __init__(self, dim, dropout=0):
super(AdditiveAttention, self).__init__()
self.w_attention_matrix = init_tensor(torch.empty(dim, dim))
self.u_attention_matrix = init_tensor(torch.empty(dim, dim))
self.v_attention_vector = init_tensor(torch.empty(dim, 1))
self.dropout = torch.nn.Dropout(p=dropout)
def __init__(self, dataset, config):
super(RegionEmbedding, self).__init__(dataset, config)
self.region_size = config.embedding.region_size
self.radius = int(self.region_size / 2)
self.linear = torch.nn.Linear(config.embedding.dimension,
len(dataset.label_map))
init_tensor(self.linear.weight, init_type=InitType.XAVIER_UNIFORM)
init_tensor(self.linear.bias, init_type=InitType.UNIFORM, low=0, high=0)
def __init__(self, dataset, config):
super(AttentiveConvNet, self).__init__(dataset, config)
self.attentive_conv_net_type = config.AttentiveConvNet.type
self.attention_type = config.AttentiveConvNet.attention_type
self.dim = config.embedding.dimension
self.attention_dim = self.dim
self.margin_size = config.AttentiveConvNet.margin_size
assert self.margin_size % 2 == 1, \
"AttentiveConvNet margin size should be odd!"
self.radius = int(self.margin_size / 2)
if self.attentive_conv_net_type == AttentiveConvNetType.ADVANCED:
self.attention_dim *= 2
self.x_context_highway = self.get_highway(self.dim,
self.margin_size)
self.x_self_highway = self.get_highway(self.dim, 1)
self.a_context_highway = self.get_highway(self.dim,
self.margin_size)
self.a_self_highway = self.get_highway(self.dim, 1)
self.beneficiary_highway = self.get_highway(self.dim, 1)
if self.attention_type == AttentionType.DOT:
self.dot_product_attention = DotProductAttention(1.0)
elif self.attention_type == AttentionType.BILINEAR:
self.bilinear_matrix = init_tensor(
torch.empty(self.attention_dim,
self.attention_dim)).to(config.device)
self.dot_product_attention = DotProductAttention(1.0)
elif self.attention_type == AttentionType.ADDITIVE_PROJECTION:
self.additive_projection = AdditiveAttention2D(self.attention_dim)
else:
raise TypeError("Unsupported AttentionType: %s." %
self.attention_type)
self.attentive_conv = init_tensor(
torch.empty(self.attention_dim, self.dim)).to(config.device)
self.x_conv = torch.nn.Sequential(
torch.nn.Conv1d(self.dim,
self.dim,
self.margin_size,
padding=self.radius), torch.nn.Tanh())
self.bias = torch.zeros([self.dim]).to(config.device)
self.hidden_size = config.AttentiveConvNet.hidden_size
self.hidden1_matrix = init_tensor(
torch.empty(self.dim, self.hidden_size)).to(config.device)
self.hidden2_matrix = init_tensor(
torch.empty(self.hidden_size, self.hidden_size)).to(config.device)
self.linear = torch.nn.Linear(self.dim + 2 * self.hidden_size,
len(dataset.label_map))
def __init__(self,
input_dimension,
attention_dimension,
device,
dropout=0):
super(SumAttention, self).__init__()
self.attention_matrix = \
torch.nn.Linear(input_dimension, attention_dimension).to(device)
self.attention_vector = torch.nn.Linear(attention_dimension,
1,
bias=False).to(device)
init_tensor(self.attention_matrix.weight)
init_tensor(self.attention_vector.weight)
self.dropout = torch.nn.Dropout(p=dropout)
def __init__(self,
dict_map,
embedding_dim,
name,
config,
padding_idx=None,
pretrained_embedding_file=None,
mode=EmbeddingProcessType.FLAT,
dropout=0,
init_type=InitType.XAVIER_UNIFORM,
low=0,
high=1,
mean=0,
std=1,
activation_type=ActivationType.NONE,
fan_mode=FAN_MODE.FAN_IN,
negative_slope=0,
model_mode=ModeType.TRAIN):
super(Embedding, self).__init__()
self.logger = Logger(config)
self.dropout = torch.nn.Dropout(p=dropout)
self.mode = mode
if self.mode == EmbeddingProcessType.FLAT:
self.embedding = torch.nn.Embedding(len(dict_map),
embedding_dim,
padding_idx=padding_idx)
else:
self.embedding = torch.nn.EmbeddingBag(len(dict_map),
embedding_dim,
mode=mode)
embedding_lookup_table = init_tensor(tensor=torch.empty(
len(dict_map), embedding_dim),
init_type=init_type,
low=low,
high=high,
mean=mean,
std=std,
activation_type=activation_type,
fan_mode=fan_mode,
negative_slope=negative_slope)
if model_mode == ModeType.TRAIN and \
pretrained_embedding_file is not None and \
pretrained_embedding_file != "":
self.load_pretrained_embedding(embedding_lookup_table, dict_map,
embedding_dim, name,
pretrained_embedding_file)
if padding_idx is not None:
embedding_lookup_table[padding_idx] = 0.0
self.embedding.weight.data.copy_(embedding_lookup_table)