def __init__(self):
self.best_step = sys.maxsize
self.sols = {}
self.solsSum = {}
self.hidden_size = 50
self.lr = 0.01
self.activation_hidden = 'relu6'
self.activation_output = 'sigmoid'
self.activations = {
'sigmoid': nn.Sigmoid(),
'relu': nn.ReLU(),
'relu6': nn.ReLU6(),
'rrelu0103': nn.RReLU(0.1, 0.3),
'rrelu0205': nn.RReLU(0.2, 0.5),
'htang1': nn.Hardtanh(-1, 1),
'htang2': nn.Hardtanh(-2, 2),
'htang3': nn.Hardtanh(-3, 3),
'tanh': nn.Tanh(),
'elu': nn.ELU(),
'selu': nn.SELU(),
'hardshrink': nn.Hardshrink(),
'leakyrelu01': nn.LeakyReLU(0.1),
'leakyrelu001': nn.LeakyReLU(0.01),
'logsigmoid': nn.LogSigmoid(),
'prelu': nn.PReLU(),
}
self.hidden_size_grid = [16, 20, 26, 32, 36, 40, 45, 50, 54]
self.lr_grid = [0.0001, 0.001, 0.005, 0.01, 0.1, 1]
# self.lr_grid = [0.1, .5, 1, 1.5, 2, 3, 5, 10]
# self.activation_hidden_grid = list(self.activations.keys())
# self.activation_output_grid = list(self.activations.keys())
self.grid_search = GridSearch(self)
self.grid_search.set_enabled(False)
def sigmoid_loss(emb_u, emb_v, emb_neg, pos_weight):
logsigmoid = nn.LogSigmoid()
emb_v = emb_v.view(emb_u.size()[0], -1, emb_u.size()[1])
emb_neg = emb_neg.view(emb_u.size()[0], -1, emb_u.size()[1])
pos = logsigmoid((emb_u.unsqueeze(1) * emb_v).sum(dim=2)).mean()
neg = logsigmoid(-(emb_u.unsqueeze(1) * emb_neg).sum(dim=2)).mean()
return - (pos + neg)
def __init__(self,
d_cov,
d_inp,
d_out,
n_inp=-1,
n_out=-1,
n_iters=3,
single_beta=False,
eps=1e-5):
super().__init__()
self.n_iters, self.eps = (n_iters, eps)
self.n_inp_is_fixed, self.n_out_is_fixed = (n_inp > 0, n_out > 0)
one_or_n_inp, one_or_n_out = (max(1, n_inp), max(1, n_out))
self.register_buffer('CONST_one', torch.tensor(1.0))
self.W = nn.Parameter(
torch.empty(one_or_n_inp, one_or_n_out, d_inp, d_out).normal_() /
d_inp)
self.B = nn.Parameter(
torch.zeros(one_or_n_inp, one_or_n_out, d_cov, d_out))
self.beta_use = nn.Parameter(torch.zeros(one_or_n_inp, one_or_n_out))
self.beta_ign = self.beta_use if single_beta else nn.Parameter(
torch.zeros(one_or_n_inp, one_or_n_out))
self.f = nn.Sigmoid()
self.log_f = nn.LogSigmoid()
self.softmax = nn.Softmax(dim=-1)
def __init__(self, eta: float, k: float):
"""Inits an instance of DRO_loss with the given hyperparameters."""
super(DRO_loss, self).__init__()
self.eta = eta
self.k = k
self.logsig = nn.LogSigmoid()
self.relu = nn.ReLU()
def __init__(self, autoregressive_nn, sigmoid_bias=2.0):
super(InverseAutoregressiveFlowStable, self).__init__(cache_size=1)
self.arn = autoregressive_nn
self.sigmoid = nn.Sigmoid()
self.logsigmoid = nn.LogSigmoid()
self.sigmoid_bias = sigmoid_bias
self._cached_log_scale = None
def __init__(self):
super(old_evaluator, self).__init__()
self.bn1 = nn.BatchNorm3d(32)
self.bn2 = nn.BatchNorm3d(128)
self.relu = nn.ReLU(inplace=True)
self.conv3d_1 = nn.Conv3d(in_channels=2, out_channels=32, kernel_size=3, stride=(1, 1, 1), padding=1)
self.conv3d_2 = nn.Conv3d(in_channels=32, out_channels=32, kernel_size=3, stride=(1, 2, 2), padding=1)
self.conv3d_3 = nn.Conv3d(in_channels=32, out_channels=64, kernel_size=3, stride=(1, 1, 1), padding=1)
self.conv3d_4 = nn.Conv3d(in_channels=64, out_channels=64, kernel_size=3, stride=(2, 2, 2), padding=1)
self.conv3d_5 = nn.Conv3d(in_channels=64, out_channels=128, kernel_size=3, stride=(1, 1, 1), padding=1)
self.conv3d_6 = nn.Conv3d(in_channels=128, out_channels=128, kernel_size=3, stride=(1, 1, 1), padding=1)
self.conv3d_7 = nn.Conv3d(in_channels=128, out_channels=64, kernel_size=3, stride=(1, 1, 1), padding=1)
self.conv3d_8 = nn.Conv3d(in_channels=128, out_channels=128, kernel_size=3, stride=(1, 1, 1), padding=1)
self.conv3d_9 = nn.Conv3d(in_channels=128, out_channels=16, kernel_size=5, stride=(1, 1, 1), padding=0)
# self.conv3d_10 = nn.Conv3d(in_channels=16, out_channels=8, kernel_size=3, stride=(1, 1, 1), padding=1)
self.drop3d_1 = nn.Dropout3d(0.25)
self.drop2d_1 = nn.Dropout2d(0.25)
self.drop2d_2 = nn.Dropout2d(0.1)
self.sigmoid = nn.LogSigmoid()
self.fc1 = nn.Linear(76800, 128)
# self.fc1 = nn.Linear(25600, 128)
self.fc2 = nn.Linear(128, 32)
self.fc3 = nn.Linear(32, 1)
def __init__(self):
super(AVNet, self).__init__()
# fusion layers
self.f_conv1 = nn.Conv3d(in_channels=192,
out_channels=512,
kernel_size=[1, 1, 1],
bias=False)
self.f_conv2 = nn.Conv3d(in_channels=512,
out_channels=128,
kernel_size=[1, 1, 1])
self.bn_f = nn.BatchNorm3d(128)
self.relu_f = nn.ReLU(inplace=True)
self.c_res1 = residual_block(128, 128, [3, 3, 3], (1, 1, 1), 1)
self.c_res2 = residual_block(128, 128, [3, 3, 3], (1, 1, 1), 1)
self.c_res3 = residual_block(128, 256, [3, 3, 3], (1, 1, 1), [2, 2, 2])
self.c_res4 = residual_block(256, 256, [3, 3, 3], (1, 1, 1), 1)
self.c_res5 = residual_block(256, 512, [3, 3, 3], (1, 1, 1), [1, 2, 2])
self.c_res6 = residual_block(512, 512, [3, 3, 3], (1, 1, 1), 1)
self.avgpool = nn.AvgPool3d([16, 7, 7])
self.f_fcn = nn.Linear(512, 1)
self.cmm_weights = self.f_fcn.weight
# activations and fc layers
self.sigmoid = nn.Sigmoid()
self.relu = nn.ReLU()
self.logsigmoid = nn.LogSigmoid()
def binary_cross_entropy_with_logits_custom(input, target, weight=None, size_average=None,
reduce=None, reduction='elementwise_mean', pos_weight=None, neg_weight=None):
if size_average is not None or reduce is not None:
reduction = _Reduction.legacy_get_string(size_average, reduce)
if not (target.size() == input.size()):
raise ValueError("Target size ({}) must be the same as input size ({})".format(target.size(), input.size()))
max_val = (-input).clamp(min=0)
log_sigmoid = nn.LogSigmoid()
if pos_weight is None and neg_weight is None: # no positive/negative weight
loss = input - input * target + max_val + ((-max_val).exp() + (-input - max_val).exp()).log()
elif pos_weight is not None and neg_weight is not None: # both positive/negative weight
log_weight = neg_weight + (pos_weight - neg_weight) * target
loss = neg_weight * input - neg_weight * input * target + log_weight * (max_val + ((-max_val).exp() + (-input - max_val).exp()).log())
#loss = - pos_weight * target * log_sigmoid(input) - neg_weight * (1 - target) * log_sigmoid(-input)
elif pos_weight is not None: # only positive weight
log_weight = 1 + (pos_weight - 1) * target
loss = input - input * target + log_weight * (max_val + ((-max_val).exp() + (-input - max_val).exp()).log())
else: # only negative weight
log_weight = neg_weight + (1 - neg_weight) * target
loss = neg_weight * input - neg_weight * input * target + log_weight * (max_val + ((-max_val).exp() + (-input - max_val).exp()).log())
#loss = - target * log_sigmoid(input) - neg_weight * (1 - target) * log_sigmoid(-input)
if weight is not None:
loss = loss * weight
if reduction == 'none':
return loss
elif reduction == 'elementwise_mean':
return loss.mean()
else:
return loss.sum()
def create_str_to_activations_converter(self):
"""Creates a dictionary which converts strings to activations"""
str_to_activations_converter = {
"elu": nn.ELU(),
"hardshrink": nn.Hardshrink(),
"hardtanh": nn.Hardtanh(),
"leakyrelu": nn.LeakyReLU(),
"logsigmoid": nn.LogSigmoid(),
"prelu": nn.PReLU(),
"relu": nn.ReLU(),
"relu6": nn.ReLU6(),
"rrelu": nn.RReLU(),
"selu": nn.SELU(),
"sigmoid": nn.Sigmoid(),
"softplus": nn.Softplus(),
"logsoftmax": nn.LogSoftmax(),
"softshrink": nn.Softshrink(),
"softsign": nn.Softsign(),
"tanh": nn.Tanh(),
"tanhshrink": nn.Tanhshrink(),
"softmin": nn.Softmin(),
"softmax": nn.Softmax(dim=1),
"none": None
}
return str_to_activations_converter
def __init__(self, input_size, output_size):
super(Controller, self).__init__()
# --- Mappings ---
self.fc = nn.Linear(input_size, output_size)
self.softmax = nn.Softmax(dim=1)
self.log_sig = nn.LogSigmoid()
def __init__(self, alpha=1.0):
super().__init__()
self.activations = [
nn.ELU(),
nn.Hardshrink(),
nn.Hardtanh(),
nn.LeakyReLU(),
nn.LogSigmoid(),
nn.ReLU(),
nn.PReLU(),
nn.SELU(),
nn.CELU(),
nn.Sigmoid(),
nn.Softplus(),
nn.Softshrink(),
nn.Softsign(),
nn.Tanh(),
nn.Tanhshrink()
]
self.P = [
torch.nn.Parameter(torch.randn(1, requires_grad=True))
for _ in self.activations
]
for activation, param in zip(self.activations, self.P):
activation_name = str(activation).split("(")[0]
self.add_module(name=activation_name, module=activation)
self.register_parameter(name=activation_name + "p", param=param)
def __init__(self):
super(ResGenerator2D, self).__init__()
self.in_conv = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=(2, 2), stride=1)
self.in_bn = nn.BatchNorm2d(num_features=16)
self.in_relu = nn.ReLU()
self.resblock1 = ResBlock2D(in_channel=16, out_channel=32)
self.resblock2 = ResBlock2D(in_channel=32, out_channel=64)
self.resblock3 = ResBlock2D(in_channel=64, out_channel=128)
self.latent_conv = LatentConvLayer(in_channel=128, out_channel=256)
self.latent_transconv = LatentTransConvLayer(in_channel=256, out_channel=128)
self.up1 = ResUpSample2D(in_channel=256, out_channel=128)
self.up2 = ResUpSample2D(in_channel=256, out_channel=64)
self.up3 = ResUpSample2D(in_channel=128, out_channel=64)
self.up4 = ResUpSample2D(in_channel=128, out_channel=32)
self.up5 = ResUpSample2D(in_channel=64, out_channel=32)
self.up6 = ResUpSample2D(in_channel=64, out_channel=16)
self.final_transconv = nn.ConvTranspose2d(in_channels=32,
out_channels=32,
kernel_size=(2, 2),
stride=1)
self.final_bn = nn.BatchNorm2d(num_features=32)
self.final_relu = nn.ReLU()
self.final_downsample = nn.Conv2d(in_channels=32, out_channels=1, kernel_size=1)
self.final_activation = nn.LogSigmoid()
self.init_weights()
def forward(self, input, target):
log_probs = nn.LogSigmoid()(input)
with torch.no_grad():
true_dist = torch.zeros_like(log_probs)
true_dist.fill_(self.smoothing / (self.classes - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.conf)
return torch.mean(torch.sum(-true_dist * log_probs, dim=self.dim))
def __init__(self, config):
super(BCMModel, self).__init__(config)
self.train_objective = config['train_objective']
# Dictionary of additional data loaders
# for evaluation on DPR and WSC
self.eval_loaders = None
self.logsoftmax = nn.LogSoftmax(dim=2)
self.softmax = nn.Softmax(dim=1)
self.log_sigmoid = nn.LogSigmoid()
if self.config['sem_sim_type'] == 'cosine':
self.cosine_sim = nn.CosineSimilarity(dim=1)
elif self.config['sem_sim_type'] == 'additive':
self.sem_sim = nn.Sequential(
nn.Linear(2 * self.d_model, self.d_model), nn.Tanh(),
nn.Linear(self.d_model, 1))
else:
raise NotImplementedError
if self.config['use_mlp_head']:
# Add an MLP on top of the two candidate attention sums for MAS
# num_attns is how many attn heads there are in all layers
num_attns = self.encoder.config.num_attention_heads * \
self.encoder.config.num_hidden_layers
self.mlp_head = nn.Sequential(
nn.Linear(2 * num_attns, 2 * num_attns), nn.Tanh(),
nn.Linear(2 * num_attns, 2 * num_attns), nn.Tanh(),
nn.Linear(2 * num_attns, 2))
def basic_loss(self, fx, convex):
if convex:
negative_logistic = nn.LogSigmoid()
return -negative_logistic(fx)
else:
sigmoid = nn.Sigmoid()
return sigmoid(-fx)
def parse_activation(act):
if act is None:
return lambda x: x
act, kwargs = parse_str(act)
if act == 'sigmoid': return nn.Sigmoid(**kwargs)
if act == 'tanh': return nn.Tanh(**kwargs)
if act == 'relu': return nn.ReLU(**kwargs, inplace=True)
if act == 'relu6': return nn.ReLU6(**kwargs, inplace=True)
if act == 'elu': return nn.ELU(**kwargs, inplace=True)
if act == 'selu': return nn.SELU(**kwargs, inplace=True)
if act == 'prelu': return nn.PReLU(**kwargs)
if act == 'leaky_relu': return nn.LeakyReLU(**kwargs, inplace=True)
if act == 'threshold': return nn.Threshold(**kwargs, inplace=True)
if act == 'hardtanh': return nn.Hardtanh(**kwargs, inplace=True)
if act == 'log_sigmoid': return nn.LogSigmoid(**kwargs)
if act == 'softplus': return nn.Softplus(**kwargs)
if act == 'softshrink': return nn.Softshrink(**kwargs)
if act == 'tanhshrink': return nn.Tanhshrink(**kwargs)
if act == 'softmin': return nn.Softmin(**kwargs)
if act == 'softmax': return nn.Softmax(**kwargs)
if act == 'softmax2d': return nn.Softmax2d(**kwargs)
if act == 'log_softmax': return nn.LogSoftmax(**kwargs)
raise ValueError(f'unknown activation: {repr(act)}')
def __init__(self, n_usr, n_itm, k, gamma=[1,1,1,1], alpha=0.0, norm=False):
super(BFM, self).__init__()
# parameters
self.n = n_usr
self.m = n_itm
self.k = k
# biases
# self.w_0 = nn.Parameter(torch.randn(1))
self.w_0 = nn.Parameter(torch.zeros(1))
self.w_bias = nn.Parameter(torch.randn(n_usr+2*n_itm, 1))
# latent vectors (input must one-hot vector)
# self.u_V = nn.Parameter(torch.normal(0, 1, size=(n, k)))
# self.t_V = nn.Parameter(torch.normal(0, 1, size=(m, k)))
# self.b_V = nn.Parameter(torch.normal(0, 1, size=(m, k)))
self.u_V = nn.Parameter(torch.randn(n_usr, k))
# self.t_V = nn.Parameter(torch.randn(n_itm, k))
self.b_V = nn.Parameter(torch.randn(n_itm, k))
self.t_V = self.b_V
# sigmoid
# self.sigmoid = nn.Sigmoid()
self.lnsigmoid = nn.LogSigmoid()
# hyper parameter
self.alpha = alpha
self.gamma = gamma
self.norm = norm
def __init__(self):
super(Cnn, self).__init__()
self.convolution = nn.Sequential(
nn.Conv2d(2, 16, kernel_size=3, stride=2),
nn.ReLU(),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(16, 32, kernel_size=3, stride=2),
nn.ReLU(),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(32, 64, kernel_size=3, stride=1),
nn.ReLU(),
nn.MaxPool2d(kernel_size=2, stride=2, padding=1),
nn.Conv2d(64, 64, kernel_size=3, stride=1),
nn.ReLU(),
nn.MaxPool2d(kernel_size=2, stride=1),
nn.Conv2d(64, 128, kernel_size=2, stride=1),
nn.ReLU(),
)
self.fc = nn.Sequential(
nn.Linear(128, 128),
nn.LogSigmoid(),
nn.Dropout(p=.5),
nn.Linear(128, 64),
nn.ReLU(),
nn.Dropout(p=.5),
nn.Linear(64, 2),
)
def __init__(self, input_size, hidden_size, output_size):
super(Encoder, self).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(input_size, hidden_size)
self.gru = nn.GRU(hidden_size, hidden_size)
self.out = nn.Linear(hidden_size, output_size)
self.sigmoid = nn.LogSigmoid()
def __init__(self,
docvecs,
item_tags,
embeddings_shape=(20, 20),
hidden_dimension=20,
out_dimension=10):
super(SkipGramModel, self).__init__()
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
self.docvecs = torch.Tensor(docvecs).to(self.device)
self.item_padding_index = docvecs.shape[0]
padding_row = torch.Tensor([0] * docvecs.shape[1]).to(self.device)
padding_row = padding_row.reshape(1, docvecs.shape[1]).to(self.device)
self.docvecs = torch.cat((self.docvecs, padding_row), 0)
self.tag_embeddings = nn.Embedding(embeddings_shape[0]+1,
embeddings_shape[1],
padding_idx=embeddings_shape[0]).to(self.device)
self.item_tags = torch.LongTensor(item_tags).to(self.device)
padding_row = torch.LongTensor([self.tag_embeddings.padding_idx] *
self.item_tags.shape[1]).to(self.device)
padding_row = padding_row.reshape(1, self.item_tags.shape[1])
self.item_tags = torch.cat((self.item_tags, padding_row), 0)
self.linear1 = nn.Linear(embeddings_shape[1]+docvecs.shape[1], hidden_dimension).to(self.device)
self.linear2 = nn.Linear(hidden_dimension, out_dimension).to(self.device)
self.logsigmoid = nn.LogSigmoid().to(self.device)
def __init__(self, args, pretrained_embed=None):
super(modeler_asp2vec, self).__init__()
self.num_aspects = args.num_aspects
self.num_nodes = args.num_nodes
self.logsigmoid = nn.LogSigmoid()
self.dim = args.dim
self.device = args.device
self.aspect_embedding = nn.Embedding(args.num_nodes * args.num_aspects,
args.dim)
self.center_embedding = nn.Embedding(args.num_nodes, args.dim)
self.pooling = args.pooling
self.isInit = args.isInit
self.isReg = args.isReg
if self.isReg:
self.reg_coef = args.reg_coef
self.threshold = args.threshold
self.isSoftmax = args.isSoftmax
if self.isSoftmax:
self.isGumbelSoftmax = args.isGumbelSoftmax
self.isNormalSoftmax = args.isNormalSoftmax
if self.isGumbelSoftmax:
self.tau_gumbel = args.tau_gumbel
self.isHard = args.isHard
if self.isInit:
self.init_weights(pretrained_embed=pretrained_embed)
else:
self.init_weights()
def __init__(self, inp, hid, out, num_layer=1, activation="Tanh"):
super(FFNN, self).__init__()
self.num_layer = num_layer
self.V = nn.Linear(inp, hid)
self.g = nn.Tanh()
self.W = nn.Linear(hid, out)
#self.softmax = nn.Softmax(dim=0)
# Initialize weights according to the Xavier Glorot formula
nn.init.xavier_uniform(self.V.weight)
nn.init.xavier_uniform(self.W.weight)
if num_layer == 0:
self.V0 = nn.Linear(inp, out)
if num_layer == 2:
self.W = nn.Linear(hid, hid)
self.g1 = nn.Tanh()
self.W1 = nn.Linear(hid, out)
if activation == "ReLU":
self.g = nn.ReLU()
if activation == "LogSigmoid":
self.g = nn.LogSigmoid()
if activation == "RReLU":
self.g = nn.RReLU()
if activation == "Softplus":
self.g = nn.Softplus()
def __init__(self):
super(MyLeNet, self).__init__()
self.choice_conv = nn.ModuleDict({
'conv1': nn.Conv2d(1, 10, 3, 1, 2),
'conv2': nn.Conv2d(10, 10, 3, 1),
'conv3': nn.Conv2d(1, 10, 3, 1),
'conv4': nn.Conv2d(1, 10, 5, 1),
'conv5': nn.Conv2d(1, 6, 5, 1), ##c1
'conv6': nn.Conv2d(6, 16, 5, 1), ##c2
'conv7': nn.Conv2d(16, 120, 5, 1) ##c3
})
self.choice_pooling = nn.ModuleDict({
'maxpooling1': nn.MaxPool2d(2, 2),
#'maxpooling2':nn.MaxPool2d(1,1),
'avgpooling1': nn.AvgPool2d(2, 2),
})
self.choice_activations = nn.ModuleDict({
'rule': nn.ReLU(),
'leakyrule': nn.LeakyReLU(),
'logsigmoid': nn.LogSigmoid(),
'prelu': nn.PReLU(),
'sigmoid': nn.Sigmoid(),
'tanh': nn.Tanh(),
'softmin': nn.Softmin(),
'softmax': nn.Softmax(),
'softmax2': nn.Softmax2d()
})
self.choice_fc = nn.ModuleDict({
'f1': nn.Linear(120, 84),
'f2': nn.Linear(84, 10)
})
def __init__(self,
d_cov,
d_inp,
d_out,
n_inp=-1,
n_out=-1,
n_iters=3,
single_beta=False,
p_model='gaussian',
eps=1e-5):
super().__init__()
assert p_model in ['gaussian',
'skm'], 'Unrecognized value for p_model.'
self.n_iters, self.p_model, self.eps = (n_iters, p_model, eps)
self.n_inp_is_fixed, self.n_out_is_fixed = (n_inp > 0, n_out > 0)
one_or_n_inp, one_or_n_out = (max(1, n_inp), max(1, n_out))
self.register_buffer('CONST_one', torch.tensor(1.0))
self.W = nn.Parameter(
torch.empty(one_or_n_inp, one_or_n_out, d_inp, d_out).normal_() /
d_inp)
self.B = nn.Parameter(
torch.zeros(one_or_n_inp, one_or_n_out, d_cov, d_out))
if not self.n_out_is_fixed:
self.B_brk = nn.Parameter(torch.zeros(1, d_cov, d_out))
self.beta_use = nn.Parameter(torch.zeros(one_or_n_inp, one_or_n_out))
self.beta_ign = self.beta_use if single_beta else nn.Parameter(
torch.zeros(one_or_n_inp, one_or_n_out))
self.f, self.log_f = (nn.Sigmoid(), nn.LogSigmoid())
self.softmax, self.log_softmax = (nn.Softmax(dim=-1),
nn.LogSoftmax(dim=-1))
def __init__(self,
vocab_size,
emb_dim,
weights_neg,
n_negs=7,
context_size=4,
batch_size=4096):
"""
Модель для обучения word2vec с использованием negative sample
:param vocab_size: размер словаря
:param emb_dim: размерность эмбеддинга
:param weights_neg: распределение вероятностей для выбора negative sample
:param n_negs: кол-во негативных контекстов
:param context_size: 2*window_size или размер контекста
:param batch_size: размер батча
"""
super(model, self).__init__()
self.target_emb = nn.Embedding(vocab_size, emb_dim)
self.context_emb = nn.Embedding(vocab_size, emb_dim)
self.weights_neg = weights_neg
self.batch_size = batch_size
self.context_size = context_size
self.n_negs = n_negs
self.log_sigmoid = nn.LogSigmoid()
self.device = t.device("cuda")
def __init__(self, size_layer, num_class=2, activation='relu'):
"""Multilayer Perceptrons as a decoder
Args:
size_layer: list of int, define the size of MLP layers
num_class: int, num of class in output, should be 2 or the last layer's size
activation: str or function, the activation function for hidden layers
"""
super(MLP, self).__init__()
self.hiddens = nn.ModuleList()
self.output = None
for i in range(1, len(size_layer)):
if i + 1 == len(size_layer):
self.output = nn.Linear(size_layer[i-1], size_layer[i])
else:
self.hiddens.append(nn.Linear(size_layer[i-1], size_layer[i]))
if num_class == 2:
self.out_active = nn.LogSigmoid()
elif num_class == size_layer[-1]:
self.out_active = nn.LogSoftmax(dim=1)
else:
raise ValueError("should set output num_class correctly: {}".format(num_class))
actives = {
'relu': nn.ReLU(),
'tanh': nn.Tanh()
}
if activation in actives:
self.hidden_active = actives[activation]
elif isinstance(activation, callable):
self.hidden_active = activation
else:
raise ValueError("should set activation correctly: {}".format(activation))
def __init__(self,
features,
input_dim=64,
hidden_dim=64,
output_dim=64,
num_layers=2,
dropout_rate=0.1):
super(graphsage, self).__init__()
self.features = features
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.output_dim = output_dim
self.num_layers = num_layers
#hypermeter
self.dropout_rate = dropout_rate
#build weight
self.linear_list = nn.ModuleList()
self.dropout_list = nn.ModuleList()
self.linear_list.append(nn.Linear(input_dim, hidden_dim))
self.dropout_list.append(nn.Dropout(p=dropout_rate))
for i in range(num_layers - 2):
self.linear_list.append(nn.Linear(hidden_dim, hidden_dim))
self.dropout_list.append(nn.Dropout(p=dropout_rate))
self.linear_list.append(nn.Linear(hidden_dim, output_dim))
self.dropout_list.append(nn.Dropout(p=dropout_rate))
self.transform = nn.LogSigmoid()
def __init__(self,
vocab_size,
embedding_size,
w2v_init=True,
w2v_embeds=None):
super(SkipGram, self).__init__()
self.vocab_size = vocab_size
self.embedding_size = embedding_size
self.i_embedding = nn.Embedding(vocab_size, embedding_size)
self.o_embedding = nn.Embedding(vocab_size, embedding_size)
if w2v_embeds:
# Initialising weights to word2vec Google News 300 pretrained
# (Assume w2v_embeds are given)
print(
f'Initialising input embeddings with pre-trained word embeddings in tensor of size {w2v_embeds.shape}'
)
self.i_embedding.weight.data.copy_(w2v_embeds)
# if w2v_path is None:
# self.i_embedding.weight.data.copy_(get_word2vec_vectors())
# else:
# self.i_embedding.weight.data.copy_(get_word2vec_vectors(w2v_path))
# Sanity check: does first vector match
# the first vector in Word2Vec?
# print('First word vector')
# print(self.i_embedding(torch.tensor(0)))
self.logsigmoid = nn.LogSigmoid()
def __init__(self):
super(NNActivationModule, self).__init__()
self.activations = nn.ModuleList([
nn.ELU(),
nn.Hardshrink(),
nn.Hardsigmoid(),
nn.Hardtanh(),
nn.Hardswish(),
nn.LeakyReLU(),
nn.LogSigmoid(),
# nn.MultiheadAttention(),
nn.PReLU(),
nn.ReLU(),
nn.ReLU6(),
nn.RReLU(),
nn.SELU(),
nn.CELU(),
nn.GELU(),
nn.Sigmoid(),
nn.SiLU(),
nn.Mish(),
nn.Softplus(),
nn.Softshrink(),
nn.Softsign(),
nn.Tanh(),
nn.Tanhshrink(),
# nn.Threshold(0.1, 20),
nn.GLU(),
nn.Softmin(),
nn.Softmax(),
nn.Softmax2d(),
nn.LogSoftmax(),
# nn.AdaptiveLogSoftmaxWithLoss(),
])
def __init__(self,w_init):
super().__init__()
self.w_init = w_init
self.initializations = {
'uniform': init.uniform_,
'normal': init.normal_,
'dirac': init.dirac_,
'xavier_uniform': init.xavier_uniform_,
'xavier_normal': init.xavier_normal_,
'kaiming_uniform': init.kaiming_uniform_,
'kaiming_normal': init.kaiming_normal_,
'orthogonal': init.orthogonal_,
'ones' : init.ones_
}
self.activations = nn.ModuleDict([
['ELU', nn.ELU()],
['ReLU', nn.ReLU()],
['Tanh', nn.Tanh()],
['LogSigmoid', nn.LogSigmoid()],
['LeakyReLU', nn.LeakyReLU()],
['SELU', nn.SELU()],
['CELU', nn.CELU()],
['GELU', nn.GELU()],
['Sigmoid', nn.Sigmoid()],
['Softmax', nn.Softmax()],
['LogSoftmax', nn.LogSoftmax()]
])
