def __init__(self, config):
super(MRR_in_N, self).__init__(config)
assert self.AN <= self.N
self.name = "MRR_in_{}".format(self.AN)
logger.info(
utils.generate_module_info(self.name, "N", self.N, "AN", self.AN,
"skip", self.skip))
def __init__(self, config):
super(KLDivLoss, self).__init__()
self.size_average = config[
"size_average"] if "size_average" in config else False
self.reduce = config["reduce"] if "reduce" in config else True
self.reduction = config[
"reduction"] if "reduction" in config else "elementwise_mean"
# parameters for smoothing label
self.do_label_smoothing = config[
"do_label_smoothing"] if "do_label_smoothing" in config else False
self.ratio = config["ratio"] if "ratio" in config else 2
self.epsilon = config["epsilon"] if "epsilon" in config else 0.1
self.loss_module = torch.nn.KLDivLoss(size_average=self.size_average,
reduce=self.reduce,
reduction=self.reduction)
self.name = config[
"name"] if "name" in config else " KL Divergence Loss"
logger.info(
utils.generate_module_info(self.name, "size_average",
self.size_average, "reduce",
self.reduce, "reduction",
self.reduction))
def __init__(self, config):
super(AttentiveModule, self).__init__()
assert "x_dim" in config and "y_dim" in config
# Attention layer
attention_config = deepcopy(config)
attention_config["name"] = "Attention"
self.attention = Attention(attention_config)
self.is_layer_norm = config[
"is_layer_norm"] if "is_layer_norm" in config else True
if self.is_layer_norm:
# Attention layer norm
self.attention_layer_norm = nn.LayerNorm([config["y_dim"]],
eps=1e-6)
# self.attention_layer_norm = LayerNorm(
# {"name": "Attention_layer_norm", "parameter_shape": [config["x_dim"]], "axis": [-1]})
# FFN layer norm
self.ffn_layer_norm = nn.LayerNorm([config["y_dim"]], eps=1e-6)
# self.ffn_layer_norm = LayerNorm(
# {"name": "FFN_layer_norm", "parameter_shape": [config["x_dim"]], "axis": [-1]})
self.ffn = FFN({
"name": "FFN",
"input_dim": config["y_dim"],
"out_dim_0": config["y_dim"],
"out_dim_1": config["y_dim"]
})
self.name = config["name"] if "name" in config else "AttentiveModule"
logger.info(
utils.generate_module_info(self.name, "is_layer_norm",
self.is_layer_norm))
def __init__(self, config):
super(Recall_N_at_K, self).__init__(config)
self.K = self.config["k"] if "k" in self.config else 1
assert self.AN >= self.K and self.AN <= self.N
self.name = "R{}@{}".format(self.AN, self.K)
logger.info(
utils.generate_module_info(self.name, "N", self.N, "AN", self.AN,
"K", self.K, "skip", self.skip))
def __init__(self, config):
super(MarginRankingLoss, self).__init__()
self.margin = config["margin"] if "margin" in config else 0.0
self.reduction = config[
"reduction"] if "reduction" in config else "elementwise_mean"
self.sigmoid = torch.nn.Sigmoid()
self.loss_module = torch.nn.MarginRankingLoss(margin=self.margin,
reduction=self.reduction)
self.name = config[
"name"] if "name" in config else "Margin Ranking Loss"
logger.info(
utils.generate_module_info(self.name, "margin", self.margin,
"reduction", self.reduction))
def __init__(self, config):
super(LayerNorm, self).__init__()
assert "parameter_shape" in config and isinstance(
config["parameter_shape"], list or tuple)
self.parameter_shape = config["parameter_shape"]
assert "axis" in config and isinstance(config["axis"], list or tuple)
self.axis = config["axis"]
self.scale = nn.Parameter(torch.ones(self.parameter_shape))
self.bias = nn.Parameter(torch.zeros(self.parameter_shape))
self.name = config["name"] if "name" in config else "Layer Norm"
logger.info(
utils.generate_module_info(self.name, "parameter_shape",
self.parameter_shape, "axis",
self.axis))
def __init__(self, config):
# lr_scheduler
if "lr_scheduler" in config["params"]:
self.lr_scheduler = LrScheduler(config["params"]["lr_scheduler"])
config["params"].pop("lr_scheduler")
else:
self.lr_scheduler = LrScheduler()
# optimizer
self.optimizer = utils.name2function(config["function"])
self.optimizer_params = config["params"]
if isclass(self.optimizer) and issubclass(self.optimizer,
torch.optim.Optimizer):
self.name = config["function"].strip().rsplit('.', maxsplit=1)[-1]
logger.info(
utils.generate_module_info(self.name, **self.optimizer_params))
def __init__(self, config):
super(Attention, self).__init__()
assert "x_dim" in config and "y_dim" in config
self.x_dim = config["x_dim"]
self.y_dim = config["y_dim"]
self.drop_prob = config["drop_prob"] if "drop_prob" in config else None
self.bilinear_matrix = nn.Linear(in_features=self.x_dim,
out_features=self.y_dim,
bias=False)
if self.drop_prob is not None:
self.dropout = nn.Dropout(p=self.drop_prob)
self.name = config["name"] if "name" in config else "Attention"
logger.info(
utils.generate_module_info(self.name, "x_dim", self.x_dim, "y_dim",
self.y_dim, "drop_prob",
self.drop_prob))
def SGDOptimizer(params, **config):
lr = config["lr"] if "lr" in config else 0.001
momentum = config["momentum"] if "momentum" in config else 0
weight_decay = config["weight_decay"] if "weight_decay" in config else 0
dampening = config["dampening"] if "dampening" in config else 0
nesterov = config["nesterov"] if "nesterov" in config else False
name = "SGD Optimizer"
logger.info(
utils.generate_module_info(name, "lr", lr, "momentum", momentum,
"weight_decay", weight_decay, "dampening",
dampening, "nesterov", nesterov))
return torch.optim.SGD(params,
lr=lr,
momentum=momentum,
weight_decay=weight_decay,
dampening=dampening,
nesterov=nesterov)
def AdamOptimizer(params, **config):
lr = config["lr"] if "lr" in config else 0.001
betas = config["betas"] if "betas" in config else (0.9, 0.999)
eps = config["eps"] if "eps" in config else 1e-08
weight_decay = config["weight_decay"] if "weight_decay" in config else 0
amsgrad = config["amsgrad"] if "amsgrad" in config else False
name = "Adam Optimizer"
logger.info(
utils.generate_module_info(name, "lr", lr, "betas", betas, "eps", eps,
"weight_decay", weight_decay, "amsgrad",
amsgrad))
return torch.optim.Adam(params,
lr=lr,
betas=betas,
eps=eps,
weight_decay=weight_decay,
amsgrad=amsgrad)
def __init__(self, config):
super(PositionEncoder, self).__init__()
assert "lambda_size" in config
self.lambda_size = config["lambda_size"]
self.min_timescale = config[
"min_timescale"] if "min_timescale" in config else 1.0
self.max_timescale = config[
"max_timescale"] if "max_timescale" in config else 1.0e4
self.fill_value = config["fill_value"] if "fill_value" in config else 0
self._lambda = nn.Parameter(
torch.full((self.lambda_size, 1), self.fill_value))
self.name = config["name"] if "name" in config else "Position Encoder"
logger.info(
utils.generate_module_info(self.name, "lambda_size",
self.lambda_size, "min_timescale",
self.min_timescale, "max_timescale",
self.max_timescale, "fill_value",
self.fill_value))
def __init__(self, config):
super(MultiHeadedAttentiveModule, self).__init__()
assert "x_dim" in config and "y_dim" in config and "head_num" in config
# Attention layer
attention_config = deepcopy(config)
attention_config["name"] = "Attention"
self.attention = Attention(attention_config)
self.input_dim = config["x_dim"]
self.output_dim = config["y_dim"]
self.head_num = config["head_num"]
assert self.input_dim % self.head_num == 0
self.sub_input_dim = self.input_dim // self.head_num
self.input_linears = utils.clones(
nn.Linear(self.input_dim, self.output_dim), 3)
self.output_linear = nn.Linear(self.output_dim, self.output_dim)
self.is_layer_norm = config[
"is_layer_norm"] if "is_layer_norm" in config else True
if self.is_layer_norm:
# Attention layer norm
self.attention_layer_norm = nn.LayerNorm([self.output_dim],
eps=1e-6)
# FFN layer norm
self.ffn_layer_norm = nn.LayerNorm([self.output_dim], eps=1e-6)
self.ffn = FFN({
"name": "FFN",
"input_dim": self.output_dim,
"out_dim_0": self.output_dim,
"out_dim_1": self.output_dim
})
self.name = config[
"name"] if "name" in config else "MultiHeadAttentiveModule"
logger.info(
utils.generate_module_info(self.name, "head_num", self.head_num,
"input_dim", self.input_dim,
"output_dim", self.output_dim,
"is_layer_norm", self.is_layer_norm))
def __init__(self, config=None):
config = deepcopy(config)
if config == None or not isinstance(config, dict):
config = {
"function": "torch.optim.lr_scheduler.StepLR",
"params": {
"step_size": 1000000,
"gamma": 1
}
}
self.lr_scheduler = utils.name2function(config["function"])
self.lr_scheduler_params = config["params"]
if isclass(self.lr_scheduler) and issubclass(
self.lr_scheduler,
(torch.optim.lr_scheduler._LRScheduler,
torch.optim.lr_scheduler.ReduceLROnPlateau)):
self.name = config["function"].strip().rsplit(
'.', maxsplit=1)[-1] + " Scheduler"
logger.info(
utils.generate_module_info(self.name,
**self.lr_scheduler_params))
def __init__(self, config):
super(CrossEntropyLoss, self).__init__()
if "weight" in config and isinstance(
config["weight"], (list, tuple, np.ndarray, torch.Tensor)):
self.weight = torch.Tensor(config["weight"])
else:
self.weight = None
self.ignore_index = config[
"ignore_index"] if "ignore_index" in config else -100
self.reduction = config[
"reduction"] if "reduction" in config else "elementwise_mean"
self.loss_module = torch.nn.CrossEntropyLoss(
weight=self.weight,
ignore_index=self.ignore_index,
reduction=self.reduction)
self.name = config["name"] if "name" in config else "Cross Entropy Loss"
logger.info(
utils.generate_module_info(self.name, "weight", self.weight,
"ignore_index", self.ignore_index,
"reduction", self.reduction))
def __init__(self, config):
super(FFN, self).__init__()
assert "input_dim" in config and "out_dim_0" in config and "out_dim_1" in config
self.input_dim = config["input_dim"]
self.out_dim_0 = config["out_dim_0"]
self.out_dim_1 = config["out_dim_1"]
self.linear_1 = nn.Linear(in_features=self.input_dim,
out_features=self.out_dim_0,
bias=False)
self.bias_1 = nn.Parameter(torch.zeros(1))
self.relu = nn.ReLU(inplace=True)
self.linear_2 = nn.Linear(in_features=self.out_dim_0,
out_features=self.out_dim_1,
bias=False)
self.bias_2 = nn.Parameter(torch.zeros(1))
self.name = config["name"] if "name" in config else "FFN"
logger.info(
utils.generate_module_info(self.name, "input_dim", self.input_dim,
"out_dim_0", self.out_dim_0,
"out_dim_1", self.out_dim_1))
def __init__(self, config):
super(BCEWithLogitsLoss, self).__init__()
if "weight" in config and isinstance(
config["weight"], (list, tuple, np.ndarray, torch.Tensor)):
self.weight = torch.Tensor(config["weight"])
else:
self.weight = None
if "pos_weight" in config and isinstance(
config["pos_weight"], (list, tuple, np.ndarray, torch.Tensor)):
self.pos_weight = torch.Tensor(config["pos_weight"])
else:
self.pos_weight = None
self.reduction = config[
"reduction"] if "reduction" in config else "elementwise_mean"
self.loss_module = nn.BCEWithLogitsLoss(weight=self.weight,
reduction=self.reduction,
pos_weight=self.pos_weight)
self.name = config[
"name"] if "name" in config else "Binary Cross Entropy"
logger.info(
utils.generate_module_info(self.name, "weight", self.weight,
"pos_weight", self.pos_weight,
"reduction", self.reduction))
def __init__(self, config):
self.name = "Accuracy"
logger.info(utils.generate_module_info(self.name))
