def __init__(self, config, dataset): super(FDSA, self).__init__(config, dataset) # load parameters info self.n_layers = config['n_layers'] self.n_heads = config['n_heads'] self.hidden_size = config['hidden_size'] # same as embedding_size self.inner_size = config['inner_size'] # the dimensionality in feed-forward layer self.hidden_dropout_prob = config['hidden_dropout_prob'] self.attn_dropout_prob = config['attn_dropout_prob'] self.hidden_act = config['hidden_act'] self.layer_norm_eps = config['layer_norm_eps'] self.selected_features = config['selected_features'] self.pooling_mode = config['pooling_mode'] self.device = config['device'] self.num_feature_field = len(config['selected_features']) self.initializer_range = config['initializer_range'] self.loss_type = config['loss_type'] # define layers and loss self.item_embedding = nn.Embedding(self.n_items, self.hidden_size, padding_idx=0) self.position_embedding = nn.Embedding(self.max_seq_length, self.hidden_size) self.feature_embed_layer = FeatureSeqEmbLayer(dataset, self.hidden_size, self.selected_features, self.pooling_mode, self.device) self.item_trm_encoder = TransformerEncoder(n_layers=self.n_layers, n_heads=self.n_heads, hidden_size=self.hidden_size, inner_size=self.inner_size, hidden_dropout_prob=self.hidden_dropout_prob, attn_dropout_prob=self.attn_dropout_prob, hidden_act=self.hidden_act, layer_norm_eps=self.layer_norm_eps) self.feature_att_layer = VanillaAttention(self.hidden_size, self.hidden_size) # For simplicity, we use same architecture for item_trm and feature_trm self.feature_trm_encoder = TransformerEncoder(n_layers=self.n_layers, n_heads=self.n_heads, hidden_size=self.hidden_size, inner_size=self.inner_size, hidden_dropout_prob=self.hidden_dropout_prob, attn_dropout_prob=self.attn_dropout_prob, hidden_act=self.hidden_act, layer_norm_eps=self.layer_norm_eps) self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.dropout = nn.Dropout(self.hidden_dropout_prob) self.concat_layer = nn.Linear(self.hidden_size * 2, self.hidden_size) if self.loss_type == 'BPR': self.loss_fct = BPRLoss() elif self.loss_type == 'CE': self.loss_fct = nn.CrossEntropyLoss() else: raise NotImplementedError("Make sure 'loss_type' in ['BPR', 'CE']!") # parameters initialization self.apply(self._init_weights)
def __init__(self, config, dataset): super().__init__() self.n_layers = config['n_layers'] self.n_heads = config['n_heads'] self.hidden_size = config['embedding_size'] self.inner_size = config['inner_size'] self.hidden_dropout_prob = config['hidden_dropout_prob'] self.attn_dropout_prob = config['attn_dropout_prob'] self.hidden_act = config['hidden_act'] self.layer_norm_eps = config['layer_norm_eps'] self.initializer_range = config['initializer_range'] self.position_embedding = nn.Embedding(dataset.field2seqlen[config['ITEM_ID_FIELD'] + config['LIST_SUFFIX']], self.hidden_size) self.trm_encoder = TransformerEncoder( n_layers=self.n_layers, n_heads=self.n_heads, hidden_size=self.hidden_size, inner_size=self.inner_size, hidden_dropout_prob=self.hidden_dropout_prob, attn_dropout_prob=self.attn_dropout_prob, hidden_act=self.hidden_act, layer_norm_eps=self.layer_norm_eps ) self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.dropout = nn.Dropout(self.hidden_dropout_prob) self.fn = nn.Linear(self.hidden_size, 1) self.apply(self._init_weights)
def __init__(self, config, dataset): super(BERT4Rec, self).__init__(config, dataset) # load parameters info self.n_layers = config['n_layers'] self.n_heads = config['n_heads'] self.hidden_size = config['hidden_size'] # same as embedding_size self.inner_size = config[ 'inner_size'] # the dimensionality in feed-forward layer self.hidden_dropout_prob = config['hidden_dropout_prob'] self.attn_dropout_prob = config['attn_dropout_prob'] self.hidden_act = config['hidden_act'] self.layer_norm_eps = config['layer_norm_eps'] self.mask_ratio = config['mask_ratio'] self.loss_type = config['loss_type'] self.initializer_range = config['initializer_range'] # load dataset info self.mask_token = self.n_items self.mask_item_length = int(self.mask_ratio * self.max_seq_length) # define layers and loss self.item_embedding = nn.Embedding(self.n_items + 1, self.hidden_size, padding_idx=0) # mask token add 1 self.position_embedding = nn.Embedding( self.max_seq_length + 1, self.hidden_size) # add mask_token at the last self.trm_encoder = TransformerEncoder( n_layers=self.n_layers, n_heads=self.n_heads, hidden_size=self.hidden_size, inner_size=self.inner_size, hidden_dropout_prob=self.hidden_dropout_prob, attn_dropout_prob=self.attn_dropout_prob, hidden_act=self.hidden_act, layer_norm_eps=self.layer_norm_eps) self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.dropout = nn.Dropout(self.hidden_dropout_prob) # we only need compute the loss at the masked position try: assert self.loss_type in ['BPR', 'CE'] except AssertionError: raise AssertionError("Make sure 'loss_type' in ['BPR', 'CE']!") # parameters initialization self.apply(self._init_weights)
def __init__(self, config, dataset): super(SASRec, self).__init__(config, dataset) # load parameters info self.n_layers = config['n_layers'] self.n_heads = config['n_heads'] self.hidden_size = config['hidden_size'] # same as embedding_size self.inner_size = config[ 'inner_size'] # the dimensionality in feed-forward layer self.hidden_dropout_prob = config['hidden_dropout_prob'] self.attn_dropout_prob = config['attn_dropout_prob'] self.hidden_act = config['hidden_act'] self.layer_norm_eps = config['layer_norm_eps'] self.initializer_range = config['initializer_range'] self.loss_type = config['loss_type'] # define layers and loss self.item_embedding = nn.Embedding(self.n_items, self.hidden_size, padding_idx=0) self.position_embedding = nn.Embedding(self.max_seq_length, self.hidden_size) self.trm_encoder = TransformerEncoder( n_layers=self.n_layers, n_heads=self.n_heads, hidden_size=self.hidden_size, inner_size=self.inner_size, hidden_dropout_prob=self.hidden_dropout_prob, attn_dropout_prob=self.attn_dropout_prob, hidden_act=self.hidden_act, layer_norm_eps=self.layer_norm_eps) self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.dropout = nn.Dropout(self.hidden_dropout_prob) if self.loss_type == 'BPR': self.loss_fct = BPRLoss() elif self.loss_type == 'CE': self.loss_fct = nn.CrossEntropyLoss() else: raise NotImplementedError( "Make sure 'loss_type' in ['BPR', 'CE']!") # parameters initialization self.apply(self._init_weights)
def __init__(self, config, dataset): super(GCSAN, self).__init__(config, dataset) # load parameters info self.n_layers = config['n_layers'] self.n_heads = config['n_heads'] self.hidden_size = config['hidden_size'] # same as embedding_size self.inner_size = config[ 'inner_size'] # the dimensionality in feed-forward layer self.hidden_dropout_prob = config['hidden_dropout_prob'] self.attn_dropout_prob = config['attn_dropout_prob'] self.hidden_act = config['hidden_act'] self.layer_norm_eps = config['layer_norm_eps'] self.step = config['step'] self.device = config['device'] self.weight = config['weight'] self.reg_weight = config['reg_weight'] self.loss_type = config['loss_type'] self.initializer_range = config['initializer_range'] # define layers and loss self.item_embedding = nn.Embedding(self.n_items, self.hidden_size, padding_idx=0) self.gnn = GNN(self.hidden_size, self.step) self.self_attention = TransformerEncoder( n_layers=self.n_layers, n_heads=self.n_heads, hidden_size=self.hidden_size, inner_size=self.inner_size, hidden_dropout_prob=self.hidden_dropout_prob, attn_dropout_prob=self.attn_dropout_prob, hidden_act=self.hidden_act, layer_norm_eps=self.layer_norm_eps) self.reg_loss = EmbLoss() if self.loss_type == 'BPR': self.loss_fct = BPRLoss() elif self.loss_type == 'CE': self.loss_fct = nn.CrossEntropyLoss() else: raise NotImplementedError( "Make sure 'loss_type' in ['BPR', 'CE']!") # parameters initialization self.apply(self._init_weights)
def __init__(self, config, dataset): super(S3Rec, self).__init__(config, dataset) # load parameters info self.n_layers = config['n_layers'] self.n_heads = config['n_heads'] self.hidden_size = config['hidden_size'] # same as embedding_size self.inner_size = config['inner_size'] # the dimensionality in feed-forward layer self.hidden_dropout_prob = config['hidden_dropout_prob'] self.attn_dropout_prob = config['attn_dropout_prob'] self.hidden_act = config['hidden_act'] self.layer_norm_eps = config['layer_norm_eps'] self.FEATURE_FIELD = config['item_attribute'] self.FEATURE_LIST = self.FEATURE_FIELD + config['LIST_SUFFIX'] self.train_stage = config['train_stage'] # pretrain or finetune self.pre_model_path = config['pre_model_path'] # We need this for finetune self.mask_ratio = config['mask_ratio'] self.aap_weight = config['aap_weight'] self.mip_weight = config['mip_weight'] self.map_weight = config['map_weight'] self.sp_weight = config['sp_weight'] self.initializer_range = config['initializer_range'] self.loss_type = config['loss_type'] # load dataset info self.n_items = dataset.item_num + 1 # for mask token self.mask_token = self.n_items - 1 self.n_features = dataset.num(self.FEATURE_FIELD) - 1 # we don't need padding self.item_feat = dataset.get_item_feature() # define layers and loss # modules shared by pre-training stage and fine-tuning stage self.item_embedding = nn.Embedding(self.n_items, self.hidden_size, padding_idx=0) self.position_embedding = nn.Embedding(self.max_seq_length, self.hidden_size) self.feature_embedding = nn.Embedding(self.n_features, self.hidden_size, padding_idx=0) self.trm_encoder = TransformerEncoder(n_layers=self.n_layers, n_heads=self.n_heads, hidden_size=self.hidden_size, inner_size=self.inner_size, hidden_dropout_prob=self.hidden_dropout_prob, attn_dropout_prob=self.attn_dropout_prob, hidden_act=self.hidden_act, layer_norm_eps=self.layer_norm_eps) self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.dropout = nn.Dropout(self.hidden_dropout_prob) # modules for pretrain # add unique dense layer for 4 losses respectively self.aap_norm = nn.Linear(self.hidden_size, self.hidden_size) self.mip_norm = nn.Linear(self.hidden_size, self.hidden_size) self.map_norm = nn.Linear(self.hidden_size, self.hidden_size) self.sp_norm = nn.Linear(self.hidden_size, self.hidden_size) self.loss_fct = nn.BCELoss(reduction='none') # modules for finetune if self.loss_type == 'BPR' and self.train_stage == 'finetune': self.loss_fct = BPRLoss() elif self.loss_type == 'CE' and self.train_stage == 'finetune': self.loss_fct = nn.CrossEntropyLoss() elif self.train_stage == 'finetune': raise NotImplementedError("Make sure 'loss_type' in ['BPR', 'CE']!") # parameters initialization assert self.train_stage in ['pretrain', 'finetune'] if self.train_stage == 'pretrain': self.apply(self._init_weights) else: # load pretrained model for finetune pretrained = torch.load(self.pre_model_path) self.logger.info('Load pretrained model from', self.pre_model_path) self.load_state_dict(pretrained['state_dict'])
def __init__(self, config, dataset): super(SASRecF, self).__init__(config, dataset) # load parameters info self.n_layers = config['n_layers'] self.n_heads = config['n_heads'] self.hidden_size = config['hidden_size'] # same as embedding_size self.inner_size = config[ 'inner_size'] # the dimensionality in feed-forward layer self.hidden_dropout_prob = config['hidden_dropout_prob'] self.attn_dropout_prob = config['attn_dropout_prob'] self.hidden_act = config['hidden_act'] self.layer_norm_eps = config['layer_norm_eps'] self.selected_features = config['selected_features'] self.pooling_mode = config['pooling_mode'] self.device = config['device'] self.num_feature_field = sum( 1 if dataset.field2type[field] != FeatureType.FLOAT_SEQ else dataset.num(field) for field in config['selected_features']) self.initializer_range = config['initializer_range'] self.loss_type = config['loss_type'] # define layers and loss self.item_embedding = nn.Embedding(self.n_items, self.hidden_size, padding_idx=0) self.position_embedding = nn.Embedding(self.max_seq_length, self.hidden_size) self.feature_embed_layer = FeatureSeqEmbLayer(dataset, self.hidden_size, self.selected_features, self.pooling_mode, self.device) self.trm_encoder = TransformerEncoder( n_layers=self.n_layers, n_heads=self.n_heads, hidden_size=self.hidden_size, inner_size=self.inner_size, hidden_dropout_prob=self.hidden_dropout_prob, attn_dropout_prob=self.attn_dropout_prob, hidden_act=self.hidden_act, layer_norm_eps=self.layer_norm_eps) self.concat_layer = nn.Linear( self.hidden_size * (1 + self.num_feature_field), self.hidden_size) self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.dropout = nn.Dropout(self.hidden_dropout_prob) if self.loss_type == 'BPR': self.loss_fct = BPRLoss() elif self.loss_type == 'CE': self.loss_fct = nn.CrossEntropyLoss() else: raise NotImplementedError( "Make sure 'loss_type' in ['BPR', 'CE']!") # parameters initialization self.apply(self._init_weights) self.other_parameter_name = ['feature_embed_layer']