def __init__(self, dim_sound, num_tokens=30522, dim_model=768, nhead=12, name="music_bert",\
num_encoder_layers=12, d_feed=3072, dropout=0.1, n_convs=4):
super(MusicBert, self).__init__()
self.d_model = dim_model
self.num_tokens = num_tokens
self.PATH = "models/" + name + ".pth"
# Token Embedder
self.embedder = nn.Embedding(num_tokens, dim_model)
self.sound_compressor = SoundCNNEncoder([dim_sound] + [dim_model] *
(n_convs - 1))
self.sound_decompressor = SoundCNNDecoder([dim_model] * n_convs)
self.position_embeddings = PositionalEncodings(dim_model, dropout)
encoder_layer = TransformerEncoderLayer(dim_model, nhead, \
dim_feedforward=d_feed, \
dropout=dropout,
activation='gelu')
device = self.get_device()
# Registers IDs in the buffers memory
self.register_buffer("CLS_ID", torch.tensor([[101]]).to(device).long())
self.register_buffer("SEP_ID", torch.tensor([[102]]).to(device).long())
self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers)
def __init__(self, config ):
super(Nucleus, self).__init__()
self.config = config
# Embedding Layer.
self.embedding = nn.Embedding( bittensor.__vocab_size__, bittensor.__network_dim__ )
# Local Model
local_layers = TransformerEncoderLayer( bittensor.__network_dim__, self.config.nucleus.nhead, self.config.nucleus.nhid, self.config.nucleus.dropout, batch_first=True)
local_hidden_layers = TransformerEncoderLayer( bittensor.__network_dim__, self.config.nucleus.nhead, self.config.nucleus.nhid, self.config.nucleus.dropout, batch_first=True )
self.local_pos_encoder = PositionalEncoding(bittensor.__network_dim__, self.config.nucleus.dropout)
self.local_encoder = TransformerEncoder( local_layers, self.config.nucleus.nlayers )
self.local_hidden = TransformerEncoder( local_hidden_layers, self.config.nucleus.nlayers )
self.local_decoder = nn.Linear( bittensor.__network_dim__, bittensor.__vocab_size__ , bias=False)
# Remote Model
remote_context_layers = TransformerEncoderLayer( bittensor.__network_dim__, self.config.nucleus.nhead, self.config.nucleus.nhid, self.config.nucleus.dropout, batch_first=True )
self.remote_hidden = TransformerEncoder( remote_context_layers, self.config.nucleus.nlayers )
self.remote_decoder = nn.Linear( bittensor.__network_dim__, bittensor.__vocab_size__ , bias=False)
self.loss_fct = nn.CrossEntropyLoss()
self.noise_multiplier = self.config.nucleus.noise_multiplier
self.peer_weights = nn.Parameter(torch.ones( [0] , requires_grad=True))
self.init_weights()
self.metagraph = None
self.dendrite = None
def __init__(self, config, metagraph, dendrite, device):
super(Validator, self).__init__()
self.embedding = torch.nn.Embedding(bittensor.__vocab_size__,
bittensor.__network_dim__)
self.layers = TransformerEncoderLayer(bittensor.__network_dim__,
config.nucleus.nhead,
config.nucleus.nhid,
config.nucleus.dropout,
batch_first=True)
self.encoder = TransformerEncoder(self.layers, config.nucleus.nlayers)
self.c_layers = TransformerEncoderLayer(bittensor.__network_dim__,
config.nucleus.nhead,
config.nucleus.nhid,
config.nucleus.dropout,
batch_first=True)
self.local_encoder = TransformerEncoder(self.c_layers, 1)
self.decoder = torch.nn.Linear(bittensor.__network_dim__,
bittensor.__vocab_size__,
bias=False)
self.loss_fct = torch.nn.CrossEntropyLoss()
self.peer_weights = torch.ones([metagraph().n.item()],
requires_grad=False,
device=device)
self.metagraph = metagraph
self.dendrite = dendrite
self.config = config
self.device = device
self.gates = {}
self.sync_with_chain_state()
def __init__(self,
nlayers=3,
hdim_rgb=1024,
hdim_audio=128,
nhead=8,
dropout=0.5):
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.model_type = 'Transformer'
self.name = 'Transformer_gate_' + str(nlayers)
self.dropout = dropout
self.src_mask = None
self.pos_encoder_rgb = PositionalEncoding(hdim_rgb, dropout)
self.pos_encoder_audio = PositionalEncoding(hdim_audio, dropout)
encoder_layers_rgb = TransformerEncoderLayer(hdim_rgb, nhead, hdim_rgb,
dropout)
encoder_layers_audio = TransformerEncoderLayer(hdim_audio, nhead,
hdim_audio, dropout)
self.transformer_rgb = TransformerEncoder(encoder_layers_rgb, nlayers)
self.transformer_audio = TransformerEncoder(encoder_layers_audio,
nlayers)
self.fc = nn.Linear((hdim_audio + hdim_rgb) * 5, NUM_CLASSES)
self.gate_fc = nn.Linear((hdim_audio + hdim_rgb) * 5,
(hdim_audio + hdim_rgb) * 5)
self.init_weights()
def __init__(self, nuser, nloc, ntime, nquadkey, user_dim, loc_dim, time_dim, reg_dim, nhid, nhead_enc, nhead_dec, nlayers, dropout=0.5, **extra_config):
super(QuadKeyLocPredictor, self).__init__()
self.emb_user = embedding(nuser, user_dim, zeros_pad=True, scale=True)
self.emb_loc = embedding(nloc, loc_dim, zeros_pad=True, scale=True)
self.emb_reg = embedding(nquadkey, reg_dim, zeros_pad=True, scale=True)
self.emb_time = embedding(ntime, time_dim, zeros_pad=True, scale=True)
ninp = user_dim
pos_encoding = extra_config.get("position_encoding", "transformer")
if pos_encoding == "embedding":
self.pos_encoder = PositionalEmbedding(loc_dim + reg_dim, dropout)
elif pos_encoding == "transformer":
self.pos_encoder = PositionalEncoding(loc_dim + reg_dim, dropout)
self.enc_layer = TransformerEncoderLayer(loc_dim + reg_dim, nhead_enc, loc_dim + reg_dim, dropout)
self.encoder = TransformerEncoder(self.enc_layer, nlayers)
self.region_pos_encoder = PositionalEmbedding(reg_dim, dropout, max_len=20)
self.region_enc_layer = TransformerEncoderLayer(reg_dim, 1, reg_dim, dropout=dropout)
self.region_encoder = TransformerEncoder(self.region_enc_layer, 2)
if not extra_config.get("use_location_only", False):
if extra_config.get("embedding_fusion", "multiply") == "concat":
if extra_config.get("user_embedding", False):
self.lin = nn.Linear(user_dim + loc_dim + reg_dim + time_dim, ninp)
else:
self.lin = nn.Linear(loc_dim + reg_dim, ninp)
ident_mat = torch.eye(ninp)
self.register_buffer('ident_mat', ident_mat)
self.layer_norm = nn.LayerNorm(ninp)
self.extra_config = extra_config
self.dropout = dropout
def __init__(self, config, data_feature):
super().__init__(config, data_feature)
self.device = config['device']
# depend on dataset
self.num_neg = config['executor_config']['train']['num_negative_samples']
self.temperature = config['executor_config']['train']['temperature']
# from dataset
# from train_dataset!!
nuser = data_feature['nuser']
nloc = data_feature['nloc']
ntime = data_feature['ntime']
nquadkey = data_feature['nquadkey']
# from config
user_dim = int(config['model_config']['user_embedding_dim'])
loc_dim = int(config['model_config']['location_embedding_dim'])
time_dim = int(config['model_config']['time_embedding_dim'])
reg_dim = int(config['model_config']['region_embedding_dim'])
# nhid = int(config['model_config']['hidden_dim_encoder'])
nhead_enc = int(config['model_config']['num_heads_encoder'])
# nhead_dec = int(config['model_config']['num_heads_decoder'])
nlayers = int(config['model_config']['num_layers_encoder'])
dropout = float(config['model_config']['dropout'])
extra_config = config['model_config']['extra_config']
# print(f"nloc: {nloc} \t loc_dim: {loc_dim}")
# essential
self.emb_loc = Embedding(nloc, loc_dim, zeros_pad=True, scale=True)
self.emb_reg = Embedding(nquadkey, reg_dim, zeros_pad=True, scale=True)
# optional
self.emb_user = Embedding(nuser, user_dim, zeros_pad=True, scale=True)
self.emb_time = Embedding(ntime, time_dim, zeros_pad=True, scale=True)
ninp = user_dim
pos_encoding = extra_config.get("position_encoding", "transformer")
if pos_encoding == "embedding":
self.pos_encoder = PositionalEmbedding(loc_dim + reg_dim, dropout)
elif pos_encoding == "transformer":
self.pos_encoder = PositionalEncoding(loc_dim + reg_dim, dropout)
self.enc_layer = TransformerEncoderLayer(loc_dim + reg_dim, nhead_enc, loc_dim + reg_dim, dropout)
self.encoder = TransformerEncoder(self.enc_layer, nlayers)
self.region_pos_encoder = PositionalEmbedding(reg_dim, dropout, max_len=20)
self.region_enc_layer = TransformerEncoderLayer(reg_dim, 1, reg_dim, dropout=dropout)
self.region_encoder = TransformerEncoder(self.region_enc_layer, 2)
if not extra_config.get("use_location_only", False):
if extra_config.get("embedding_fusion", "multiply") == "concat":
if extra_config.get("user_embedding", False):
self.lin = nn.Linear(user_dim + loc_dim + reg_dim + time_dim, ninp)
else:
self.lin = nn.Linear(loc_dim + reg_dim, ninp)
ident_mat = torch.eye(ninp)
self.register_buffer('ident_mat', ident_mat)
self.layer_norm = nn.LayerNorm(ninp)
self.extra_config = extra_config
self.dropout = dropout
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, params, dropout=0.5, embedding=None):
super().__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.model_type = 'TransformerEncoder'
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.embedding = nn.Embedding(ntoken, ninp) if embedding is None else embedding
self.ninp = ninp
self.params = params
def __init__(self,
ntoken_in,
ntoken_out,
embed_size,
drop_it,
trns_model='base'):
super(Transformer_Model, self).__init__()
self.src_mask = None
self.trg_mask = None
self.drop = drop_it
self.PAD_token = 2
self.src_pad_mask = None
self.trg_pad_mask = None
self.embed_in = nn.Embedding(ntoken_in, embed_size)
self.embed_out = nn.Embedding(ntoken_out, embed_size)
self.pos_encoder = PositionalEncoding(embed_size,
self.drop,
max_len=5000)
if trns_model == 'base':
# base model
encoder_layer = TransformerEncoderLayer(512, 8, 2048, 0.1)
encoder_norm = LayerNorm(512)
self.encoder = TransformerEncoder(encoder_layer, 6, encoder_norm)
decoder_layer = TransformerDecoderLayer(512, 8, 2048, 0.1)
decoder_norm = LayerNorm(512)
self.decoder = TransformerDecoder(decoder_layer, 6, decoder_norm)
else:
# big model
encoder_layer = TransformerEncoderLayer(1024, 16, 4096, 0.3)
encoder_norm = LayerNorm(1024)
self.encoder = TransformerEncoder(encoder_layer, 6, encoder_norm)
decoder_layer = TransformerDecoderLayer(1024, 16, 4096, 0.3)
decoder_norm = LayerNorm(1024)
self.decoder = TransformerDecoder(decoder_layer, 6, decoder_norm)
self.ninp = embed_size
self.linear_dec = nn.Linear(embed_size, ntoken_out)
# initialise embedding & linear layer parameters
self.init_weights()
# initialise transformer parameters
self.reset_params()
def __init__(self,
in_dim,
num_heads,
hidden_dim,
num_layers,
device,
dropout=0.5,
use_pos_encoder=False):
super(TransformerModel, self).__init__()
self.in_dim = in_dim
self.num_heads = num_heads
self.hidden_dim = hidden_dim
self.num_layers = num_layers
self.device = device
self.batch_size = 128
if dropout is None:
dropout = 0
if use_pos_encoder:
self.pos_encoder = PositionalEncoding(in_dim, dropout)
else:
self.pos_encoder = lambda x: x
encoder_layers = TransformerEncoderLayer(in_dim, num_heads, hidden_dim,
dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers,
num_layers)
def __init__(self, num_layers, em_dim, num_heads, ff_dim, input_vocab,
target_vocab, max_pos_enc, max_pos_dec):
super().__init__()
self.em_dim = em_dim
# encoder layers
self.encode_embedding = nn.Embedding(input_vocab, em_dim)
self.encode_pos_embedding = PositionalEncoding(max_pos=max_pos_enc,
em_dim=em_dim)
encoder_layer = TransformerEncoderLayer(em_dim,
nhead=num_heads,
dim_feedforward=ff_dim)
self.transformer_encoder = TransformerEncoder(
encoder_layer=encoder_layer, num_layers=num_layers)
self.encode_dropout = nn.Dropout(p=0.1)
# decoder layers
self.decode_embedding = nn.Embedding(target_vocab, em_dim)
self.decode_pos_embedding = PositionalEncoding(max_pos=max_pos_dec,
em_dim=em_dim)
decoder_layer = TransformerDecoderLayer(em_dim,
nhead=num_heads,
dim_feedforward=ff_dim)
self.transformer_decoder = TransformerDecoder(
decoder_layer=decoder_layer, num_layers=num_layers)
self.decode_dropout = nn.Dropout(p=0.1)
self.final_linear = nn.Linear(em_dim, target_vocab)
def __init__(self,
input_size,
num_sources,
dmodel,
num_heads,
num_layers,
hidden_size,
dropout=0.5,
num_splits=2):
super().__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.m = input_size
self.s = num_sources
self.dmodel = dmodel
self.ns = num_splits
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(dmodel, dropout)
encoder_layers = TransformerEncoderLayer(dmodel, num_heads,
hidden_size, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers,
num_layers)
self.encoder = nn.Linear(2 * input_size, self.ns * dmodel)
self.decoder = nn.Linear(dmodel, 2 * self.s * self.m // self.ns)
self.init_weights()
def __init__(self,
n_enc_layer,
embed_size,
n_head,
intermediate_size,
max_seq_len=512,
dropout=0.1,
**kwargs):
super(Transformer_Encoder_Extraction_Layer, self).__init__(**kwargs)
assert embed_size % n_head == 0
self.n_enc_layer = n_enc_layer
self.embed_size = embed_size
self.max_seq_len = max_seq_len
self.n_head = n_head
self.intermediate_size = intermediate_size
self.dropout = dropout
self.positional_encoder = Positional_Encoding_Layer(
embed_size, max_seq_len=max_seq_len)
transformer_encoder_layer = TransformerEncoderLayer(
embed_size,
n_head,
dim_feedforward=intermediate_size,
dropout=dropout)
self.transformer_encoder = TransformerEncoder(
transformer_encoder_layer, n_enc_layer)
self._init_weights()
def __init__(self,
d_model: int = 512,
num_heads: int = 8,
num_encoder_layers: int = 6,
num_decoder_layers: int = 6,
units: int = 2048,
dropout: float = 0.1,
activation: str = "relu") -> NoReturn:
"""
:param d_model: 深度，词嵌入维度
:param num_heads: 注意力头数
:param num_encoder_layers: encoder层数
:param num_decoder_layers: decoder层数
:param units: 单元数
:param dropout: 采样率
:param activation: 激活方法
"""
super(Transformer, self).__init__()
encoder_layer = TransformerEncoderLayer(d_model, num_heads, units,
dropout, activation)
encoder_norm = torch.nn.LayerNorm(d_model)
self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers,
encoder_norm)
decoder_layer = TransformerDecoderLayer(d_model, num_heads, units,
dropout, activation)
decoder_norm = torch.nn.LayerNorm(d_model)
self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers,
decoder_norm)
self._reset_parameters()
self.d_model = d_model
self.num_heads = num_heads
def __init__(self,
ntoken,
ninp=128,
nhead=8,
nhid=128,
nlayers=8,
dropout=0.5):
#ntoken: len(dictionary)
#ninp : embedding dimension
#nhead: # of multiheadattention
#nhid : dim(feedforward network model)
#nlayer: # of nn.TransofrmerEncoderLayer
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
if not ntoken:
self.ntoken = len(load_object("engDictAnn.pkl"))
else:
self.ntoken = ntoken
self.model_type = "Transformer"
self.pos_encoder = PositionalEncoding(ninp, dropout)
encdoer_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encdoer_layers, nlayers)
self.encoder = nn.Embedding(ntoken, ninp)
self.ninp = ninp
self.decoder = nn.Linear(ninp, self.ntoken)
self.init_weights()
def __init__(self, vocab, nemb, nhead, nhid, nlayer, nclass, dropout=0.1):
super(Transformer, self).__init__()
try:
from torch.nn import TransformerEncoder, TransformerEncoderLayer
except:
raise ImportError('TransformerEncoder module does not exist in PyTorch 1.1 or lower.')
self.model_type = 'Transformer'
self.ntoken = len(vocab)
self.ninp = nemb
self.nhead = nhead
self.nhid = nhid
self.nlayer = nlayer
self.nclass = nclass
self.dropout = dropout
#self.encoder = nn.Embedding.from_pretrained(vocab.vectors, freeze=False)
self.embedding = nn.Embedding(self.ntoken, self.ninp)
#self.dropout1 = nn.Dropout(self.dropout)
self.pos_encoder = PositionalEncoding(nemb, dropout)
self.src_mask = None
encoder_layers = TransformerEncoderLayer(self.ninp, self.nhead, self.nhid, self.dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, self.nlayer)
self.dropout2 = nn.Dropout(self.dropout)
#self.pool = nn.AdaptiveMaxPool1d(1)
self.pool = nn.AdaptiveAvgPool1d(1)
self.decoder = nn.Linear(self.ninp, self.nclass)
def __init__(self,
ntoken,
ninp,
max_len,
nhead,
nhid,
nlayers,
nclasses,
cat_tokens,
cat_size,
cat_len,
dropout=0.5):
super(TransformerModel, self).__init__()
self.model_type = 'Transformer'
# transformer layers
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, ninp)
# embed extra categorical variables
self.cat_embed = nn.Embedding(cat_tokens, cat_size)
# final fc
self.decoder = nn.Linear(ninp * max_len + cat_size * cat_len, nclasses)
#dimensions
self.ninp = ninp
self.max_len = max_len
self.cat_size = cat_size
self.cat_len = cat_len
self.init_weights()
def __init__(self, nuser, nloc, ntime, nreg, user_dim, loc_dim, time_dim, reg_dim, nhid, nhead_enc, nhead_dec, nlayers, dropout=0.5, **extra_config):
super(LocPredictor, self).__init__()
self.emb_user = embedding(nuser, user_dim, zeros_pad=True, scale=True)
self.emb_loc = embedding(nloc, loc_dim, zeros_pad=True, scale=True)
self.emb_reg = embedding(nreg, reg_dim, zeros_pad=True, scale=True)
self.emb_time = embedding(ntime, time_dim, zeros_pad=True, scale=True)
if not ((user_dim == loc_dim) and (user_dim == time_dim) and (user_dim == reg_dim)):
raise Exception('user, location, time and region should have the same embedding size')
ninp = user_dim
pos_encoding = extra_config.get("position_encoding", "transformer")
if pos_encoding == "embedding":
self.pos_encoder = PositionalEmbedding(ninp, dropout)
elif pos_encoding == "transformer":
self.pos_encoder = PositionalEncoding(ninp, dropout)
self.enc_layer = TransformerEncoderLayer(ninp, nhead_enc, nhid, dropout)
self.encoder = TransformerEncoder(self.enc_layer, nlayers)
if not extra_config.get("use_location_only", False):
if extra_config.get("embedding_fusion", "multiply") == "concat":
if extra_config.get("user_embedding", False):
self.lin = nn.Linear(user_dim + loc_dim + reg_dim + time_dim, ninp)
else:
self.lin = nn.Linear(loc_dim + reg_dim + time_dim, ninp)
ident_mat = torch.eye(ninp)
self.register_buffer('ident_mat', ident_mat)
self.layer_norm = nn.LayerNorm(ninp)
self.extra_config = extra_config
self.dropout = dropout
def __init__(self,
ntoken=549,
ninp=256,
nhead=2,
nhid=200,
nlayers=2,
dropout=0.1):
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.input_linear = nn.Linear(ntoken, ninp)
self.ninp = ninp
self.input_size = ntoken
self.decoder_rnn = nn.GRU(ninp, nhid, batch_first=True)
self.domain_classify_layer1 = nn.Linear(ninp, 64)
self.domain_classify_layer2 = nn.Linear(64, 16)
self.domain_classify_layer3 = nn.Linear(16, 8)
self.linear1 = nn.Linear(ntoken, nip)
self.linear2 = nn.Linear(nip, nip)
self.output_linear = nn.Linear(nhid, ntoken)
self.init_weights()
def __init__(
self,
ntoken,
ninp,
nhead,
nhid,
nlayers,
dropout=0.5,
activation="relu",
tie_weights=False,
):
super(TransformerModel, self).__init__()
try:
from torch.nn import TransformerEncoder, TransformerEncoderLayer
except ImportError:
raise ImportError("TransformerEncoder module does not exist in "
"PyTorch 1.1 or lower.")
self.model_type = "Transformer"
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout,
activation)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, ninp)
self.ninp = ninp
self.decoder = nn.Linear(ninp, ntoken)
if tie_weights:
if nhid != ninp:
raise ValueError(
"When using the tied flag, nhid must be equal "
"to emsize.")
self.decoder.weight = self.encoder.weight
self.init_weights()
def __init__(self, cfg: DictConfig, cwd: Path):
super().__init__()
#self.example_input_array = torch.rand(2, 1, 1024, 128)
self.cfg = cfg
self.dataset = cfg.dataset
self.hparams = cfg.hparams
self.cwd = cwd
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(self.hparams["n_mels"], self.hparams["dropout"])
encoder_layers = TransformerEncoderLayer(self.hparams["n_mels"],
self.hparams["n_head"],
self.hparams["n_hid"],
self.hparams["dropout"])
self.encoder = TransformerEncoder(encoder_layers, self.hparams["n_layers"])
decoder_layers = TransformerDecoderLayer(self.hparams["n_mels"],
self.hparams["n_head"],
self.hparams["n_hid"],
self.hparams["dropout"])
self.decoder = TransformerDecoder(decoder_layers, self.hparams["n_layers"])
def __init__(
self,
feature_size,
output_size,
ninp,
nhead,
nhid,
nlayers,
dropout=0.5,
condition_decoder=False,
transformer_norm=False,
):
"""This model is built upon https://pytorch.org/tutorials/beginner/transformer_tutorial.html"""
super(TransformerModel, self).__init__()
self.model_type = "Transformer"
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(
encoder_layers,
nlayers,
norm=nn.LayerNorm(ninp) if transformer_norm else None,
)
self.encoder = nn.Linear(feature_size, ninp)
self.ninp = ninp
self.condition_decoder = condition_decoder
self.decoder = nn.Linear(
ninp + feature_size if condition_decoder else ninp, output_size
)
self.init_weights()
def __init__(self, src_vocab_size=128, tgt_vocab_size=128,
embedding_dim=128, fcn_hidden_dim=128,
num_heads=4, num_layers=2, dropout=0.2):
super(Transformer, self).__init__()
self.embedding_dim = embedding_dim
# Source and Encoder layers
self.src_embed = Embedding(src_vocab_size, embedding_dim, padding_idx=PAD_ID)
self.src_pos_encoder = PositionalEncoding(embedding_dim)
encoder_layer = TransformerEncoderLayer(d_model=embedding_dim, nhead=num_heads,
dim_feedforward=fcn_hidden_dim, dropout=dropout)
encoder_norm = nn.LayerNorm(embedding_dim)
self.encoder = TransformerEncoder(encoder_layer, num_layers, encoder_norm)
# Target and Decoder layers
self.tgt_embed = Embedding(tgt_vocab_size, embedding_dim, padding_idx=PAD_ID)
self.tgt_pos_encoder = PositionalEncoding(embedding_dim)
decoder_layer = nn.TransformerDecoderLayer(d_model=embedding_dim, nhead=num_heads,
dim_feedforward=fcn_hidden_dim, dropout=dropout)
decoder_norm = nn.LayerNorm(embedding_dim)
self.decoder = nn.TransformerDecoder(decoder_layer, num_layers, decoder_norm)
# Final linear layer
self.final_out = nn.Linear(embedding_dim, tgt_vocab_size)
# Initialize masks
self.src_mask = None
self.tgt_mask = None
self.mem_mask = None
# Initialize weights of model
self._reset_parameters()
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
super(BertModel, self).__init__()
self.model_type = 'Transformer'
self.bert_embed = BertEmbedding(ntoken, ninp)
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.ninp = ninp
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
super(TransformerModel, self).__init__()
try:
from torch.nn import TransformerEncoder, TransformerEncoderLayer, TransformerDecoder, TransformerDecoderLayer
except:
raise ImportError('Use a newer version of PyTorch')
self.model_type = 'transformer'
self.src_mask = None
self.ninp = ninp
self.ntoken = ntoken
self.pos_encoder = PositionalEncoding(ninp, dropout)
self.encoder = nn.Embedding(ntoken, ninp)
encoder_norm = nn.LayerNorm(ninp)
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers,
encoder_norm)
#decoder_norm = nn.LayerNorm(ninp)
#decoder_layers = TransformerDecoderLayer(ninp, nhead, nhid, dropout)
#self.transformer_decoder = TransformerDecoder(decoder_layers, nlayers, decoder_norm)
self.decoder = nn.Linear(ninp, ntoken)
self.init_weights()
def __init__(self,
src_feature_size,
ntoken,
nhead,
nhid,
nlayers,
dropout=0.5,
decoder_d_model=2048):
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer, TransformerDecoder, TransformerDecoderLayer, Transformer
self.encoder_d_model = src_feature_size
self.decoder_d_model = decoder_d_model
self.noken = ntoken
self.model_type = 'Transformer'
self.trg_mask = None
self.encoder_pre = PreProcessing()
self.enc_linear = nn.Linear(src_feature_size, decoder_d_model)
self.pos_encoder = PositionalEncoding(self.decoder_d_model, dropout)
encoder_layers = TransformerEncoderLayer(d_model=src_feature_size,
nhead=nhead,
dim_feedforward=nhid,
dropout=dropout)
self.transformer_encoder = TransformerEncoder(
encoder_layer=encoder_layers, num_layers=nlayers)
self.embedder = Embedder(ntoken, decoder_d_model)
self.pos_decoder = PositionalEncoding(decoder_d_model, dropout)
decoder_layers = TransformerDecoderLayer(d_model=decoder_d_model,
nhead=nhead,
dim_feedforward=nhid,
dropout=dropout)
self.transformer_decoder = TransformerDecoder(
decoder_layer=decoder_layers, num_layers=nlayers)
self.out = nn.Linear(decoder_d_model, ntoken)
def __init__(self, kg_graph_repr: Dict[str, np.ndarray], config: dict, id2e: tuple = None):
super(self.__class__, self).__init__(kg_graph_repr, config)
self.model_name = 'StarE_Transformer_Statement'
self.hid_drop2 = config['STAREARGS']['HID_DROP2']
self.feat_drop = config['STAREARGS']['FEAT_DROP']
self.num_transformer_layers = config['STAREARGS']['T_LAYERS']
self.num_heads = config['STAREARGS']['T_N_HEADS']
self.num_hidden = config['STAREARGS']['T_HIDDEN']
self.d_model = config['EMBEDDING_DIM']
self.positional = config['STAREARGS']['POSITIONAL']
self.object_mask_emb = torch.nn.Parameter(torch.randn(1, self.emb_dim,dtype=torch.float32),True)
self.hidden_drop = torch.nn.Dropout(self.hid_drop)
self.hidden_drop2 = torch.nn.Dropout(self.hid_drop2)
self.feature_drop = torch.nn.Dropout(self.feat_drop)
encoder_layers = TransformerEncoderLayer(self.d_model, self.num_heads, self.num_hidden, config['STAREARGS']['HID_DROP2'])
self.encoder = TransformerEncoder(encoder_layers, config['STAREARGS']['T_LAYERS'])
self.position_embeddings = nn.Embedding(config['MAX_QPAIRS'], self.d_model)
self.layer_norm = torch.nn.LayerNorm(self.emb_dim)
self.flat_sz = self.emb_dim * (config['MAX_QPAIRS'] - 1)
self.fc = torch.nn.Linear(self.emb_dim, self.emb_dim)
def __init__(self, num_tokens, embedding_table, max_time_steps, text_dim, quat_dim, quat_channels,
offsets_dim, intended_emotion_dim, intended_polarity_dim, acting_task_dim,
gender_dim, age_dim, handedness_dim, native_tongue_dim, num_heads, num_hidden_units,
num_layers, dropout=0.5):
super(T2GNet, self).__init__()
self.T = max_time_steps
self.text_dim = text_dim
self.quat_channels = quat_channels
self.text_mask = None
self.quat_mask = None
self.text_embedding = nn.Embedding.from_pretrained(embedding_table, freeze=True)
self.text_pos_encoder = PositionalEncoding(text_dim, dropout)
encoder_layers = TransformerEncoderLayer(text_dim, num_heads, num_hidden_units, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers)
intermediate_dim = int((text_dim + quat_dim) / 2)
self.text_embed = nn.Linear(text_dim + intended_emotion_dim + intended_polarity_dim +\
acting_task_dim + gender_dim + age_dim + handedness_dim + native_tongue_dim, intermediate_dim)
self.text_offsets_to_gestures = nn.Linear(intermediate_dim + offsets_dim, quat_dim)
self.quat_pos_encoder = PositionalEncoding(quat_dim, dropout)
decoder_layers = TransformerDecoderLayer(quat_dim, num_heads, num_hidden_units, dropout)
self.transformer_decoder = TransformerDecoder(decoder_layers, num_layers)
self.temporal_smoothing = nn.ModuleList((
nn.Conv1d(max_time_steps, max_time_steps, 3, padding=1),
nn.Conv1d(max_time_steps, max_time_steps, 3, padding=1),
))
self.decoder = nn.Linear(text_dim, num_tokens)
self.init_weights()
def __init__(self, kg_graph_repr: Dict[str, np.ndarray], config: dict, id2e: tuple = None):
if id2e is not None:
super(self.__class__, self).__init__(kg_graph_repr, config, id2e[1])
else:
super(self.__class__, self).__init__(kg_graph_repr, config)
self.model_name = 'StarE_Transformer_Statement'
self.hid_drop2 = config['STAREARGS']['HID_DROP2']
self.feat_drop = config['STAREARGS']['FEAT_DROP']
self.num_transformer_layers = config['STAREARGS']['T_LAYERS']
self.num_heads = config['STAREARGS']['T_N_HEADS']
self.num_hidden = config['STAREARGS']['T_HIDDEN']
self.d_model = config['EMBEDDING_DIM']
self.positional = config['STAREARGS']['POSITIONAL']
self.pooling = config['STAREARGS']['POOLING'] # min / avg / concat
self.hidden_drop = torch.nn.Dropout(self.hid_drop)
self.hidden_drop2 = torch.nn.Dropout(self.hid_drop2)
self.feature_drop = torch.nn.Dropout(self.feat_drop)
encoder_layers = TransformerEncoderLayer(self.d_model, self.num_heads, self.num_hidden, config['STAREARGS']['HID_DROP2'])
self.encoder = TransformerEncoder(encoder_layers, config['STAREARGS']['T_LAYERS'])
self.position_embeddings = nn.Embedding(config['MAX_QPAIRS'] - 1, self.d_model)
self.layer_norm = torch.nn.LayerNorm(self.emb_dim)
if self.pooling == "concat":
self.flat_sz = self.emb_dim * (config['MAX_QPAIRS'] - 1)
self.fc = torch.nn.Linear(self.flat_sz, self.emb_dim)
else:
self.fc = torch.nn.Linear(self.emb_dim, self.emb_dim)
def __init__(self,
ntoken,
ninp,
nhead,
nhid,
nlayers,
dropout=0.5,
norm=True,
sp=0.0,
activation='relu'):
super(TransformerModel, self).__init__()
try:
from torch.nn import TransformerEncoder, TransformerEncoderLayer
except:
raise ImportError(
'TransformerEncoder module does not exist in PyTorch 1.1 or lower.'
)
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layers = TransformerEncoderLayer(ninp,
nhead,
nhid,
dropout,
norm=norm,
sp=sp,
activation=activation) #Yang
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, ninp)
self.ninp = ninp
self.decoder = nn.Linear(ninp, ntoken)
self.init_weights()
def __init__(self, config: dict):
super().__init__(config)
#self.emb_dim = config['EMBEDDING_DIM']
self.entities = get_param((self.num_ent, self.emb_dim))
self.relations = get_param((2 * self.num_rel, self.emb_dim))
self.model_name = 'Transformer_Statement'
self.hid_drop2 = config['STAREARGS']['HID_DROP2']
self.feat_drop = config['STAREARGS']['FEAT_DROP']
self.num_transformer_layers = config['STAREARGS']['T_LAYERS']
self.num_heads = config['STAREARGS']['T_N_HEADS']
self.num_hidden = config['STAREARGS']['T_HIDDEN']
self.d_model = config['EMBEDDING_DIM']
self.positional = config['STAREARGS']['POSITIONAL']
self.pooling = config['STAREARGS']['POOLING'] # min / avg / concat
self.device = config['DEVICE']
self.hidden_drop = torch.nn.Dropout(self.hid_drop)
self.hidden_drop2 = torch.nn.Dropout(self.hid_drop2)
self.feature_drop = torch.nn.Dropout(self.feat_drop)
encoder_layers = TransformerEncoderLayer(self.d_model, self.num_heads, self.num_hidden,
config['STAREARGS']['HID_DROP2'])
self.encoder = TransformerEncoder(encoder_layers, config['STAREARGS']['T_LAYERS'])
self.position_embeddings = nn.Embedding(config['MAX_QPAIRS'] - 1, self.d_model)
self.layer_norm = torch.nn.LayerNorm(self.emb_dim)
if self.pooling == "concat":
self.flat_sz = self.emb_dim * (config['MAX_QPAIRS'] - 1)
self.fc = torch.nn.Linear(self.flat_sz, self.emb_dim)
else:
self.fc = torch.nn.Linear(self.emb_dim, self.emb_dim)
