def __init__(self, eye_size, output_size):
super(EYEFeatureExtractor, self).__init__()
self.eye_size = eye_size
self.output_size = output_size
self.regions = 5
self.regions_indexs = [
torch.LongTensor(e)
for e in [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [12, 13, 14, 15],
[16, 17], [18, 19, 20, 21],
[22, 23, 24, 25, 26, 27, 28, 29, 30]]
]
eye_extractor = []
eye_extractor.append(
FCNet(in_size=12, out_size=output_size, activate='relu'))
eye_extractor.append(
FCNet(in_size=4, out_size=output_size, activate='relu'))
eye_extractor.append(
FCNet(in_size=2, out_size=output_size, activate='relu'))
eye_extractor.append(
FCNet(in_size=4, out_size=output_size, activate='relu'))
eye_extractor.append(
FCNet(in_size=9, out_size=output_size, activate='relu'))
self.eyeFeatureExtractor = nn.ModuleList(eye_extractor)
self.bn = nn.BatchNorm1d(num_features=self.regions)
def __init__(self,
num_blocks,
v_size,
q_size,
output_size,
num_inter_head,
num_intra_head,
drop=0.0):
super(MultiBlocks, self).__init__()
self.v_size = v_size
self.q_size = q_size
self.output_size = output_size
self.num_inter_head = num_inter_head
self.num_intra_head = num_intra_head
self.num_blocks = num_blocks
self.v_lin = FCNet(v_size, output_size, drop=drop)
self.q_lin = FCNet(q_size, output_size, drop=drop)
blocks = []
for i in range(self.num_blocks):
#blocks.append(OneSideInterModalityUpdate(output_size, output_size, output_size, num_inter_head, drop))
#blocks.append(OneSideInterModalityUpdate(output_size, output_size, output_size, num_inter_head, drop))
blocks.append(
InterModalityUpdate(output_size, output_size, output_size,
num_inter_head, drop))
blocks.append(
DyIntraModalityUpdate(output_size, output_size, output_size,
num_intra_head, drop))
self.multi_blocks = nn.ModuleList(blocks)
def __init__(self,
num_blocks,
v_size,
q_size,
output_size,
num_inter_head,
num_intra_head,
drop=0.0):
super(SingleBlock, self).__init__()
self.v_size = v_size
self.q_size = q_size
self.output_size = output_size
self.num_inter_head = num_inter_head
self.num_intra_head = num_intra_head
self.num_block = num_blocks
self.v_lin = FCNet(v_size, output_size, drop=drop)
self.q_lin = FCNet(q_size, output_size, drop=drop)
self.v2q_interBlock = OneSideInterModalityUpdate(
output_size, output_size, output_size, num_inter_head, drop)
self.q2v_interBlock = OneSideInterModalityUpdate(
output_size, output_size, output_size, num_inter_head, drop)
self.intraBlock = DyIntraModalityUpdate(output_size, output_size,
output_size, num_intra_head,
drop)
def __init__(self, src_size, tgt_size, output_size, num_head, drop=0.0):
super(OneSideInterModalityUpdate, self).__init__()
self.src_size = src_size
self.tgt_size = tgt_size
self.output_size = output_size
self.num_head = num_head
self.src_lin = FCNet(src_size, output_size * 2, drop=drop)
self.tgt_lin = FCNet(tgt_size, output_size, drop=drop)
self.tgt_output = FCNet(output_size + tgt_size, output_size, drop=drop)
def __init__(self, num_layers, hidden_size=256, input_size=341, output_size=4,drop=0.0):
super(SimpleDNN, self).__init__()
self.num_layers = num_layers
self.lin = FCNet(in_size=input_size, out_size=hidden_size, activate='relu', drop=drop)
layers = []
for i in range(num_layers):
layers.append(FCNet(in_size=hidden_size,out_size=hidden_size, activate='relu', drop=drop))
self.layers = nn.ModuleList(layers)
self.classfier = nn.Sequential(
FCNet(hidden_size, 512, activate='relu', drop=drop),
FCNet(512, output_size, drop=drop)
)
def __init__(self, in_features, mid_features, out_features, drop=0.0):
super(Classifier, self).__init__()
# define number of detector for each sentiment class
self.lin1 = FCNet(in_features,
mid_features,
activate='relu',
drop=drop)
self.lin2 = FCNet(mid_features, out_features, drop=drop)
#
self.bilinear = nn.Bilinear(in1_features=in_features,
in2_features=in_features,
out_features=in_features)
def __init__(self, v_size, q_size, output_size, num_head, drop=0.0):
super(InterModalityUpdate, self).__init__()
self.v_size = v_size
self.q_size = q_size
self.output_size = output_size
self.num_head = num_head
self.v_lin = FCNet(v_size, output_size * 3, drop=drop)
self.q_lin = FCNet(q_size, output_size * 3, drop=drop)
self.v_output = FCNet(output_size + v_size, output_size, drop=drop)
self.q_output = FCNet(output_size + q_size, output_size, drop=drop)
def __init__(self, in_features, mid_features, out_features, drop=0.0):
super(Senti_Map_Classifier, self).__init__()
# define number of detector for each sentiment class
self.k = 10
self.emotion_class = out_features
eeg_detectors = []
eye_detectors = []
for i in range(out_features):
eeg_detectors.append(
nn.Conv1d(in_channels=in_features,
out_channels=self.k,
kernel_size=1,
stride=1,
padding=0,
bias=True))
eye_detectors.append(
nn.Conv1d(in_channels=in_features,
out_channels=self.k,
kernel_size=1,
stride=1,
padding=0,
bias=True))
self.eeg_detectors = nn.ModuleList(eeg_detectors)
self.eye_detectors = nn.ModuleList(eye_detectors)
self.lin1 = FCNet(in_features * self.emotion_class,
mid_features,
activate='relu',
drop=drop)
self.lin2 = FCNet(mid_features, out_features, drop=drop)
emotion_classifer = []
self.bilinears = nn.ModuleList([
nn.Bilinear(in1_features=in_features,
in2_features=in_features,
out_features=in_features)
for _ in range(self.emotion_class)
])
for i in range(self.emotion_class):
emotion_classifer.append(
nn.Sequential(
nn.Dropout(p=drop),
FCNet(in_features,
mid_features,
activate='relu',
drop=drop), FCNet(mid_features, 1, drop=drop)))
self.emotion_classifer = nn.ModuleList(emotion_classifer)
def __init__(self, in_features, mid_features, out_features, drop=0.0):
super(Senti_Map_Classifier2, self).__init__()
# define number of detector for each sentiment class
self.k = 10
self.emotion_class = out_features
self.downconv = nn.Sequential(
nn.Conv2d(in_channels=1,
out_channels=self.emotion_class * self.k,
kernel_size=1,
stride=1,
padding=0,
bias=True))
self.GAP = nn.AvgPool2d(4)
self.lin1 = FCNet(in_features * self.emotion_class,
mid_features,
activate='relu',
drop=drop)
self.lin2 = FCNet(mid_features, out_features, drop=drop)
def __init__(self, v_size, q_size, output_size, num_head, drop=0.0):
super(DyIntraModalityUpdate, self).__init__()
self.v_size = v_size
self.q_size = q_size
self.output_size = output_size
self.num_head = num_head
self.v4q_gate_lin = FCNet(v_size, output_size, drop=drop)
self.q4v_gate_lin = FCNet(q_size, output_size, drop=drop)
self.v_lin = FCNet(v_size, output_size * 3, drop=drop)
self.q_lin = FCNet(q_size, output_size * 3, drop=drop)
self.v_output = FCNet(output_size, output_size, drop=drop)
self.q_output = FCNet(output_size, output_size, drop=drop)
self.relu = nn.ReLU()
self.tanh = nn.Tanh()
self.sigmoid = nn.Sigmoid()