def __init__(self, num_classes=61, mem_size=512, rgbm=None, fcm=None):
super(bigConvLSTM, self).__init__()
self.num_classes = num_classes
self.mem_size = mem_size
self.resNetRGB = resnetMod.resnet34(True, True)
self.resNetCol = resnetMod.resnet34(True, True)
self.lstm_cell = MyConvLSTMCell(512 * 2, mem_size)
if rgbm is not None:
model = attentionModel(num_classes, mem_size)
model.load_state_dict(
torch.load(rgbm, map_location=torch.device('cpu')))
self.resNetRGB.load_state_dict(model.resNet.state_dict())
#self.lstm_cell.load_state_dict(model.lstm_cell)
if fcm is not None:
model = noAttentionModel(num_classes, mem_size)
model.load_state_dict(
torch.load(fcm, map_location=torch.device('cpu')))
self.resNetCol.load_state_dict(model.resNet.state_dict())
#self.lstm_cell_y.load_state_dict(model.lstm_cell)
self.weight_softmax = self.resNetRGB.fc.weight
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7) # 0.7
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self,
num_classes=61,
mem_size=512,
regression=True,
seqLen=7):
super(attentionModel, self).__init__()
self.regression = regression
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
#MS net
#if MsFlag == True:
#self.relu = F.relu()
self.convMS = nn.Conv2d(512, 100, kernel_size=1, padding=0)
#view
if regression == False:
self.MSfc = nn.Linear(100 * 7 * 7, 2 * 7 * 7)
#view
self.m = nn.Softmax(2)
self.orderPredictionNetwork = OrderPredictionNetwork(numFrame=4,
seqLen=seqLen)
else:
self.MSfc = nn.Linear(100 * 7 * 7, 1 * 7 * 7)
#else :
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self, num_classes=61, mem_size=512):
super(attentionModel, self).__init__()
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
#Secondary task branch for flowx
self.xPredictor = nn.Sequential()
self.xPredictor.add_module('mmap_relu', nn.ReLU(True))
self.xPredictor.add_module('convolution',
nn.Conv2d(512, 100, kernel_size=1))
self.xPredictor.add_module('flatten', Flatten())
self.xPredictor.add_module('fc_2', nn.Linear(100 * 7 * 7, 7 * 7))
self.yPredictor = nn.Sequential()
self.yPredictor.add_module('mmap_relu', nn.ReLU(True))
self.yPredictor.add_module('convolution',
nn.Conv2d(512, 100, kernel_size=1))
self.yPredictor.add_module('flatten', Flatten())
self.yPredictor.add_module('fc_2', nn.Linear(100 * 7 * 7, 7 * 7))
def __init__(self, num_classes=61, mem_size=512, regressor=False):
super(attentionModel_ml, self).__init__()
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
self.regressor = regressor
#msNet
self.conv = nn.Sequential(
nn.ReLU(inplace=True),
nn.Conv2d(512,
100,
1,
stride=1,
padding=0,
dilation=1,
groups=1,
bias=True))
if regressor == 0:
self.clas = nn.Linear(7 * 7 * 100, 2 * 49)
self.soft = nn.Softmax(2)
elif regressor == 1:
self.clas = nn.Sequential(nn.Linear(7 * 7 * 100, 49))
def __init__(self, num_classes=61, mem_size=512):
super(attentionModel, self).__init__()
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self, num_classes=61, mem_size=512, rgbm=None, fcm=None):
super(attentionDoubleResnet, self).__init__()
self.num_classes = num_classes
self.resNet1 = resnetMod.resnet34(True, True)
if rgbm is not None:
model = attentionModel(num_classes, mem_size)
model.load_state_dict(torch.load(rgbm))
self.resNet1.load_state_dict(model.resNet.state_dict())
self.resNet2 = resnetMod.resnet34(True, True)
if fcm is not None:
model = noAttentionModel(num_classes, mem_size)
model.load_state_dict(torch.load(rgbm))
self.resNet2.load_state_dict(model.resNet.state_dict())
self.mem_size = mem_size
self.weight_softmax = self.resNet1.fc.weight
self.lstm_cell = MyConvLSTMCell(512 * 2, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(2 * mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self, num_classes=61, mem_size=512):
super(ConvLSTMDynamic, self).__init__()
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
self.dinam = DynamicFilters(512, 3 * 3)
self.conv = nn.Sequential(nn.Conv2d(512, 512, kernel_size=1, padding=0),
nn.ReLU())
def __init__(self, num_classes=61, mem_size=512, cam=True, n_channels=2):
super(SelfSupervisedAttentionModel, self).__init__()
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
self.cam = cam
# Motion Segmentation Module
self.ms_module = MotionSegmentationModule(512, n_channels=n_channels)
def __init__(self, supervision, num_classes=61, mem_size=512, loss_supervision="classification"):
super(ConvLSTMAttention, self).__init__()
self.supervision = supervision
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
if loss_supervision == "regression":
self.sup_head = SupervisionHead(512, 100, 7, 7, 1)
if loss_supervision == "classification":
self.sup_head = SupervisionHead(512, 100, 7, 7, 2)
def __init__(self, num_classes=61, mem_size=512, regression=True):
super(attentionModel, self).__init__()
self.regression = regression
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
self.convMS = nn.Conv2d(512, 100, kernel_size=1, padding=0)
if regression == False:
self.MSfc = nn.Linear(100 * 7 * 7, 2 * 7 * 7)
self.m = nn.Softmax(2)
else:
self.MSfc = nn.Linear(100 * 7 * 7, 1 * 7 * 7)
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self,
num_classes=61,
mem_size=512,
cam=True,
ms=False,
ms_task="classifier",
ordering=True,
order_type="rgb",
perm_tuple_length=3,
num_perms=10,
getBestPermutation=False): # order_type "rgb" or "of"
super(attentionModel, self).__init__()
self.num_classes = num_classes
self.resNet = resnetMod.resnet34(True, True)
self.mem_size = mem_size
self.weight_softmax = self.resNet.fc.weight
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
self.compute_cam = cam
self.include_ms = ms
self.ms_task = ms_task
self.include_ordering = ordering
self.getBestPermutation = getBestPermutation
if ms:
self.ms_conv = nn.Sequential(nn.ReLU(), nn.Conv2d(512, 100, 1))
if self.ms_task == "classifier": # ms_task either classifier or regressor
self.ms_classifier = nn.Sequential(nn.Linear(4900, 98))
else: # regressor
self.ms_classifier = nn.Sequential(nn.Linear(4900, 49))
# 4900 = 100*7*7, 98 = 2*7*7
# for classification, fc outputs 2*49 (49 samples, 2 classes) and forwards it to softmax to compute confidence scores (or does it?)
# for regression, fc outputs the predictions straight away (49 samples)
if self.include_ordering:
self.ordering_lstm_cell = MyConvLSTMCell(512, mem_size)
self.order_type = order_type
self.perm_tuple_length = perm_tuple_length
self.num_perms = num_perms
self.order_classifier = nn.Sequential(
nn.Dropout(0.7), nn.Linear(mem_size, self.perm_tuple_length))
def __init__(self,
flowModel='',
frameModel='',
num_classes=61,
mem_size=512,
attention=1):
super(attentionModel_flow, self).__init__()
self.num_classes = num_classes
self.attention = attention
self.resNetRGB = resnetMod.resnet34(True, True)
if frameModel != '':
self.resNetRGB.load_state_dict(OnlyResNet(torch.load(frameModel)))
self.flowResNet = flow_resnet.flow_resnet34(True,
channels=2,
num_classes=num_classes)
self.mem_size = mem_size
self.lstm_cell = MyConvLSTMCell(512, mem_size)
self.avgpool = nn.AvgPool2d(7)
self.dropout = nn.Dropout(0.7)
self.fc = nn.Linear(mem_size, self.num_classes)
self.classifier = nn.Sequential(self.dropout, self.fc)
