def __init__(self, num_classes=61, mem_size=512, rgbm=None, fcm=None): super(bigConvLSTM, self).__init__() self.num_classes = num_classes self.mem_size = mem_size self.resNetRGB = resnetMod.resnet34(True, True) self.resNetCol = resnetMod.resnet34(True, True) self.lstm_cell = MyConvLSTMCell(512 * 2, mem_size) if rgbm is not None: model = attentionModel(num_classes, mem_size) model.load_state_dict( torch.load(rgbm, map_location=torch.device('cpu'))) self.resNetRGB.load_state_dict(model.resNet.state_dict()) #self.lstm_cell.load_state_dict(model.lstm_cell) if fcm is not None: model = noAttentionModel(num_classes, mem_size) model.load_state_dict( torch.load(fcm, map_location=torch.device('cpu'))) self.resNetCol.load_state_dict(model.resNet.state_dict()) #self.lstm_cell_y.load_state_dict(model.lstm_cell) self.weight_softmax = self.resNetRGB.fc.weight self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) # 0.7 self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self, num_classes=61, mem_size=512, regression=True, seqLen=7): super(attentionModel, self).__init__() self.regression = regression self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) #MS net #if MsFlag == True: #self.relu = F.relu() self.convMS = nn.Conv2d(512, 100, kernel_size=1, padding=0) #view if regression == False: self.MSfc = nn.Linear(100 * 7 * 7, 2 * 7 * 7) #view self.m = nn.Softmax(2) self.orderPredictionNetwork = OrderPredictionNetwork(numFrame=4, seqLen=seqLen) else: self.MSfc = nn.Linear(100 * 7 * 7, 1 * 7 * 7) #else : self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self, num_classes=61, mem_size=512): super(attentionModel, self).__init__() self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc) #Secondary task branch for flowx self.xPredictor = nn.Sequential() self.xPredictor.add_module('mmap_relu', nn.ReLU(True)) self.xPredictor.add_module('convolution', nn.Conv2d(512, 100, kernel_size=1)) self.xPredictor.add_module('flatten', Flatten()) self.xPredictor.add_module('fc_2', nn.Linear(100 * 7 * 7, 7 * 7)) self.yPredictor = nn.Sequential() self.yPredictor.add_module('mmap_relu', nn.ReLU(True)) self.yPredictor.add_module('convolution', nn.Conv2d(512, 100, kernel_size=1)) self.yPredictor.add_module('flatten', Flatten()) self.yPredictor.add_module('fc_2', nn.Linear(100 * 7 * 7, 7 * 7))
def __init__(self, num_classes=61, mem_size=512, regressor=False): super(attentionModel_ml, self).__init__() self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc) self.regressor = regressor #msNet self.conv = nn.Sequential( nn.ReLU(inplace=True), nn.Conv2d(512, 100, 1, stride=1, padding=0, dilation=1, groups=1, bias=True)) if regressor == 0: self.clas = nn.Linear(7 * 7 * 100, 2 * 49) self.soft = nn.Softmax(2) elif regressor == 1: self.clas = nn.Sequential(nn.Linear(7 * 7 * 100, 49))
def __init__(self, num_classes=61, mem_size=512): super(attentionModel, self).__init__() self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self, num_classes=61, mem_size=512, rgbm=None, fcm=None): super(attentionDoubleResnet, self).__init__() self.num_classes = num_classes self.resNet1 = resnetMod.resnet34(True, True) if rgbm is not None: model = attentionModel(num_classes, mem_size) model.load_state_dict(torch.load(rgbm)) self.resNet1.load_state_dict(model.resNet.state_dict()) self.resNet2 = resnetMod.resnet34(True, True) if fcm is not None: model = noAttentionModel(num_classes, mem_size) model.load_state_dict(torch.load(rgbm)) self.resNet2.load_state_dict(model.resNet.state_dict()) self.mem_size = mem_size self.weight_softmax = self.resNet1.fc.weight self.lstm_cell = MyConvLSTMCell(512 * 2, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(2 * mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self, num_classes=61, mem_size=512): super(ConvLSTMDynamic, self).__init__() self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc) self.dinam = DynamicFilters(512, 3 * 3) self.conv = nn.Sequential(nn.Conv2d(512, 512, kernel_size=1, padding=0), nn.ReLU())
def __init__(self, num_classes=61, mem_size=512, cam=True, n_channels=2): super(SelfSupervisedAttentionModel, self).__init__() self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc) self.cam = cam # Motion Segmentation Module self.ms_module = MotionSegmentationModule(512, n_channels=n_channels)
def __init__(self, supervision, num_classes=61, mem_size=512, loss_supervision="classification"): super(ConvLSTMAttention, self).__init__() self.supervision = supervision self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc) if loss_supervision == "regression": self.sup_head = SupervisionHead(512, 100, 7, 7, 1) if loss_supervision == "classification": self.sup_head = SupervisionHead(512, 100, 7, 7, 2)
def __init__(self, num_classes=61, mem_size=512, regression=True): super(attentionModel, self).__init__() self.regression = regression self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) self.convMS = nn.Conv2d(512, 100, kernel_size=1, padding=0) if regression == False: self.MSfc = nn.Linear(100 * 7 * 7, 2 * 7 * 7) self.m = nn.Softmax(2) else: self.MSfc = nn.Linear(100 * 7 * 7, 1 * 7 * 7) self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc)
def __init__(self, num_classes=61, mem_size=512, cam=True, ms=False, ms_task="classifier", ordering=True, order_type="rgb", perm_tuple_length=3, num_perms=10, getBestPermutation=False): # order_type "rgb" or "of" super(attentionModel, self).__init__() self.num_classes = num_classes self.resNet = resnetMod.resnet34(True, True) self.mem_size = mem_size self.weight_softmax = self.resNet.fc.weight self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc) self.compute_cam = cam self.include_ms = ms self.ms_task = ms_task self.include_ordering = ordering self.getBestPermutation = getBestPermutation if ms: self.ms_conv = nn.Sequential(nn.ReLU(), nn.Conv2d(512, 100, 1)) if self.ms_task == "classifier": # ms_task either classifier or regressor self.ms_classifier = nn.Sequential(nn.Linear(4900, 98)) else: # regressor self.ms_classifier = nn.Sequential(nn.Linear(4900, 49)) # 4900 = 100*7*7, 98 = 2*7*7 # for classification, fc outputs 2*49 (49 samples, 2 classes) and forwards it to softmax to compute confidence scores (or does it?) # for regression, fc outputs the predictions straight away (49 samples) if self.include_ordering: self.ordering_lstm_cell = MyConvLSTMCell(512, mem_size) self.order_type = order_type self.perm_tuple_length = perm_tuple_length self.num_perms = num_perms self.order_classifier = nn.Sequential( nn.Dropout(0.7), nn.Linear(mem_size, self.perm_tuple_length))
def __init__(self, flowModel='', frameModel='', num_classes=61, mem_size=512, attention=1): super(attentionModel_flow, self).__init__() self.num_classes = num_classes self.attention = attention self.resNetRGB = resnetMod.resnet34(True, True) if frameModel != '': self.resNetRGB.load_state_dict(OnlyResNet(torch.load(frameModel))) self.flowResNet = flow_resnet.flow_resnet34(True, channels=2, num_classes=num_classes) self.mem_size = mem_size self.lstm_cell = MyConvLSTMCell(512, mem_size) self.avgpool = nn.AvgPool2d(7) self.dropout = nn.Dropout(0.7) self.fc = nn.Linear(mem_size, self.num_classes) self.classifier = nn.Sequential(self.dropout, self.fc)