def __init__(self, hyperparameters):
super(Model, self).__init__()
self.device = hyperparameters['device']
self.auxiliary_data_source = hyperparameters['auxiliary_data_source']
self.all_data_sources = ['resnet_features', self.auxiliary_data_source]
self.DATASET = hyperparameters['dataset']
self.num_shots = hyperparameters['num_shots']
self.latent_size = hyperparameters['latent_size']
self.batch_size = hyperparameters['batch_size']
self.hidden_size_rule = hyperparameters['hidden_size_rule']
self.warmup = hyperparameters['model_specifics']['warmup']
self.generalized = hyperparameters['generalized']
self.classifier_batch_size = 32
self.img_seen_samples = hyperparameters['samples_per_class'][
self.DATASET][0]
self.att_seen_samples = hyperparameters['samples_per_class'][
self.DATASET][1]
self.att_unseen_samples = hyperparameters['samples_per_class'][
self.DATASET][2]
self.img_unseen_samples = hyperparameters['samples_per_class'][
self.DATASET][3]
self.reco_loss_function = hyperparameters['loss']
self.nepoch = hyperparameters['epochs']
self.lr_cls = hyperparameters['lr_cls']
self.cross_reconstruction = hyperparameters['model_specifics'][
'cross_reconstruction']
self.cls_train_epochs = hyperparameters['cls_train_steps']
self.dataset = dataloader(self.DATASET,
copy.deepcopy(self.auxiliary_data_source),
device=self.device)
self.writer = SummaryWriter()
self.num_gen_iter = hyperparameters['num_gen_iter']
self.num_dis_iter = hyperparameters['num_dis_iter']
self.pretrain = hyperparameters['pretrain']
if self.DATASET == 'CUB':
self.num_classes = 200
self.num_novel_classes = 50
elif self.DATASET == 'SUN':
self.num_classes = 717
self.num_novel_classes = 72
elif self.DATASET == 'AWA1' or self.DATASET == 'AWA2':
self.num_classes = 50
self.num_novel_classes = 10
feature_dimensions = [2048, self.dataset.aux_data.size(1)]
# Here, the encoders and decoders for all modalities are created and put into dict
self.encoder = {}
for datatype, dim in zip(self.all_data_sources, feature_dimensions):
self.encoder[datatype] = models.encoder_template(
dim, self.latent_size, self.hidden_size_rule[datatype],
self.device)
print(str(datatype) + ' ' + str(dim))
print('latent size ' + str(self.latent_size))
self.decoder = {}
for datatype, dim in zip(self.all_data_sources, feature_dimensions):
self.decoder[datatype] = models.decoder_template(
self.latent_size, dim, self.hidden_size_rule[datatype],
self.device)
# An optimizer for all encoders and decoders is defined here
parameters_to_optimize = list(self.parameters())
for datatype in self.all_data_sources:
parameters_to_optimize += list(self.encoder[datatype].parameters())
parameters_to_optimize += list(self.decoder[datatype].parameters())
# The discriminator network is defined here
self.net_D_Att = models.Discriminator(
self.dataset.aux_data.size(1) + 2048, self.device)
self.net_D_Img = models.Discriminator(
2048 + self.dataset.aux_data.size(1), self.device)
self.optimizer_G = optim.Adam(parameters_to_optimize,
lr=hyperparameters['lr_gen_model'],
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=0.0005,
amsgrad=True)
self.optimizer_D = optim.Adam(itertools.chain(
self.net_D_Att.parameters(), self.net_D_Img.parameters()),
lr=hyperparameters['lr_gen_model'],
betas=(0.5, 0.999),
weight_decay=0.0005)
if self.reco_loss_function == 'l2':
self.reconstruction_criterion = nn.MSELoss(reduction='sum')
elif self.reco_loss_function == 'l1':
self.reconstruction_criterion = nn.L1Loss(reduction='sum')
self.MSE = nn.MSELoss(reduction='sum')
self.L1 = nn.L1Loss(reduction='sum')
self.att_fake_from_att_sample = utils.Sample_from_Pool()
self.att_fake_from_img_sample = utils.Sample_from_Pool()
self.img_fake_from_img_sample = utils.Sample_from_Pool()
self.img_fake_from_att_sample = utils.Sample_from_Pool()
if self.generalized:
print('mode: gzsl')
self.clf = LINEAR_LOGSOFTMAX(self.latent_size, self.num_classes)
else:
print('mode: zsl')
self.clf = LINEAR_LOGSOFTMAX(self.latent_size,
self.num_novel_classes)
def __init__(self,hyperparameters):
super(Model,self).__init__()
self.device = hyperparameters['device']
self.auxiliary_data_source = hyperparameters['auxiliary_data_source']
self.all_data_sources = ['resnet_features',self.auxiliary_data_source]
self.DATASET = hyperparameters['dataset']
self.num_shots = hyperparameters['num_shots']
self.latent_size = hyperparameters['latent_size']
self.batch_size = hyperparameters['batch_size']
self.hidden_size_rule = hyperparameters['hidden_size_rule']
self.warmup = hyperparameters['model_specifics']['warmup']
self.generalized = hyperparameters['generalized']
self.classifier_batch_size = 32
self.img_seen_samples = hyperparameters['samples_per_class'][self.DATASET][0]
self.att_seen_samples = hyperparameters['samples_per_class'][self.DATASET][1]
self.att_unseen_samples = hyperparameters['samples_per_class'][self.DATASET][2]
self.img_unseen_samples = hyperparameters['samples_per_class'][self.DATASET][3]
self.reco_loss_function = hyperparameters['loss']
self.nepoch = hyperparameters['epochs']
self.lr_cls = hyperparameters['lr_cls']
self.cross_reconstruction = hyperparameters['model_specifics']['cross_reconstruction']
self.cls_train_epochs = hyperparameters['cls_train_steps']
self.dataset = dataloader( self.DATASET, copy.deepcopy(self.auxiliary_data_source) , device= self.device )
if self.DATASET=='CUB':
self.num_classes=200
self.num_novel_classes = 50
elif self.DATASET=='SUN':
self.num_classes=717
self.num_novel_classes = 72
elif self.DATASET=='AWA1' or self.DATASET=='AWA2':
self.num_classes=50
self.num_novel_classes = 10
elif self.DATASET=='plant':
self.num_classes=38
self.num_novel_classes = 13
feature_dimensions = [2048, self.dataset.aux_data.size(1)]
# Here, the encoders and decoders for all modalities are created and put into dict
self.encoder = {}
for datatype, dim in zip(self.all_data_sources,feature_dimensions):
self.encoder[datatype] = models.encoder_template(dim,self.latent_size,self.hidden_size_rule[datatype],self.device)
print(str(datatype) + ' ' + str(dim))
self.decoder = {}
for datatype, dim in zip(self.all_data_sources,feature_dimensions):
self.decoder[datatype] = models.decoder_template(self.latent_size,dim,self.hidden_size_rule[datatype],self.device)
# An optimizer for all encoders and decoders is defined here
parameters_to_optimize = list(self.parameters())
for datatype in self.all_data_sources:
parameters_to_optimize += list(self.encoder[datatype].parameters())
parameters_to_optimize += list(self.decoder[datatype].parameters())
self.optimizer = optim.Adam( parameters_to_optimize ,lr=hyperparameters['lr_gen_model'], betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=True)
if self.reco_loss_function=='l2':
self.reconstruction_criterion = nn.MSELoss(size_average=False)
elif self.reco_loss_function=='l1':
self.reconstruction_criterion = nn.L1Loss(size_average=False)
def __init__(self, hyperparameters):
super(Model, self).__init__()
self.device = hyperparameters['device']
self.auxiliary_data_source = hyperparameters['auxiliary_data_source']
self.attr = hyperparameters['attr']
self.all_data_sources = ['resnet_features', 'attributes']
self.DATASET = hyperparameters['dataset']
self.num_shots = hyperparameters['num_shots']
self.latent_size = hyperparameters['latent_size']
self.batch_size = hyperparameters['batch_size']
self.hidden_size_rule = hyperparameters['hidden_size_rule']
self.warmup = hyperparameters['model_specifics']['warmup']
self.generalized = hyperparameters['generalized']
self.classifier_batch_size = 32
#self.img_seen_samples = hyperparameters['samples_per_class'][self.DATASET][0]
#self.att_seen_samples = hyperparameters['samples_per_class'][self.DATASET][1]
#self.att_unseen_samples = hyperparameters['samples_per_class'][self.DATASET][2]
# self.img_unseen_samples = hyperparameters['samples_per_class'][self.DATASET][3]
self.reco_loss_function = hyperparameters['loss']
self.margin = hyperparameters['margin_loss']
self.weight = torch.Tensor([hyperparameters['weight_loss']
]).to(self.device)
self.nepoch = hyperparameters['epochs']
self.lr_cls = hyperparameters['lr_cls']
self.cross_reconstruction = hyperparameters['model_specifics'][
'cross_reconstruction']
self.cls_train_epochs = hyperparameters['cls_train_steps']
#self.dataset = dataloader(self.DATASET, copy.deepcopy(self.auxiliary_data_source) , device= 'cuda')
self.dataset = dataLoader(copy.deepcopy(self.auxiliary_data_source),
device='cuda',
attr=self.attr)
if self.DATASET == 'CUB':
self.num_classes = 200
self.num_novel_classes = 50
elif self.DATASET == 'SUN':
self.num_classes = 717
self.num_novel_classes = 72
elif self.DATASET == 'AWA1' or self.DATASET == 'AWA2':
self.num_classes = 50
self.num_novel_classes = 10
if self.attr == 'attributes':
feature_dimensions = [2048, self.dataset.K]
elif self.attr == 'bert':
feature_dimensions = [2048, 768] #2048, 768
# Here, the encoders and decoders for all modalities are created and put into dict
self.fc_ft = nn.Linear(2048, 2048)
self.fc_ft.to(self.device)
self.ft_bn = nn.BatchNorm1d(2048).to(self.device)
self.fc_at = nn.Linear(self.dataset.K, self.dataset.K)
self.fc_at.to(self.device)
self.at_bn = nn.BatchNorm1d(self.dataset.K).to(self.device)
self.encoder = {}
for datatype, dim in zip(self.all_data_sources, feature_dimensions):
self.encoder[datatype] = models.encoder_template(
dim, self.latent_size, self.hidden_size_rule[datatype],
self.device)
print(str(datatype) + ' ' + str(dim))
self.decoder = {}
for datatype, dim in zip(self.all_data_sources, feature_dimensions):
self.decoder[datatype] = models.decoder_template(
self.latent_size, dim, self.hidden_size_rule[datatype],
self.device)
# An optimizer for all encoders and decoders is defined here
parameters_to_optimize = list(self.parameters())
for datatype in self.all_data_sources:
parameters_to_optimize += list(self.encoder[datatype].parameters())
parameters_to_optimize += list(self.decoder[datatype].parameters())
parameters_to_optimize += list(self.fc_ft.parameters())
parameters_to_optimize += list(self.fc_at.parameters())
parameters_to_optimize += list(self.ft_bn.parameters())
parameters_to_optimize += list(self.at_bn.parameters())
self.optimizer = optim.Adam(parameters_to_optimize,
lr=hyperparameters['lr_gen_model'],
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=0,
amsgrad=True)
if self.reco_loss_function == 'l2':
self.reconstruction_criterion = nn.MSELoss(size_average=False)
elif self.reco_loss_function == 'l1':
self.reconstruction_criterion = nn.L1Loss(size_average=False)
self.triplet_loss = nn.TripletMarginLoss(margin=self.margin)