class FoodIngredients(Network):
def __init__(self,
model_name='DenseNet',
model_type='food',
lr=0.02,
optimizer_name='Adam',
criterion1=nn.CrossEntropyLoss(),
criterion2=nn.BCEWithLogitsLoss(),
dropout_p=0.45,
pretrained=True,
device=None,
best_accuracy=0.,
best_validation_loss=None,
best_model_file='best_model.pth',
head1={
'num_outputs': 10,
'layers': [],
'model_type': 'classifier'
},
head2={
'num_outputs': 10,
'layers': [],
'model_type': 'multi_label_classifier'
},
class_names=[],
num_classes=None,
ingredient_names=[],
num_ingredients=None,
add_extra=True,
set_params=True,
set_head=True):
super().__init__(device=device)
self.set_transfer_model(model_name,
pretrained=pretrained,
add_extra=add_extra,
dropout_p=dropout_p)
if set_head:
self.set_model_head(model_name=model_name,
head1=head1,
head2=head2,
dropout_p=dropout_p,
criterion1=criterion1,
criterion2=criterion2,
device=device)
if set_params:
self.set_model_params(
optimizer_name=optimizer_name,
lr=lr,
dropout_p=dropout_p,
model_name=model_name,
model_type=model_type,
best_accuracy=best_accuracy,
best_validation_loss=best_validation_loss,
best_model_file=best_model_file,
class_names=class_names,
num_classes=num_classes,
ingredient_names=ingredient_names,
num_ingredients=num_ingredients,
)
self.model = self.model.to(device)
def set_model_params(self,
criterion1=nn.CrossEntropyLoss(),
criterion2=nn.BCEWithLogitsLoss(),
optimizer_name='Adam',
lr=0.1,
dropout_p=0.45,
model_name='DenseNet',
model_type='cv_transfer',
best_accuracy=0.,
best_validation_loss=None,
best_model_file='best_model_file.pth',
head1={
'num_outputs': 10,
'layers': [],
'model_type': 'classifier'
},
head2={
'num_outputs': 10,
'layers': [],
'model_type': 'muilti_label_classifier'
},
class_names=[],
num_classes=None,
ingredient_names=[],
num_ingredients=None):
print(
'Food Names: current best accuracy = {:.3f}'.format(best_accuracy))
if best_validation_loss is not None:
print('Food Ingredients: current best loss = {:.3f}'.format(
best_validation_loss))
super(FoodIngredients,
self).set_model_params(optimizer_name=optimizer_name,
lr=lr,
dropout_p=dropout_p,
model_name=model_name,
model_type=model_type,
best_accuracy=best_accuracy,
best_validation_loss=best_validation_loss,
best_model_file=best_model_file)
self.class_names = class_names
self.num_classes = num_classes
self.ingredeint_names = ingredient_names
self.num_ingredients = num_ingredients
self.criterion1 = criterion1
self.criterion2 = criterion2
def forward(self, x):
l = list(self.model.children())
for m in l[:-2]:
x = m(x)
food = l[-2](x)
ingredients = l[-1](x)
return (food, ingredients)
def compute_loss(self, outputs, labels, w1=1., w2=1.):
out1, out2 = outputs
label1, label2 = labels
loss1 = self.criterion1(out1, label1)
loss2 = self.criterion2(out2, label2)
return [(loss1 * w1) + (loss2 * w2)]
def freeze(self, train_classifier=True):
super(FoodIngredients, self).freeze()
if train_classifier:
for param in self.model.fc1.parameters():
param.requires_grad = True
for param in self.model.fc2.parameters():
param.requires_grad = True
def parallelize(self):
self.parallel = True
self.model = DataParallelModel(self.model)
self.criterion = DataParallelCriterion(self.criterion)
def set_transfer_model(self,
mname,
pretrained=True,
add_extra=True,
dropout_p=0.45):
self.model = None
models_dict = {
'densenet': {
'model': models.densenet121(pretrained=pretrained),
'conv_channels': 1024
},
'resnet34': {
'model': models.resnet34(pretrained=pretrained),
'conv_channels': 512
},
'resnet50': {
'model': models.resnet50(pretrained=pretrained),
'conv_channels': 2048
}
}
meta = models_dict[mname.lower()]
try:
model = meta['model']
for param in model.parameters():
param.requires_grad = False
self.model = model
print(
'Setting transfer learning model: self.model set to {}'.format(
mname))
except:
print(
'Setting transfer learning model: model name {} not supported'.
format(mname))
# creating and adding extra layers to the model
dream_model = None
if add_extra:
channels = meta['conv_channels']
dream_model = nn.Sequential(
nn.Conv2d(channels, channels, 3, 1, 1),
# Printer(),
nn.BatchNorm2d(channels),
nn.ReLU(True),
nn.Dropout2d(dropout_p),
nn.Conv2d(channels, channels, 3, 1, 1),
nn.BatchNorm2d(channels),
nn.ReLU(True),
nn.Dropout2d(dropout_p),
nn.Conv2d(channels, channels, 3, 1, 1),
nn.BatchNorm2d(channels),
nn.ReLU(True),
nn.Dropout2d(dropout_p))
self.dream_model = dream_model
def set_model_head(
self,
model_name='DenseNet',
head1={
'num_outputs': 10,
'layers': [],
'class_names': None,
'model_type': 'classifier'
},
head2={
'num_outputs': 10,
'layers': [],
'class_names': None,
'model_type': 'muilti_label_classifier'
},
criterion1=nn.CrossEntropyLoss(),
criterion2=nn.BCEWithLogitsLoss(),
adaptive=True,
dropout_p=0.45,
device=None):
models_meta = {
'resnet34': {
'conv_channels': 512,
'head_id': -2,
'adaptive_head': [DAI_AvgPool],
'normal_head': [nn.AvgPool2d(7, 1)]
},
'resnet50': {
'conv_channels': 2048,
'head_id': -2,
'adaptive_head': [DAI_AvgPool],
'normal_head': [nn.AvgPool2d(7, 1)]
},
'densenet': {
'conv_channels': 1024,
'head_id': -1,
'adaptive_head': [nn.ReLU(inplace=True), DAI_AvgPool],
'normal_head': [nn.ReLU(inplace=True),
nn.AvgPool2d(7, 1)]
}
}
name = model_name.lower()
meta = models_meta[name]
modules = list(self.model.children())
l = modules[:meta['head_id']]
if self.dream_model:
l += self.dream_model
heads = [head1, head2]
crits = [criterion1, criterion2]
fcs = []
for head, criterion in zip(heads, crits):
head['criterion'] = criterion
if head['model_type'].lower() == 'classifier':
head['output_non_linearity'] = None
fc = modules[-1]
try:
in_features = fc.in_features
except:
in_features = fc.model.out.in_features
fc = FC(num_inputs=in_features,
num_outputs=head['num_outputs'],
layers=head['layers'],
model_type=head['model_type'],
output_non_linearity=head['output_non_linearity'],
dropout_p=dropout_p,
criterion=head['criterion'],
optimizer_name=None,
device=device)
fcs.append(fc)
if adaptive:
l += meta['adaptive_head']
else:
l += meta['normal_head']
model = nn.Sequential(*l)
model.add_module('fc1', fcs[0])
model.add_module('fc2', fcs[1])
self.model = model
self.head1 = head1
self.head2 = head2
print('Multi-head set up complete.')
def train_(self, e, trainloader, optimizer, print_every):
epoch, epochs = e
self.train()
t0 = time.time()
t1 = time.time()
batches = 0
running_loss = 0.
for data_batch in trainloader:
inputs, label1, label2 = data_batch[0], data_batch[1], data_batch[
2]
batches += 1
inputs = inputs.to(self.device)
label1 = label1.to(self.device)
label2 = label2.to(self.device)
labels = (label1, label2)
optimizer.zero_grad()
outputs = self.forward(inputs)
loss = self.compute_loss(outputs, labels)[0]
if self.parallel:
loss.sum().backward()
loss = loss.sum()
else:
loss.backward()
loss = loss.item()
optimizer.step()
running_loss += loss
if batches % print_every == 0:
elapsed = time.time() - t1
if elapsed > 60:
elapsed /= 60.
measure = 'min'
else:
measure = 'sec'
batch_time = time.time() - t0
if batch_time > 60:
batch_time /= 60.
measure2 = 'min'
else:
measure2 = 'sec'
print(
'+----------------------------------------------------------------------+\n'
f"{time.asctime().split()[-2]}\n"
f"Time elapsed: {elapsed:.3f} {measure}\n"
f"Epoch:{epoch+1}/{epochs}\n"
f"Batch: {batches+1}/{len(trainloader)}\n"
f"Batch training time: {batch_time:.3f} {measure2}\n"
f"Batch training loss: {loss:.3f}\n"
f"Average training loss: {running_loss/(batches):.3f}\n"
'+----------------------------------------------------------------------+\n'
)
t0 = time.time()
return running_loss / len(trainloader)
def evaluate(self, dataloader, metric='accuracy'):
running_loss = 0.
classifier = None
if self.model_type == 'classifier': # or self.num_classes is not None:
classifier = Classifier(self.class_names)
y_pred = []
y_true = []
self.eval()
rmse_ = 0.
with torch.no_grad():
for data_batch in dataloader:
inputs, label1, label2 = data_batch[0], data_batch[
1], data_batch[2]
inputs = inputs.to(self.device)
label1 = label1.to(self.device)
label2 = label2.to(self.device)
labels = (label1, label2)
outputs = self.forward(inputs)
loss = self.compute_loss(outputs, labels)[0]
if self.parallel:
running_loss += loss.sum()
outputs = parallel.gather(outputs, self.device)
else:
running_loss += loss.item()
if classifier is not None and metric == 'accuracy':
classifier.update_accuracies(outputs, labels)
y_true.extend(list(labels.squeeze(0).cpu().numpy()))
_, preds = torch.max(torch.exp(outputs), 1)
y_pred.extend(list(preds.cpu().numpy()))
elif metric == 'rmse':
rmse_ += rmse(outputs, labels).cpu().numpy()
self.train()
ret = {}
# print('Running_loss: {:.3f}'.format(running_loss))
if metric == 'rmse':
print('Total rmse: {:.3f}'.format(rmse_))
ret['final_rmse'] = rmse_ / len(dataloader)
ret['final_loss'] = running_loss / len(dataloader)
if classifier is not None:
ret['accuracy'], ret[
'class_accuracies'] = classifier.get_final_accuracies()
ret['report'] = classification_report(
y_true, y_pred, target_names=self.class_names)
ret['confusion_matrix'] = confusion_matrix(y_true, y_pred)
try:
ret['roc_auc_score'] = roc_auc_score(y_true, y_pred)
except:
pass
return ret
def evaluate_food(self, dataloader, metric='accuracy'):
running_loss = 0.
classifier = None
classifier = Classifier(self.class_names)
y_pred = []
y_true = []
self.eval()
rmse_ = 0.
with torch.no_grad():
for data_batch in dataloader:
inputs, labels = data_batch[0], data_batch[1]
inputs = inputs.to(self.device)
labels = labels.to(self.device)
outputs = self.forward(inputs)[0]
if classifier is not None and metric == 'accuracy':
try:
classifier.update_accuracies(outputs, labels)
y_true.extend(list(labels.squeeze(0).cpu().numpy()))
_, preds = torch.max(torch.exp(outputs), 1)
y_pred.extend(list(preds.cpu().numpy()))
except:
pass
elif metric == 'rmse':
rmse_ += rmse(outputs, labels).cpu().numpy()
self.train()
ret = {}
# print('Running_loss: {:.3f}'.format(running_loss))
if metric == 'rmse':
print('Total rmse: {:.3f}'.format(rmse_))
ret['final_rmse'] = rmse_ / len(dataloader)
ret['final_loss'] = running_loss / len(dataloader)
if classifier is not None:
ret['accuracy'], ret[
'class_accuracies'] = classifier.get_final_accuracies()
ret['report'] = classification_report(
y_true, y_pred, target_names=self.class_names)
ret['confusion_matrix'] = confusion_matrix(y_true, y_pred)
try:
ret['roc_auc_score'] = roc_auc_score(y_true, y_pred)
except:
pass
return ret
def find_lr(self,
trn_loader,
init_value=1e-8,
final_value=10.,
beta=0.98,
plot=False):
print('\nFinding the ideal learning rate.')
model_state = copy.deepcopy(self.model.state_dict())
optim_state = copy.deepcopy(self.optimizer.state_dict())
optimizer = self.optimizer
num = len(trn_loader) - 1
mult = (final_value / init_value)**(1 / num)
lr = init_value
optimizer.param_groups[0]['lr'] = lr
avg_loss = 0.
best_loss = 0.
batch_num = 0
losses = []
log_lrs = []
for data_batch in trn_loader:
batch_num += 1
inputs, label1, label2 = data_batch[0], data_batch[1], data_batch[
2]
inputs = inputs.to(self.device)
label1 = label1.to(self.device)
label2 = label2.to(self.device)
labels = (label1, label2)
optimizer.zero_grad()
outputs = self.forward(inputs)
loss = self.compute_loss(outputs, labels)[0]
#Compute the smoothed loss
if self.parallel:
avg_loss = beta * avg_loss + (1 - beta) * loss.sum()
else:
avg_loss = beta * avg_loss + (1 - beta) * loss.item()
smoothed_loss = avg_loss / (1 - beta**batch_num)
#Stop if the loss is exploding
if batch_num > 1 and smoothed_loss > 4 * best_loss:
self.log_lrs, self.find_lr_losses = log_lrs, losses
self.model.load_state_dict(model_state)
self.optimizer.load_state_dict(optim_state)
if plot:
self.plot_find_lr()
temp_lr = self.log_lrs[np.argmin(self.find_lr_losses) -
(len(self.log_lrs) // 8)]
self.lr = (10**temp_lr)
print('Found it: {}\n'.format(self.lr))
return self.lr
#Record the best loss
if smoothed_loss < best_loss or batch_num == 1:
best_loss = smoothed_loss
#Store the values
losses.append(smoothed_loss)
log_lrs.append(math.log10(lr))
#Do the SGD step
if self.parallel:
loss.sum().backward()
else:
loss.backward()
optimizer.step()
#Update the lr for the next step
lr *= mult
optimizer.param_groups[0]['lr'] = lr
self.log_lrs, self.find_lr_losses = log_lrs, losses
self.model.load_state_dict(model_state)
self.optimizer.load_state_dict(optim_state)
if plot:
self.plot_find_lr()
temp_lr = self.log_lrs[np.argmin(self.find_lr_losses) -
(len(self.log_lrs) // 10)]
self.lr = (10**temp_lr)
print('Found it: {}\n'.format(self.lr))
return self.lr
def plot_find_lr(self):
plt.ylabel("Loss")
plt.xlabel("Learning Rate (log scale)")
plt.plot(self.log_lrs, self.find_lr_losses)
plt.show()
def classify(self,
inputs,
thresh=0.4): #,show = False,mean = None,std = None):
outputs = self.predict(inputs)
food, ing = outputs
try:
_, preds = torch.max(torch.exp(food), 1)
except:
_, preds = torch.max(torch.exp(food.unsqueeze(0)), 1)
ing_outs = ing.sigmoid()
ings = (ing_outs >= thresh)
class_preds = [str(self.class_names[p]) for p in preds]
ing_preds = [
self.ingredeint_names[p.nonzero().squeeze(1).cpu()] for p in ings
]
return class_preds, ing_preds
def _get_dropout(self):
return self.dropout_p
def get_model_params(self):
params = super(FoodIngredients, self).get_model_params()
params['class_names'] = self.class_names
params['num_classes'] = self.num_classes
params['ingredient_names'] = self.ingredient_names
params['num_ingredients'] = self.num_ingredients
params['head1'] = self.head1
params['head2'] = self.head2
return params
class TransferNetworkImg(Network):
def __init__(self,
model_name='DenseNet',
model_type='cv_transfer',
lr=0.02,
criterion=nn.CrossEntropyLoss(),
optimizer_name='Adam',
dropout_p=0.45,
pretrained=True,
device=None,
best_accuracy=0.,
best_validation_loss=None,
best_model_file='best_model.pth',
head={
'num_outputs': 10,
'layers': [],
'model_type': 'classifier'
},
class_names=[],
num_classes=None,
add_extra=True,
set_params=True,
set_head=True):
super().__init__(device=device)
self.set_transfer_model(model_name,
pretrained=pretrained,
add_extra=add_extra,
dropout_p=dropout_p)
if set_head:
self.set_model_head(model_name=model_name,
head=head,
dropout_p=dropout_p,
criterion=criterion,
device=device)
if set_params:
self.set_model_params(criterion=criterion,
optimizer_name=optimizer_name,
lr=lr,
dropout_p=dropout_p,
model_name=model_name,
model_type=model_type,
best_accuracy=best_accuracy,
best_validation_loss=best_validation_loss,
best_model_file=best_model_file,
class_names=class_names,
num_classes=num_classes)
self.model = self.model.to(device)
def set_model_params(self,
criterion=nn.CrossEntropyLoss(),
optimizer_name='Adam',
lr=0.1,
dropout_p=0.45,
model_name='DenseNet',
model_type='cv_transfer',
best_accuracy=0.,
best_validation_loss=None,
best_model_file='best_model_file.pth',
class_names=[],
num_classes=None):
print('Transfer Learning: current best accuracy = {:.3f}'.format(
best_accuracy))
super(TransferNetworkImg,
self).set_model_params(criterion=criterion,
optimizer_name=optimizer_name,
lr=lr,
dropout_p=dropout_p,
model_name=model_name,
model_type=model_type,
best_accuracy=best_accuracy,
best_validation_loss=best_validation_loss,
best_model_file=best_model_file)
self.class_names = class_names
self.num_classes = num_classes
if len(class_names) == 0:
self.class_names = {
k: str(v)
for k, v in enumerate(list(range(self.head['num_outputs'])))
}
def forward(self, x):
return self.model(x)
def freeze(self, train_classifier=True):
super(TransferNetworkImg, self).freeze()
if train_classifier:
for param in self.model.fc.parameters():
param.requires_grad = True
def parallelize(self):
self.parallel = True
self.model = DataParallelModel(self.model)
self.criterion = DataParallelCriterion(self.criterion)
def set_transfer_model(self,
mname,
pretrained=True,
add_extra=True,
dropout_p=0.45):
self.model = None
models_dict = {
'densenet': {
'model': models.densenet121(pretrained=pretrained),
'conv_channels': 1024
},
'resnet34': {
'model': models.resnet34(pretrained=pretrained),
'conv_channels': 512
},
'resnet50': {
'model': models.resnet50(pretrained=pretrained),
'conv_channels': 2048
}
}
meta = models_dict[mname.lower()]
try:
model = meta['model']
for param in model.parameters():
param.requires_grad = False
self.model = model
print(
'Setting transfer learning model: self.model set to {}'.format(
mname))
except:
print(
'Setting transfer learning model: model name {} not supported'.
format(mname))
# creating and adding extra layers to the model
dream_model = None
if add_extra:
channels = meta['conv_channels']
dream_model = nn.Sequential(
nn.Conv2d(channels, channels, 3, 1, 1),
# Printer(),
nn.BatchNorm2d(channels),
nn.ReLU(True),
nn.Dropout2d(dropout_p),
nn.Conv2d(channels, channels, 3, 1, 1),
nn.BatchNorm2d(channels),
nn.ReLU(True),
nn.Dropout2d(dropout_p),
nn.Conv2d(channels, channels, 3, 1, 1),
nn.BatchNorm2d(channels),
nn.ReLU(True),
nn.Dropout2d(dropout_p))
self.dream_model = dream_model
def set_model_head(
self,
model_name='DenseNet',
head={
'num_outputs': 10,
'layers': [],
'class_names': None,
'model_type': 'classifier'
},
criterion=nn.NLLLoss(),
adaptive=True,
dropout_p=0.45,
device=None):
models_meta = {
'resnet34': {
'conv_channels': 512,
'head_id': -2,
'adaptive_head': [DAI_AvgPool],
'normal_head': [nn.AvgPool2d(7, 1)]
},
'resnet50': {
'conv_channels': 2048,
'head_id': -2,
'adaptive_head': [DAI_AvgPool],
'normal_head': [nn.AvgPool2d(7, 1)]
},
'densenet': {
'conv_channels': 1024,
'head_id': -1,
'adaptive_head': [nn.ReLU(inplace=True), DAI_AvgPool],
'normal_head': [nn.ReLU(inplace=True),
nn.AvgPool2d(7, 1)]
}
}
name = model_name.lower()
meta = models_meta[name]
modules = list(self.model.children())
l = modules[:meta['head_id']]
if self.dream_model:
l += self.dream_model
if type(head).__name__ != 'dict':
model = nn.Sequential(*l)
for layer in head.children():
if (type(layer).__name__) == 'StdConv':
conv_module = layer
break
conv_layer = conv_module.conv
temp_args = [
conv_layer.out_channels, conv_layer.kernel_size,
conv_layer.stride, conv_layer.padding
]
temp_args.insert(0, meta['conv_channels'])
conv_layer = nn.Conv2d(*temp_args)
conv_module.conv = conv_layer
model.add_module('custom_head', head)
else:
head['criterion'] = criterion
if head['model_type'].lower() == 'classifier':
head['output_non_linearity'] = None
self.num_outputs = head['num_outputs']
fc = modules[-1]
try:
in_features = fc.in_features
except:
in_features = fc.model.out.in_features
fc = FC(num_inputs=in_features,
num_outputs=head['num_outputs'],
layers=head['layers'],
model_type=head['model_type'],
output_non_linearity=head['output_non_linearity'],
dropout_p=dropout_p,
criterion=head['criterion'],
optimizer_name=None,
device=device)
if adaptive:
l += meta['adaptive_head']
else:
l += meta['normal_head']
model = nn.Sequential(*l)
model.add_module('fc', fc)
self.model = model
self.head = head
if type(head).__name__ == 'dict':
print('Model: {}, Setting head: inputs: {} hidden:{} outputs: {}'.
format(model_name, in_features, head['layers'],
head['num_outputs']))
else:
print('Model: {}, Setting head: {}'.format(model_name,
type(head).__name__))
def _get_dropout(self):
return self.dropout_p
def _set_dropout(self, p=0.45):
if self.model.classifier is not None:
print('{}: setting head (FC) dropout prob to {:.3f}'.format(
self.model_name, p))
self.model.fc._set_dropout(p=p)
def get_model_params(self):
params = super(TransferNetworkImg, self).get_model_params()
params['class_names'] = self.class_names
params['num_classes'] = self.num_classes
params['head'] = self.head
return params