def InceptionResNetV2_Multitask_Ont(self, params):
assert len(params['INPUTS'].keys()) == 1, 'Number of inputs must be one.'
assert params['INPUTS'][params['INPUTS'].keys()[0]]['type'] == 'raw-image', 'Input must be of type "raw-image".'
self.ids_inputs = params['INPUTS'].keys()
self.ids_outputs = params['OUTPUTS'].keys()
input_shape = params['INPUTS'][params['INPUTS'].keys()[0]]['img_size_crop']
image = Input(name=self.ids_inputs[0], shape=input_shape)
##################################################
# Load Inception model pre-trained on ImageNet
self.model = InceptionResNetV2(weights='imagenet', input_tensor=image)
# Recover input layer
#image = self.model.get_layer(self.ids_inputs[0]).output
# Recover last layer kept from original model: 'fc2'
x = self.model.get_layer('avg_pool').output
##################################################
#x = Flatten()(x)
# Define outputs
outputs = []
outputs_list = []
outputs_matching = {}
num_classes_matching = {}
if 'SORTED_OUTPUTS' in params.keys():
sorted_keys = params['SORTED_OUTPUTS']
else:
sorted_keys = []
for k in params['OUTPUTS'].keys():
if params['OUTPUTS'][k]['type'] == 'sigma':
sorted_keys.append(k)
else:
sorted_keys.insert(0, k)
num_classes_list = []
for id_name in sorted_keys:
data = params['OUTPUTS'][id_name]
if data['type'] == 'sigma':
continue
else:
# Count the number of output classes
num_classes = 0
with open(params['DATA_ROOT_PATH']+'/'+data['classes'], 'r') as f:
for line in f: num_classes += 1
if data['type'] == 'binary' and params['EMPTY_LABEL']==True:
num_classes += 1 # empty label
# Define only a FC output layer per output
out = Dense(num_classes)(x)
outputs.append(out)
num_classes_list.append(num_classes)
n_multiclass = num_classes_list[0]
n_multilabel = num_classes_list[1]
total_concepts = np.sum(num_classes_list)
x = Merge()(outputs)
#Ont_Layer = OntologyLayer((None,total_concepts),"/media/HDD3TB/datasets/Recipes5k/mtannotations/Ontology_matrix.npy")
#Ont_Layer.build((None,n_multiclass))
#Ont_Layer.trainable = False
x = OntologyLayer((None,total_concepts),"/media/HDD3TB/datasets/Recipes5k/mtannotations/Ontology_matrix.npy")(x)
outputs = []
for idx, num_classes in enumerate(num_classes_list):
if idx == 0:
init_idx = 0
end_idx = num_classes
else:
init_idx = end_idx
end_idx = end_idx + num_classes
out = Lambda( lambda x: x[:, init_idx:end_idx])(x)
outputs.append(out)
curr_output = 0
for id_name in sorted_keys:
data = params['OUTPUTS'][id_name]
# Special output that calculates sigmas for uncertainty loss
if data['type'] == 'sigma':
match_output = params['OUTPUTS'][id_name]['output_id']
match_act = outputs_matching[match_output]
out_sigma = ConcatenateOutputWithSigma((None, num_classes_matching[match_output]+1), name_suffix=id_name, name=id_name)(match_act)
outputs_list.append(out_sigma)
else:
out = outputs[curr_output]
curr_output = curr_output + 1
out_act = Activation(data['activation'], name=id_name)(out)
outputs_list.append(out_act)
outputs_matching[id_name] = out_act
num_classes_matching[id_name] = num_classes
self.model = Model(input=image, output=outputs_list)
def InceptionResNetV2_Multitask_TopDown_Ont(self, params):
assert len(params['INPUTS'].keys()) == 1, 'Number of inputs must be one.'
assert params['INPUTS'][params['INPUTS'].keys()[0]]['type'] == 'raw-image', 'Input must be of type "raw-image".'
self.ids_inputs = params['INPUTS'].keys()
self.ids_outputs = params['OUTPUTS'].keys()
input_shape = params['INPUTS'][params['INPUTS'].keys()[0]]['img_size_crop']
image = Input(name=self.ids_inputs[0], shape=input_shape)
##################################################
# Load Inception model pre-trained on ImageNet
self.model = InceptionResNetV2(weights='imagenet', input_tensor=image)
# Recover input layer
#image = self.model.get_layer(self.ids_inputs[0]).output
# Recover last layer kept from original model: 'fc2'
x = self.model.get_layer('avg_pool').output
##################################################
#x = Flatten()(x)
# Define outputs
outputs = []
outputs_list = []
outputs_matching = {}
num_classes_matching = {}
if 'SORTED_OUTPUTS' in params.keys():
sorted_keys = params['SORTED_OUTPUTS']
else:
sorted_keys = []
for k in params['OUTPUTS'].keys():
if params['OUTPUTS'][k]['type'] == 'sigma':
sorted_keys.append(k)
else:
sorted_keys.insert(0, k)
num_classes_list = []
for id_name in sorted_keys:
data = params['OUTPUTS'][id_name]
if data['type'] == 'sigma':
continue
else:
# Count the number of output classes
num_classes = 0
with open(params['DATA_ROOT_PATH']+'/'+data['classes'], 'r') as f:
for line in f: num_classes += 1
if data['type'] == 'binary' and params['EMPTY_LABEL']==True:
num_classes += 1 # empty label
# Define only a FC output layer per output
out = Dense(num_classes)(x)
outputs.append(out)
num_classes_list.append(num_classes)
n_multiclass = num_classes_list[0]
n_multilabel = num_classes_list[1]
total_concepts = np.sum(num_classes_list)
x = Merge()(outputs)
x = OntologyLayer((None,total_concepts), "/media/HDD3TB/datasets/Recipes5k/mtannotations/Ontology_matrix.npy")(x)
outputs = []
for idx, num_classes in enumerate(num_classes_list):
if idx == 0:
init_idx = 0
end_idx = num_classes
else:
init_idx = end_idx
end_idx = end_idx + num_classes
out = Lambda( lambda x: x[:, init_idx:end_idx])(x)
outputs.append(out)
curr_output = 0
for id_name in sorted_keys:
data = params['OUTPUTS'][id_name]
# Special output that calculates sigmas for uncertainty loss
if data['type'] == 'sigma':
match_output = params['OUTPUTS'][id_name]['output_id']
match_act = outputs_matching[match_output]
out_sigma = ConcatenateOutputWithSigma((None, num_classes_matching[match_output]+1), name_suffix=id_name, name=id_name)(match_act)
outputs_list.append(out_sigma)
else:
out = outputs[curr_output]
curr_output = curr_output + 1
out_act = Activation(data['activation'], name=id_name)(out)
outputs_list.append(out_act)
outputs_matching[id_name] = out_act
num_classes_matching[id_name] = num_classes
self.model = Model(input=image, output=outputs_list)
def InceptionV4_PlusFC(self, params):
assert len(params['INPUTS'].keys()) == 1, 'Number of inputs must be one.'
assert params['INPUTS'][params['INPUTS'].keys()[0]]['type'] == 'raw-image', 'Input must be of type "raw-image".'
self.ids_inputs = params['INPUTS'].keys()
self.ids_outputs = params['OUTPUTS'].keys()
input_shape = params['INPUTS'][params['INPUTS'].keys()[0]]['img_size_crop']
image = Input(name=self.ids_inputs[0], shape=input_shape)
##################################################
# Load Inception model pre-trained on ImageNet
#self.model = InceptionResNetV2(weights='imagenet', input_tensor=image)
self.model = inception_v4(251, 0.5, "imagenet", False)
# Recover input layer
#image = self.model.get_layer(self.ids_inputs[0]).output
# Recover last layer kept from original model: 'fc2'
x = self.model.get_layer('concatenate_25').output
##################################################
# 1 x 1 x 1536
x = AveragePooling2D((8,8), padding='valid')(x)
#x = Dropout(0.5)(x)
x = Flatten()(x)
# 1536
# Define outputs
outputs_list = []
for id_name, data in params['OUTPUTS'].iteritems():
# Count the number of output classes
num_classes = 0
with open(params['DATA_ROOT_PATH']+'/'+data['classes'], 'r') as f:
for line in f: num_classes += 1
if params['EMPTY_LABEL']:
num_classes += 1
# Define only a FC output layer (+ activation) per output
out = Dense(num_classes)(x)
out_act = Activation(data['activation'], name=id_name)(out)
outputs_list.append(out_act)
self.model = Model(input=self.model.input, output=outputs_list)
'''
def ResNext50_PlusFC(self, params):
assert len(params['INPUTS'].keys()) == 1, 'Number of inputs must be one.'
assert params['INPUTS'][params['INPUTS'].keys()[0]]['type'] == 'raw-image', 'Input must be of type "raw-image".'
self.ids_inputs = params['INPUTS'].keys()
self.ids_outputs = params['OUTPUTS'].keys()
input_shape = params['INPUTS'][params['INPUTS'].keys()[0]]['img_size_crop']
image = Input(name=self.ids_inputs[0], shape=input_shape)
##################################################
# Load Inception model pre-trained on ImageNet
self.model = ResNeXt50(weights='imagenet', input_tensor=image)
# Recover input layer
#image = self.model.get_layer(self.ids_inputs[0]).output
# Recover last layer kept from original model: 'fc2'
x = self.model.get_layer('avg_pool').output
##################################################
#x = Flatten()(x)
# Define outputs
outputs_list = []
for id_name, data in params['OUTPUTS'].iteritems():
# Count the number of output classes
num_classes = 0
with open(params['DATA_ROOT_PATH']+'/'+data['classes'], 'r') as f:
for line in f: num_classes += 1
# Define only a FC output layer (+ activation) per output
out = Dense(num_classes)(x)
out_act = Activation(data['activation'], name=id_name)(out)
outputs_list.append(out_act)
self.model = Model(input=image, output=outputs_list)
'''
def InceptionResNetV2_Ensemble(self, params):
#assert len(params['INPUTS'].keys()) == 1, 'Number of inputs must be one.'
#assert params['INPUTS'][params['INPUTS'].keys()[0]]['type'] == 'raw-image', 'Input must be of type "raw-image".'
self.ids_inputs = params['INPUTS'].keys()
self.ids_outputs = params['OUTPUTS'].keys()
input_shape = params['INPUTS'][params['INPUTS'].keys()[0]]['img_size_crop']
image = Input(name=self.ids_inputs[0], shape=input_shape)
models_folder = ['inceptionresnetv2_LMW_adam_1',
'inceptionresnetv2_LMW_adam_2',
'inceptionresnetv2_LMW_adam_3',
'inceptionresnetv2_LMW_adam_4',
'inceptionresnetv2_LMW_adam_5',
'inceptionresnetv2_LMW_adam_6',
'inceptionresnetv2_LMW_adam_7']
models_reload_epoch = [9,
13,
14,
11,
17,
10,
12]
models = []
for idx, bmodel_folder in enumerate(models_folder):
model_folder = '/home/eduardo/Documents/workspaces/ifood2019_recognition/models/'+bmodel_folder
model_reload_epoch = models_reload_epoch[idx]
# Load model
base_model = loadModel(model_folder, model_reload_epoch).model
base_model.name = "emodel_"+str(idx)
models.append(base_model)
#models = [base_model_1, base_model_2]
merged_models = []
for j in range(len(models)):
base_model = models[j]
for i, layer in enumerate(base_model.layers[1:]):
layer.trainable = False
print layer.name
merged_models.append(base_model(image))
x = Merge()(merged_models)
##################################################
# Define outputs
outputs_list = []
for id_name, data in params['OUTPUTS'].iteritems():
# Count the number of output classes
num_classes = 0
with open(params['DATA_ROOT_PATH']+'/'+data['classes'], 'r') as f:
for line in f: num_classes += 1
if data['type'] == 'binary':
num_classes += 1 # empty label
# Define only a FC output layer (+ activation) per output
x = Dense(num_classes*len(models), activation="relu")(x)
out = Dense(num_classes, kernel_initializer="ones")(x)
out_act = Activation(data['activation'], name=id_name)(out)
outputs_list.append(out_act)
self.model = Model(input=image, output=outputs_list)
def InceptionResNetV2_Multitask_TopDown_Ont(self, params):
assert len(params['INPUTS'].keys()) == 1, 'Number of inputs must be one.'
assert params['INPUTS'][params['INPUTS'].keys()[0]]['type'] == 'raw-image', 'Input must be of type "raw-image".'
self.ids_inputs = params['INPUTS'].keys()
self.ids_outputs = params['OUTPUTS'].keys()
input_shape = params['INPUTS'][params['INPUTS'].keys()[0]]['img_size_crop']
image = Input(name=self.ids_inputs[0], shape=input_shape)
##################################################
# Load Inception model pre-trained on ImageNet
self.model = InceptionResNetV2(weights='imagenet', input_tensor=image)
# Recover input layer
#image = self.model.get_layer(self.ids_inputs[0]).output
# Recover last layer kept from original model: 'fc2'
x = self.model.get_layer('avg_pool').output
##################################################
#x = Flatten()(x)
# Define outputs
outputs = []
outputs_list = []
outputs_matching = {}
num_classes_matching = {}
if 'SORTED_OUTPUTS' in params.keys():
sorted_keys = params['SORTED_OUTPUTS']
else:
sorted_keys = []
for k in params['OUTPUTS'].keys():
if params['OUTPUTS'][k]['type'] == 'sigma':
sorted_keys.append(k)
else:
sorted_keys.insert(0, k)
num_classes_list = []
for id_name in sorted_keys:
data = params['OUTPUTS'][id_name]
if data['type'] == 'sigma':
continue
else:
# Count the number of output classes
num_classes = 0
with open(params['DATA_ROOT_PATH']+'/'+data['classes'], 'r') as f:
for line in f: num_classes += 1
if data['type'] == 'binary' and params['EMPTY_LABEL']==True:
num_classes += 1 # empty label
# Define only a FC output layer per output
out = Dense(num_classes)(x)
outputs.append(out)
num_classes_list.append(num_classes)
n_multiclass = num_classes_list[0]
n_multilabel = num_classes_list[1]
total_concepts = np.sum(num_classes_list)
x = Merge()(outputs)
x = OntologyLayer((None,total_concepts), "/media/HDD3TB/datasets/Recipes5k/mtannotations/Ontology_matrix.npy")(x)
outputs = []
for idx, num_classes in enumerate(num_classes_list):
if idx == 0:
init_idx = 0
end_idx = num_classes
else:
init_idx = end_idx
end_idx = end_idx + num_classes
out = Lambda( lambda x: x[:, init_idx:end_idx])(x)
outputs.append(out)
curr_output = 0
for id_name in sorted_keys:
data = params['OUTPUTS'][id_name]
# Special output that calculates sigmas for uncertainty loss
if data['type'] == 'sigma':
match_output = params['OUTPUTS'][id_name]['output_id']
match_act = outputs_matching[match_output]
out_sigma = ConcatenateOutputWithSigma((None, num_classes_matching[match_output]+1), name_suffix=id_name, name=id_name)(match_act)
outputs_list.append(out_sigma)
else:
out = outputs[curr_output]
curr_output = curr_output + 1
out_act = Activation(data['activation'], name=id_name)(out)
outputs_list.append(out_act)
outputs_matching[id_name] = out_act
num_classes_matching[id_name] = num_classes
self.model = Model(input=image, output=outputs_list)
def InceptionResNetV2_Multitask(self, params):
assert len(params['INPUTS'].keys()) == 1, 'Number of inputs must be one.'
assert params['INPUTS'][params['INPUTS'].keys()[0]]['type'] == 'raw-image', 'Input must be of type "raw-image".'
self.ids_inputs = params['INPUTS'].keys()
self.ids_outputs = params['OUTPUTS'].keys()
input_shape = params['INPUTS'][params['INPUTS'].keys()[0]]['img_size_crop']
image = Input(name=self.ids_inputs[0], shape=input_shape)
##################################################
# Load Inception model pre-trained on ImageNet
self.model = InceptionResNetV2(weights='imagenet', input_tensor=image)
for layer in self.model.layers:
layer.trainable=False
# Recover input layer
#image = self.model.get_layer(self.ids_inputs[0]).output
# Recover last layer kept from original model: 'fc2'
x = self.model.get_layer('avg_pool').output
#x = self.model.output
##################################################
#x = Flatten()(x)
# Define outputs
outputs_list = []
outputs_matching = {}
num_classes_matching = {}
if 'SORTED_OUTPUTS' in params.keys():
sorted_keys = params['SORTED_OUTPUTS']
else:
sorted_keys = []
for k in params['OUTPUTS'].keys():
if params['OUTPUTS'][k]['type'] == 'sigma':
sorted_keys.append(k)
else:
sorted_keys.insert(0, k)
#for id_name, data in params['OUTPUTS'].iteritems():
for id_name in sorted_keys:
data = params['OUTPUTS'][id_name]
# Special output that calculates sigmas for uncertainty loss
if data['type'] == 'sigma':
match_output = params['OUTPUTS'][id_name]['output_id']
match_act = outputs_matching[match_output]
out_sigma = ConcatenateOutputWithSigma((None, num_classes_matching[match_output]+1), name_suffix=id_name, name=id_name)(match_act)
outputs_list.append(out_sigma)
else:
# Count the number of output classes
num_classes = 0
with open(params['DATA_ROOT_PATH']+'/'+data['classes'], 'r') as f:
for line in f: num_classes += 1
if data['type'] == 'binary' and params['EMPTY_LABEL']==True:
num_classes += 1 # empty label
# Define only a FC output layer (+ activation) per output
out = Dense(num_classes)(x)
out_act = Activation(data['activation'], name=id_name)(out)
outputs_list.append(out_act)
outputs_matching[id_name] = out_act
num_classes_matching[id_name] = num_classes
self.model = Model(input=image, output=outputs_list)