class VGG19_BN():
def __init__(self):
self.config_file = None
self.io = EmbeddingIO(None)
self.init = False
self.cfgs = None
#def
def configure(self, config_file):
self.config_file = config_file
self.init = False
if not os.path.exists(self.config_file):
self.io.print_error(
'Could not find config file for VGG-19-BN {0}'.format(
self.config_file))
self.init = False
return
#if
pfile = open(self.config_file, 'r')
self.cfgs = yaml.load(pfile)
pfile.close()
self.init = True
#def
def define(self):
#
# TODO : get the filter size, number of filters, reception field size from yaml file
#
try:
self.model = Sequential()
# C1
self.model.add(
ZeroPadding2D((1, 1),
input_shape=(self.cfgs['n_channels'],
self.cfgs['image_height'],
self.cfgs['image_width'])))
self.model.add(Convolution2D(64, 3, 3,
init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(Convolution2D(64, 3, 3,
init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# C1
# C2
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(128, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(128, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# C2
# C3
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(256, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(256, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(256, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(256, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# C3
# C4
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(512, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(512, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(512, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(512, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# C4
# C5
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(512, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(512, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(512, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(ZeroPadding2D((1, 1)))
self.model.add(
Convolution2D(512, 3, 3, init=self.cfgs['conv_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['conv_non_linearity']))
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# C5
# FC6
self.model.add(Flatten())
self.model.add(Dense(4096, init=self.cfgs['fc_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['fc_non_linearity']))
# FC6
# FC7
self.model.add(Dropout(self.cfgs['drop_out']))
self.model.add(Dense(4096, init=self.cfgs['fc_init']))
self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1))
self.model.add(Activation(self.cfgs['fc_non_linearity']))
# Fc7
# FC8
self.model.add(Dropout(self.cfgs['drop_out']))
self.model.add(
Dense(self.cfgs['nb_classes'], init=self.cfgs['fc_init']))
self.model.add(Activation(self.cfgs['activation']))
# FC8
if not self.cfgs['model_weights_file'] == None:
self.init_from_this()
#if
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
#def
def init_from_this(self):
weights_file = self.cfgs['model_weights_file']
if weights_file.endswith('.hdf5'):
self.load_weights(weights_file)
self.io.print_info('Initalized from {0}'.format(weights_file))
#if
if weights_file.endswith('.caffemodel'):
self.io.print_warning(
'Loading from caffe model not implemented starting with random weights'
)
#if
#def
def load_weights(self, filepath):
'''
This method does not make use of Sequential.set_weights()
for backwards compatibility.
'''
# Loads weights from HDF5 file
import h5py
f = h5py.File(filepath)
for k in range(f.attrs['nb_layers']):
layer_type = self.model.layers[k].get_config()['name']
layer_name_string = '{0}_layer_{1}'.format(layer_type, k)
if layer_name_string in self.cfgs['ignore_while_loading']:
self.io.print_warning(
'Ignoring weights from {0}'.format(layer_name_string))
continue
#if
layer_name = 'layer_{}'.format(k)
g = f[layer_name]
self.io.print_info(
'Transfering parameters from {0}'.format(layer_name_string))
weights = [
g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])
]
self.model.layers[k].set_weights(weights)
#for
f.close()
self.io.print_info(self.model.layers[-1].get_config().values())
#def
def compile(self, compile_cfgs):
try:
sgd = SGD(lr=compile_cfgs['lr'],
decay=compile_cfgs['decay'],
momentum=compile_cfgs['momentum'],
nesterov=True)
self.model.compile(loss=loss_functions_dict[compile_cfgs['loss']],
optimizer=sgd)
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
class InceptionV3_finetune():
def __init__(self):
self.layer_list = []
self.config_file = None
self.io = EmbeddingIO(None)
self.init = False
self.cfgs = None
self.loss_functions = ObjectiveFunctions()
#def
def configure(self, config_file):
self.config_file = config_file
self.init = False
if not os.path.exists(self.config_file):
self.io.print_error(
'Could not find config file for InceptionV3 {0}'.format(
self.config_file))
self.init = False
return
#if
pfile = open(self.config_file, 'r')
self.cfgs = yaml.load(pfile)
self.betas = random_float(0, 20,
self.cfgs['n_channels_partClassifiers'])
pfile.close()
#Tracer()()
self.init = True
#def
def add_to_graph(self, *args, **kwargs):
self.model.add_node(*args, **kwargs)
self.last_added_node = kwargs['name']
self.layer_list.append(kwargs['name'])
return kwargs['name']
#def
def add_bn_conv_layer(self, *args, **kwargs):
layer_name = kwargs['name']
input_layer = kwargs['input']
del kwargs['name']
del kwargs['input']
if 'padding' in kwargs:
layer_name = layer_name + '_pad'
self.add_to_graph(ZeroPadding2D(padding=kwargs['padding']),
name=layer_name,
input=input_layer)
input_layer = layer_name
del kwargs['padding']
#if
# CONV with linear activation by default
layer_name = layer_name + '_conv'
self.add_to_graph(Convolution2D(*args, border_mode='valid', **kwargs),
name=layer_name,
input=input_layer)
# Batch normalization added directly on output of a linear layer
input_layer = layer_name
layer_name = layer_name + '_bn'
_ = self.add_to_graph(BatchNormalization(mode=0,
epsilon=0.0001,
axis=1),
name=layer_name,
input=input_layer)
# Standard normalization
input_layer = layer_name
layer_name = layer_name + '_nonlin'
_ = self.add_to_graph(Activation('relu'),
name=layer_name,
input=input_layer)
return layer_name
#def
def add_inceptionA(self, input_layer, list_nb_filter, base_name):
l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1',
input=input_layer,
nb_filter=list_nb_filter[0][0],
nb_row=1,
nb_col=1)
l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1',
input=input_layer,
nb_filter=list_nb_filter[1][0],
nb_row=1,
nb_col=1)
l2_2 = self.add_bn_conv_layer(name=base_name + '_l2_2',
input=l2_1,
nb_filter=list_nb_filter[1][1],
nb_row=5,
nb_col=5,
padding=(2, 2))
l3_1 = self.add_bn_conv_layer(name=base_name + '_l3_1',
input=input_layer,
nb_filter=list_nb_filter[2][0],
nb_row=1,
nb_col=1)
l3_2 = self.add_bn_conv_layer(name=base_name + '_l3_2',
input=l3_1,
nb_filter=list_nb_filter[2][1],
nb_row=3,
nb_col=3,
padding=(1, 1))
l3_3 = self.add_bn_conv_layer(name=base_name + '_l3_3',
input=l3_2,
nb_filter=list_nb_filter[2][2],
nb_row=3,
nb_col=3,
padding=(1, 1))
l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1, 1)),
name=base_name + '_14_1',
input=input_layer)
l4_2 = self.add_to_graph(AveragePooling2D(pool_size=(3, 3),
strides=(1, 1)),
name=base_name + '_14_2',
input=l4_1)
l4_3 = self.add_bn_conv_layer(name=base_name + '_l4_3',
input=l4_2,
nb_filter=list_nb_filter[3][0],
nb_row=1,
nb_col=1)
self.add_to_graph(Activation("linear"),
name=base_name,
inputs=[l1_1, l2_2, l3_3, l4_3],
merge_mode="concat",
concat_axis=1)
self.io.print_info('Added Inception-A {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def add_inceptionB(self, input_layer, list_nb_filter, base_name):
l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1',
input=input_layer,
nb_filter=list_nb_filter[0][0],
nb_row=3,
nb_col=3,
subsample=(2, 2))
l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1',
input=input_layer,
nb_filter=list_nb_filter[1][0],
nb_row=1,
nb_col=1)
l2_2 = self.add_bn_conv_layer(name=base_name + '_l2_2',
input=l2_1,
nb_filter=list_nb_filter[1][1],
nb_row=3,
nb_col=3,
padding=(1, 1))
l2_3 = self.add_bn_conv_layer(name=base_name + '_l2_3',
input=l2_2,
nb_filter=list_nb_filter[1][2],
nb_row=3,
nb_col=3,
subsample=(2, 2))
l3_1 = self.add_to_graph(MaxPooling2D(pool_size=(3, 3),
strides=(2, 2)),
name=base_name + '_13_1',
input=input_layer)
self.add_to_graph(Activation("linear"),
name=base_name,
inputs=[l1_1, l2_3, l3_1],
merge_mode="concat",
concat_axis=1)
self.io.print_info('Added Inception-B {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def add_inceptionC(self, input_layer, list_nb_filter, base_name):
l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1',
input=input_layer,
nb_filter=list_nb_filter[0][0],
nb_row=1,
nb_col=1)
l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1',
input=input_layer,
nb_filter=list_nb_filter[1][0],
nb_row=1,
nb_col=1)
l2_2 = self.add_bn_conv_layer(name=base_name + '_l2_2',
input=l2_1,
nb_filter=list_nb_filter[1][1],
nb_row=1,
nb_col=7,
padding=(0, 3))
l2_3 = self.add_bn_conv_layer(name=base_name + '_l2_3',
input=l2_2,
nb_filter=list_nb_filter[1][2],
nb_row=7,
nb_col=1,
padding=(3, 0))
## padding and nb_row might not match with the lasagne weights
l3_1 = self.add_bn_conv_layer(name=base_name + '_l3_1',
input=input_layer,
nb_filter=list_nb_filter[2][0],
nb_row=1,
nb_col=1)
l3_2 = self.add_bn_conv_layer(name=base_name + '_l3_2',
input=l3_1,
nb_filter=list_nb_filter[2][1],
nb_row=7,
nb_col=1,
padding=(3, 0))
l3_3 = self.add_bn_conv_layer(name=base_name + '_l3_3',
input=l3_2,
nb_filter=list_nb_filter[2][2],
nb_row=1,
nb_col=7,
padding=(0, 3))
l3_4 = self.add_bn_conv_layer(name=base_name + '_l3_4',
input=l3_3,
nb_filter=list_nb_filter[2][3],
nb_row=7,
nb_col=1,
padding=(3, 0))
l3_5 = self.add_bn_conv_layer(name=base_name + '_l3_5',
input=l3_4,
nb_filter=list_nb_filter[2][4],
nb_row=1,
nb_col=7,
padding=(0, 3))
l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1, 1)),
name=base_name + '_14_1',
input=input_layer)
l4_2 = self.add_to_graph(AveragePooling2D(pool_size=(3, 3),
strides=(1, 1)),
name=base_name + '_14_2',
input=l4_1)
l4_3 = self.add_bn_conv_layer(name=base_name + '_l4_3',
input=l4_2,
nb_filter=list_nb_filter[3][0],
nb_row=1,
nb_col=1)
self.add_to_graph(Activation("linear"),
name=base_name,
inputs=[l1_1, l2_3, l3_5, l4_3],
merge_mode="concat",
concat_axis=1)
self.io.print_info('Added Inception-C {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def add_inceptionD(self, input_layer, list_nb_filter, base_name):
l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1',
input=input_layer,
nb_filter=list_nb_filter[0][0],
nb_row=1,
nb_col=1)
l1_2 = self.add_bn_conv_layer(name=base_name + '_l1_2',
input=l1_1,
nb_filter=list_nb_filter[0][1],
nb_row=3,
nb_col=3,
subsample=(2, 2))
l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1',
input=input_layer,
nb_filter=list_nb_filter[1][0],
nb_row=1,
nb_col=1)
l2_2 = self.add_bn_conv_layer(name=base_name + '_l2_2',
input=l2_1,
nb_filter=list_nb_filter[1][1],
nb_row=1,
nb_col=7,
padding=(0, 3))
l2_3 = self.add_bn_conv_layer(name=base_name + '_l2_3',
input=l2_2,
nb_filter=list_nb_filter[1][2],
nb_row=7,
nb_col=1,
padding=(3, 0))
l2_4 = self.add_bn_conv_layer(name=base_name + '_l2_4',
input=l2_3,
nb_filter=list_nb_filter[1][2],
nb_row=3,
nb_col=3,
subsample=(2, 2))
l3_1 = self.add_to_graph(MaxPooling2D(pool_size=(3, 3),
strides=(2, 2)),
name=base_name + '_13_1',
input=input_layer)
self.add_to_graph(Activation("linear"),
name=base_name,
inputs=[l1_2, l2_4, l3_1],
merge_mode="concat",
concat_axis=1)
self.io.print_info('Added Inception-D {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def add_inceptionE(self, input_layer, list_nb_filter, base_name,
pool_mode):
l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1',
input=input_layer,
nb_filter=list_nb_filter[0][0],
nb_row=1,
nb_col=1)
l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1',
input=input_layer,
nb_filter=list_nb_filter[1][0],
nb_row=1,
nb_col=1)
l2_2a = self.add_bn_conv_layer(name=base_name + '_l2_2a',
input=l2_1,
nb_filter=list_nb_filter[1][1],
nb_row=1,
nb_col=3,
padding=(0, 1))
l2_2b = self.add_bn_conv_layer(name=base_name + '_l2_2b',
input=l2_1,
nb_filter=list_nb_filter[1][1],
nb_row=3,
nb_col=1,
padding=(1, 0))
l3_1 = self.add_bn_conv_layer(name=base_name + '_l3_1',
input=input_layer,
nb_filter=list_nb_filter[2][0],
nb_row=1,
nb_col=1)
l3_2 = self.add_bn_conv_layer(name=base_name + '_l3_2',
input=l3_1,
nb_filter=list_nb_filter[2][1],
nb_row=3,
nb_col=3,
padding=(1, 1))
l3_3a = self.add_bn_conv_layer(name=base_name + '_l3_3a',
input=l3_2,
nb_filter=list_nb_filter[2][2],
nb_row=1,
nb_col=3,
padding=(0, 1))
l3_3b = self.add_bn_conv_layer(name=base_name + '_l3_3b',
input=l3_2,
nb_filter=list_nb_filter[2][3],
nb_row=3,
nb_col=1,
padding=(1, 0))
l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1, 1)),
name=base_name + '_14_1',
input=input_layer)
l4_2 = self.add_to_graph(pooling_dict[pool_mode](pool_size=(3, 3),
strides=(1, 1)),
name=base_name + '_14_2',
input=l4_1)
l4_3 = self.add_bn_conv_layer(name=base_name + '_l4_3',
input=l4_2,
nb_filter=list_nb_filter[3][0],
nb_row=1,
nb_col=1)
self.add_to_graph(Activation("linear"),
name=base_name,
inputs=[l1_1, l2_2a, l2_2b, l3_3a, l3_3b, l4_3],
merge_mode="concat",
concat_axis=1)
self.io.print_info('Added Inception-E {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def define(self):
try:
self.model = Graph()
#self.model.add_input(name='input', input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width']))
self.model.add_input(name='input',
input_shape=(self.cfgs['n_channels'], None,
None))
#
# Part of the network which is defined in the config file should move here
#
cgfs_nodes = self.cfgs['nodes']
for node in cgfs_nodes:
if not node['type'] == 'Activation':
self.add_to_graph(
layer_dict[node['type']](**node['parameter']),
name=node['name'],
input=node['input'])
else:
self.add_to_graph(layer_dict[node['type']](
node['parameter']['mode']),
name=node['name'],
input=node['input'])
#if
self.io.print_info('Added {1}:{0}'.format(
node['type'], node['name']))
#for
self.add_inceptionA(input_layer=self.last_added_node,
list_nb_filter=((64, ), (48, 64), (64, 96, 96),
(32, )),
base_name='mixed')
self.add_inceptionA(input_layer=self.last_added_node,
list_nb_filter=((64, ), (48, 64), (64, 96, 96),
(64, )),
base_name='mixed_1')
self.add_inceptionA(input_layer=self.last_added_node,
list_nb_filter=((64, ), (48, 64), (64, 96, 96),
(64, )),
base_name='mixed_2')
self.add_inceptionB(input_layer=self.last_added_node,
list_nb_filter=((384, ), (64, 96, 96)),
base_name='mixed_3')
self.add_inceptionC(input_layer=self.last_added_node,
list_nb_filter=((192, ), (128, 128, 192),
(128, 128, 128, 128,
192), (192, )),
base_name='mixed_4')
self.add_inceptionC(input_layer=self.last_added_node,
list_nb_filter=((192, ), (160, 160, 192),
(160, 160, 160, 160,
192), (192, )),
base_name='mixed_5')
self.add_inceptionC(input_layer=self.last_added_node,
list_nb_filter=((192, ), (160, 160, 192),
(160, 160, 160, 160,
192), (192, )),
base_name='mixed_6')
self.add_inceptionC(input_layer=self.last_added_node,
list_nb_filter=((192, ), (192, 192, 192),
(192, 192, 192, 192,
192), (192, )),
base_name='mixed_7')
self.add_inceptionD(input_layer=self.last_added_node,
list_nb_filter=((192, 320), (192, 192, 192,
192)),
base_name='mixed_8')
self.add_inceptionE(input_layer=self.last_added_node,
list_nb_filter=((320, ), (384, 384, 384),
(448, 384, 384, 384), (192, )),
pool_mode='average',
base_name='mixed_9')
self.add_inceptionE(input_layer=self.last_added_node,
list_nb_filter=((320, ), (384, 384, 384),
(448, 384, 384, 384), (192, )),
pool_mode='max',
base_name='mixed_10')
#self.add_to_graph(AveragePooling2D(pool_size=(5,5)), name='pool3', input=self.last_added_node)
#self.io.print_info('Added {1}:{0}'.format('AveragePooling',self.last_added_node))
"""
self.add_to_graph(Flatten(), name='flatten', input='pool3')
self.io.print_info('Added {1}:{0}'.format('Flatten',self.last_added_node))
self.add_to_graph(Dense(self.cfgs['nb_classes']), name='softmax', input='flatten')
self.io.print_info('Added {1}:{0}'.format('Dense',self.last_added_node))
"""
#print 'Adding finetuning layers'---------------------------------------------------------------------------------------
self.add_to_graph(BatchNormalization(mode=0,
epsilon=0.0001,
axis=1),
name='batchnorm',
input=self.last_added_node)
self.io.print_info('Added {1}:{0}'.format(
'BatchNormalization before finetuning', self.last_added_node))
self.add_to_graph(Convolution2D(
self.cfgs['n_channels_partClassifiers'], 1, 1),
name='partClassifiers',
input=self.last_added_node)
self.io.print_info('Added {1}:{0}'.format('partClassifiers',
self.last_added_node))
self.add_to_graph(Activation('relu'),
name='relu_on_partClassifiers',
input=self.last_added_node)
self.io.print_info('Added {1}:{0}'.format(
'ReLu on partClassifiers', self.last_added_node))
if self.cfgs['pooling_AtfineTuning'] == 'avg':
self.add_to_graph(global_Average_Pooling(),
name='custom_pool',
input=self.last_added_node)
self.io.print_info('Added {1}:{0}'.format(
'Custom Pooling Layer:' +
self.cfgs['pooling_AtfineTuning'], self.last_added_node))
elif self.cfgs['pooling_AtfineTuning'] == 'softpool':
self.add_to_graph(power_mean_Layer(self.betas),
name='custom_pool',
input=self.last_added_node) # Learnable
self.io.print_info('Added {1}:{0}'.format(
'Custom Pooling Layer:' +
self.cfgs['pooling_AtfineTuning'], self.last_added_node))
self.add_to_graph(Flatten(),
name='flatten',
input=self.last_added_node)
self.io.print_info('Added {1}:{0}'.format('Flatten',
self.last_added_node))
self.add_to_graph(Dense(self.cfgs['nb_classes']),
name='softmax',
input=self.last_added_node) #Learnable
self.io.print_info('Added {1}:{0}'.format('Softmax-dense weights',
self.last_added_node))
self.add_to_graph(Activation(self.cfgs['activation']),
name='prob',
input=self.last_added_node)
self.io.print_info('Added {1}:{0}'.format('Activation',
self.last_added_node))
#Harsimrat original architecture ---------------------------------------------------------------------------------------
# self.add_to_graph(AveragePooling2D(pool_size=(5,5)), name='pool3', input=self.last_added_node)
# self.io.print_info('Added {1}:{0}'.format('AveragePooling',self.last_added_node))
# """
# self.add_to_graph(Flatten(), name='flatten', input='pool3')
# self.io.print_info('Added {1}:{0}'.format('Flatten',self.last_added_node))
# self.add_to_graph(Dense(self.cfgs['nb_classes']), name='softmax', input='flatten')
# self.io.print_info('Added {1}:{0}'.format('Dense',self.last_added_node))
# """
# self.add_to_graph(Convolution2D(self.cfgs['nb_classes'],1,1),name='softmax',input='pool3')
# self.add_to_graph(Activation(self.cfgs['activation']), name='prob', input='softmax')
#Adding dense finetunning layers
#Re
# Output to the Graph
self.model.add_output(name='output', input='prob')
#Tracer()()
if not self.cfgs['model_weights_file'] == 'None':
#import ipdb; ipdb.set_trace()
self.init_from_this()
#import ipdb; ipdb.set_trace()
#if
except Exception as err:
self.io.print_error('Error configuring the model, {0}'.format(err))
self.init = False
return
#try
self.init = True
#def
def init_from_this(self):
weights_file = self.cfgs['model_weights_file']
#Tracer()()
if not weights_file == 'None':
self.load_weights(weights_file)
self.io.print_info(
'Weights Initalized from {0}'.format(weights_file))
#if
#def
def load_weights(self, filepath):
#Tracer()()
if filepath.endswith('.npz'):
pfile = open(filepath, 'r')
graph = np.load(pfile)['graph'].item()
for node_name, weights in graph.items():
if node_name in self.cfgs['ignore_while_loading']:
self.io.print_warning(
'Ignoring weights from {0}'.format(node_name))
continue
#if
self.io.print_info(
'Transfering parameters from {0}'.format(node_name))
self.model.nodes[node_name].set_weights(weights)
#for
pfile.close()
elif filepath.endswith('.npy'):
pfile = open(filepath, 'r')
graph = np.load(pfile).item()
for node_name, weights in graph.items():
if node_name in self.cfgs['ignore_while_loading']:
self.io.print_warning(
'Ignoring weights from {0}'.format(node_name))
continue
#if
self.io.print_info(
'Transfering parameters from {0}'.format(node_name))
self.model.nodes[node_name].set_weights(weights)
#for
pfile.close()
elif filepath.endswith('.hdf5'):
self.model.load_weights(filepath)
else:
self.io.print_error(
'Unknown model weights file {}'.format(filepath))
#if
self.io.print_info(self.model.nodes['softmax'].get_config())
#def
def setup_loss_function(self, w):
self.loss_functions.set_weights(w)
#def
def compile(self, compile_cfgs):
try:
#opt = optimizer_dict[compile_cfgs['optimizer']](lr=compile_cfgs['lr'], epsilon=compile_cfgs['epsilon'])
opt = SGD(lr=compile_cfgs['lr'])
#model_layer = dict([(ler.name, layer) for layer in self.model.layesrs])
for layer_name, model_layer in self.model.nodes.items():
if layer_name not in self.cfgs['trainable_layers']:
model_layer.trainable = False
else:
self.io.print_info(
'Trainable Layer: {0}'.format(layer_name))
self.model.compile(loss={
'output':
self.loss_functions.dict[compile_cfgs['loss']]
},
optimizer=opt)
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
def get_model(self):
return self.model
class SqueezeNet():
def __init__(self):
self.config_file = None
self.io = EmbeddingIO(None)
self.init = False
self.cfgs = None
#def
def configure(self,config_file):
self.config_file = config_file
self.init = False
if not os.path.exists(self.config_file):
self.io.print_error('Could not find config file for SqueezeNet {0}'.format(self.config_file))
self.init = False
return
#if
pfile = open(self.config_file,'r')
self.cfgs = yaml.load(pfile)
pfile.close()
self.init = True
#def
def define(self):
#
# TODO : get the filter size, number of filters, reception field size from yaml file
#
try:
self.model = Graph()
self.model.add_input(name='input',input_shape=(3,224,224))
#conv 1
self.model.add_node(Convolution2D(96, 3, 3, activation='relu', init='glorot_uniform',subsample=(2,2),border_mode='valid'),name='conv1', input='input')
#maxpool 1
self.model.add_node(MaxPooling2D((2,2)),name='pool1', input='conv1')
#fire 2
self.model.add_node(Convolution2D(16, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire2/conv1x1_1', input='pool1')
self.model.add_node(Convolution2D(64, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire2/conv1x1_2', input='fire2/conv1x1_1')
self.model.add_node(Convolution2D(64, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire2/conv3x3_2', input='fire2/conv1x1_1')
self.model.add_node(Activation("linear"),name='fire2/concat', inputs=["fire2/conv1x1_2","fire2/conv3x3_2"], merge_mode="concat", concat_axis=1)
#fire 3
self.model.add_node(Convolution2D(16, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='"fire3/conv1x1_1', input='fire2/concat')
self.model.add_node(Convolution2D(64, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire3/conv1x1_2', input='"fire3/conv1x1_1')
self.model.add_node(Convolution2D(64, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire3/conv3x3_2', input='"fire3/conv1x1_1')
self.model.add_node(Activation("linear"),name='fire3/concat', inputs=["fire3/conv1x1_2","fire3/conv3x3_2"], merge_mode="concat", concat_axis=1)
#fire 4
self.model.add_node(Convolution2D(32, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='"fire4/conv1x1_1', input='fire3/concat')
self.model.add_node(Convolution2D(128, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire4/conv1x1_2', input='"fire4/conv1x1_1')
self.model.add_node(Convolution2D(128, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire4/conv3x3_2', input='"fire4/conv1x1_1')
self.model.add_node(Activation("linear"),name='fire4/concat', inputs=["fire4/conv1x1_2","fire4/conv3x3_2"], merge_mode="concat", concat_axis=1)
#maxpool 4
self.model.add_node(MaxPooling2D((2,2)),name='pool4', input='fire4/concat')
#fire 5
self.model.add_node(Convolution2D(32, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire5/conv1x1_1', input='pool4')
self.model.add_node(Convolution2D(128, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire5/conv1x1_2', input='fire5/conv1x1_1')
self.model.add_node(Convolution2D(128, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire5/conv3x3_2', input='fire5/conv1x1_1')
self.model.add_node(Activation("linear"),name='fire5/concat', inputs=["fire5/conv1x1_2","fire5/conv3x3_2"], merge_mode="concat", concat_axis=1)
#fire 6
self.model.add_node(Convolution2D(48, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire6/conv1x1_1', input='fire5/concat')
self.model.add_node(Convolution2D(192, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire6/conv1x1_2', input='fire6/conv1x1_1')
self.model.add_node(Convolution2D(192, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire6/conv3x3_2', input='fire6/conv1x1_1')
self.model.add_node(Activation("linear"),name='fire6/concat', inputs=["fire6/conv1x1_2","fire6/conv3x3_2"], merge_mode="concat", concat_axis=1)
#fire 7
self.model.add_node(Convolution2D(48, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire7/conv1x1_1', input='fire6/concat')
self.model.add_node(Convolution2D(192, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire7/conv1x1_2', input='fire7/conv1x1_1')
self.model.add_node(Convolution2D(192, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire7/conv3x3_2', input='fire7/conv1x1_1')
self.model.add_node(Activation("linear"),name='fire7/concat', inputs=["fire7/conv1x1_2","fire7/conv3x3_2"], merge_mode="concat", concat_axis=1)
#fire 8
self.model.add_node(Convolution2D(64, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire8/conv1x1_1', input='fire7/concat')
self.model.add_node(Convolution2D(256, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire8/conv1x1_2', input='fire8/conv1x1_1')
self.model.add_node(Convolution2D(256, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire8/conv3x3_2', input='fire8/conv1x1_1')
self.model.add_node(Activation("linear"),name='fire8/concat', inputs=["fire8/conv1x1_2","fire8/conv3x3_2"], merge_mode="concat", concat_axis=1)
#maxpool 8
self.model.add_node(MaxPooling2D((2,2)),name='pool8', input='fire8/concat')
#fire 9
self.model.add_node(Convolution2D(64, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire9/conv1x1_1', input='pool8')
self.model.add_node(Convolution2D(256, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire9/conv1x1_2', input='fire9/conv1x1_1')
self.model.add_node(Convolution2D(256, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire9/conv3x3_2', input='fire9/conv1x1_1')
self.model.add_node(Activation("linear"),name='fire9/concat', inputs=["fire9/conv1x1_2","fire9/conv3x3_2"], merge_mode="concat", concat_axis=1)
self.model.add_node(Dropout(0.5),input='fire9/concat',name='drop9')
#conv 10
self.model.add_node(Convolution2D(self.cfgs['nb_classes'], 1, 1, activation='relu', init='glorot_uniform',border_mode='valid'),name='conv_final', input='drop9')
#avgpool 1
self.model.add_node(AveragePooling2D((13,13)),name='pool_final', input='conv_final')
self.model.add_node(Flatten(),name='flatten',input='pool_final')
self.model.add_node(Activation(self.cfgs['activation']),input='flatten',name='prob')
self.model.add_output(name='output',input='prob')
if not self.cfgs['model_weights_file'] == None:
self.init_from_this()
#if
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
#def
def init_from_this(self):
weights_file = self.cfgs['model_weights_file']
if weights_file.endswith('.hdf5'):
self.load_weights(weights_file)
self.io.print_info('Initalized from {0}'.format(weights_file))
#if
if weights_file.endswith('.caffemodel'):
self.io.print_warning('Loading from caffe model not implemented starting with random weights')
#if
#def
def load_weights(self, filepath):
'''
This method does not make use of Sequential.set_weights()
for backwards compatibility.
'''
# Loads weights from HDF5 file
import h5py
f = h5py.File(filepath)
for k in range(f.attrs['nb_layers']):
layer_type = self.model.layers[k].get_config()['name']
layer_name_string = '{0}_layer_{1}'.format(layer_type,k)
if layer_name_string in self.cfgs['ignore_while_loading']:
self.io.print_warning('Ignoring weights from {0}'.format(layer_name_string))
continue
#if
layer_name = 'layer_{}'.format(k)
g = f[layer_name]
self.io.print_info('Transfering parameters from {0}'.format(layer_name_string))
weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
self.model.layers[k].set_weights(weights)
#for
f.close()
self.io.print_info(self.model.layers[-1].get_config().values())
#def
def compile(self,compile_cfgs):
try:
sgd = SGD(lr=compile_cfgs['lr'], decay=compile_cfgs['decay'], momentum=compile_cfgs['momentum'], nesterov=True)
self.model.compile(loss=loss_functions_dict[compile_cfgs['loss']], optimizer=sgd)
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
#def
class GoogleNet():
def __init__(self):
self.layer_list = []
self.config_file = None
self.io = EmbeddingIO(None)
self.init = False
self.cfgs = None
self.loss_functions = ObjectiveFunctions()
#def
def configure(self,config_file):
self.config_file = config_file
self.init = False
if not os.path.exists(self.config_file):
self.io.print_error('Could not find config file for GoogLeNet {0}'.format(self.config_file))
self.init = False
return
#if
pfile = open(self.config_file,'r')
self.cfgs = yaml.load(pfile)
pfile.close()
self.init = True
#def
def add_to_graph(self, *args, **kwargs):
self.model.add_node(*args, **kwargs)
self.last_added_node = kwargs['name']
self.layer_list.append(kwargs['name'])
return kwargs['name']
#def
def add_bn_conv_layer(self, *args, **kwargs):
layer_name = kwargs['name']
input_layer = kwargs['input']
del kwargs['name']
del kwargs['input']
kwargs['border_mode'] = 'same'
if 'padding' in kwargs:
layer_name = layer_name + '_pad'
self.add_to_graph(ZeroPadding2D(padding=kwargs['padding']), name=layer_name, input=input_layer)
input_layer = layer_name
del kwargs['padding']
#if
# CONV with linear activation by default
layer_name = layer_name + '_conv'
self.add_to_graph(Convolution2D(*args, **kwargs), name=layer_name, input=input_layer)
# Batch normalization added directly on output of a linear layer
input_layer = layer_name
layer_name = layer_name + '_bn'
_ = self.add_to_graph(BatchNormalization(mode=0, epsilon=0.0001, axis=1), name=layer_name, input=input_layer)
# Standard normalization
input_layer = layer_name
layer_name = layer_name + '_nonlin'
_ = self.add_to_graph(Activation('relu'), name=layer_name, input=input_layer)
return layer_name
#def
def add_inception(self, input_layer, list_nb_filter, base_name):
tower_1_1 = self.add_bn_conv_layer(name=base_name+'tower_1_1', input=input_layer, nb_filter=list_nb_filter[0], nb_row=1, nb_col=1)
tower_2_1 = self.add_bn_conv_layer(name=base_name+'tower_2_1', input=input_layer, nb_filter=list_nb_filter[1], nb_row=1, nb_col=1)
tower_2_2 = self.add_bn_conv_layer(name=base_name+'tower_2_2', input=tower_2_1, nb_filter=list_nb_filter[2], nb_row=3, nb_col=3)
tower_3_1 = self.add_bn_conv_layer(name=base_name+'tower_3_1', input=input_layer, nb_filter=list_nb_filter[3], nb_row=1, nb_col=1)
tower_3_2 = self.add_bn_conv_layer(name=base_name+'tower_3_2', input=tower_3_1, nb_filter=list_nb_filter[4], nb_row=5, nb_col=5)
tower_4_1 = self.add_to_graph(MaxPooling2D((3, 3), strides=(1, 1), border_mode='same'), name=base_name+'tower_4_1', input=input_layer)
tower_4_2 = self.add_bn_conv_layer(name=base_name+'tower_4_2', input=tower_4_1, nb_filter=list_nb_filter[5], nb_row=1, nb_col=1)
self.add_to_graph(Activation("linear"), name=base_name, inputs=[tower_1_1,tower_2_2,tower_3_2,tower_4_2], merge_mode="concat", concat_axis=1)
self.io.print_info('Added Inception {0}'.format(base_name))
def define(self):
try:
self.model = Graph()
self.model.add_input(name='input', input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width']))
pad_1 = self.add_to_graph(ZeroPadding2D((3,3)), name='pad_1',input='input')
conv_1 = self.add_bn_conv_layer(name='conv_1', input=self.last_added_node, nb_filter=64, nb_row=7, nb_col=7,subsample=(2,2))
pad_2 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_2',input=self.last_added_node)
max_1 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name='max_1', input=self.last_added_node)
pad_3 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_3', input=self.last_added_node)
conv_2 = self.add_bn_conv_layer( name='conv_2', input=self.last_added_node, nb_filter=192, nb_row=3, nb_col=3 )
pad_4 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_4', input=self.last_added_node)
max_2 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name='max_2', input=self.last_added_node)
#Inception_layer 1
nb_kernel_inception_1=[64,96,128,16,32,32]
self.add_inception(self.last_added_node,nb_kernel_inception_1,'inception_1')
#Inception_layer 2
nb_kernel_inception_2=[128,128,192,32,96,64]
self.add_inception(self.last_added_node,nb_kernel_inception_2,'inception_2')
pad_5 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_5', input=self.last_added_node)
max_3 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name='max_3', input=self.last_added_node)
#Inception_layer 3
nb_kernel_inception_3=[192,96,208,16,48,64]
self.add_inception(self.last_added_node,nb_kernel_inception_3,'inception_3')
#Inception_layer 4
nb_kernel_inception_4=[160,112,224,24,64,64]
self.add_inception(self.last_added_node,nb_kernel_inception_4,'inception_4')
#Inception_layer 5
nb_kernel_inception_5=[128,128,256,24,64,64]
self.add_inception(self.last_added_node,nb_kernel_inception_5,'inception_5')
#Inception_layer 6
nb_kernel_inception_6=[112,144,288,32,64,64]
self.add_inception(self.last_added_node,nb_kernel_inception_6,'inception_6')
#Inception_layer 7
nb_kernel_inception_7=[256,160,320,32,128,128]
self.add_inception(self.last_added_node,nb_kernel_inception_7,'inception_7')
pad_6 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_6', input=self.last_added_node)
max_4 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name='max_4', input=self.last_added_node)
#Inception_layer 8
nb_kernel_inception_8=[256,160,320,32,128,128]
self.add_inception(self.last_added_node,nb_kernel_inception_8,'inception_8')
#Inception_layer 9
nb_kernel_inception_9=[384,192,384,48,128,128]
self.add_inception(self.last_added_node,nb_kernel_inception_9,'inception_9')
self.add_to_graph(AveragePooling2D((7,7)), name='pool_1', input=self.last_added_node)
self.add_to_graph(Flatten(), name='flatten', input='pool_1')
self.add_to_graph(Dense(self.cfgs['nb_classes']), name='logits', input='flatten')
self.add_to_graph(Activation(self.cfgs['activation']), name='prob', input='logits')
# Output to the Graph
self.model.add_output(name='output', input='prob')
self.init_from_this()
except Exception as err:
self.io.print_error('Error configuring the model, {0}'.format(err))
self.init = False
return
#try
self.init = True
#def
def init_from_this(self):
weights_file = self.cfgs['model_weights_file']
if not weights_file == 'None':
self.load_weights(weights_file)
self.io.print_info('Weights Initalized from {0}'.format(weights_file))
#if
#def
def load_weights(self, filepath):
if filepath.endswith('.npz'):
pfile = open(filepath,'r')
graph = np.load(pfile)['graph'].item()
for node_name,weights in graph.items():
if node_name in self.cfgs['ignore_while_loading']:
self.io.print_warning('Ignoring weights from {0}'.format(node_name))
continue
#if
self.io.print_info('Transfering parameters from {0}'.format(node_name))
self.model.nodes[node_name].set_weights(weights)
#for
pfile.close()
elif filepath.endswith('.hdf5'):
self.model.load_weights(filepath)
else:
self.io.print_error('Unknown model weights file {}'.format(filepath))
#if
self.io.print_info(self.model.nodes['prob'].get_config())
def setup_loss_function(self,w):
self.loss_functions.set_weights(w)
#def
def compile(self,compile_cfgs):
try:
opt = optimizer_dict[compile_cfgs['optimizer']](lr=compile_cfgs['lr'], epsilon=compile_cfgs['epsilon'])
self.model.compile(loss={'output':self.loss_functions.dict[compile_cfgs['loss']]}, optimizer=opt)
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
class InceptionV3RNN():
def __init__(self):
self.layer_list = []
self.config_file = None
self.io = EmbeddingIO(None)
self.init = False
self.cfgs = None
#def
def configure(self,config_file):
self.config_file = config_file
self.init = False
if not os.path.exists(self.config_file):
self.io.print_error('Could not find config file for InceptionV3 {0}'.format(self.config_file))
self.init = False
return
#if
pfile = open(self.config_file,'r')
self.cfgs = yaml.load(pfile)
pfile.close()
self.init = True
#def
def add_to_graph(self, *args, **kwargs):
self.model.add_node(*args, **kwargs)
self.last_added_node = kwargs['name']
self.layer_list.append(kwargs['name'])
return kwargs['name']
#def
def add_bn_conv_layer(self, *args, **kwargs):
layer_name = kwargs['name']
input_layer = kwargs['input']
del kwargs['name']
del kwargs['input']
if 'padding' in kwargs:
layer_name = layer_name + '_pad'
self.add_to_graph(ZeroPadding2D(padding=kwargs['padding']), name=layer_name, input=input_layer)
input_layer = layer_name
del kwargs['padding']
#if
# CONV with linear activation by default
layer_name = layer_name + '_conv'
self.add_to_graph(Convolution2D(*args, **kwargs), name=layer_name, input=input_layer)
# Batch normalization added directly on output of a linear layer
input_layer = layer_name
layer_name = layer_name + '_bn'
_ = self.add_to_graph(BatchNormalization(mode=0, epsilon=0.0001, axis=1), name=layer_name, input=input_layer)
# Standard normalization
input_layer = layer_name
layer_name = layer_name + '_nonlin'
_ = self.add_to_graph(Activation('relu'), name=layer_name, input=input_layer)
return layer_name
#def
def add_inceptionA(self, input_layer, list_nb_filter, base_name):
l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1)
l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1)
l2_2 = self.add_bn_conv_layer(name=base_name+'_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=5, nb_col=5, padding=(2,2))
l3_1 = self.add_bn_conv_layer(name=base_name+'_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1)
l3_2 = self.add_bn_conv_layer(name=base_name+'_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=3, nb_col=3, padding=(1,1))
l3_3 = self.add_bn_conv_layer(name=base_name+'_l3_3', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=3, nb_col=3, padding=(1,1))
l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1,1)), name=base_name+'_14_1', input=input_layer)
l4_2 = self.add_to_graph(AveragePooling2D(pool_size=(3,3), strides=(1,1)), name=base_name+'_14_2', input=l4_1)
l4_3 = self.add_bn_conv_layer(name=base_name+'_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1)
self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_2, l3_3, l4_3], merge_mode="concat", concat_axis=1)
self.io.print_info('Added Inception-A {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def add_inceptionB(self, input_layer, list_nb_filter, base_name):
l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=3, nb_col=3, subsample=(2, 2))
l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1)
l2_2 = self.add_bn_conv_layer(name=base_name+'_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=3, nb_col=3, padding=(1,1))
l2_3 = self.add_bn_conv_layer(name=base_name+'_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=3, nb_col=3, subsample=(2,2))
l3_1 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name=base_name+'_13_1', input=input_layer)
self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_3, l3_1], merge_mode="concat", concat_axis=1)
self.io.print_info('Added Inception-B {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def add_inceptionC(self, input_layer, list_nb_filter, base_name):
l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1)
l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1)
l2_2 = self.add_bn_conv_layer(name=base_name+'_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=7, padding=(0,3))
l2_3 = self.add_bn_conv_layer(name=base_name+'_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=7, nb_col=1, padding=(3,0))
## padding and nb_row might not match with the lasagne weights
l3_1 = self.add_bn_conv_layer(name=base_name+'_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1)
l3_2 = self.add_bn_conv_layer(name=base_name+'_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=7, nb_col=1, padding=(3,0))
l3_3 = self.add_bn_conv_layer(name=base_name+'_l3_3', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=1, nb_col=7, padding=(0,3))
l3_4 = self.add_bn_conv_layer(name=base_name+'_l3_4', input=l3_3, nb_filter=list_nb_filter[2][3], nb_row=7, nb_col=1, padding=(3,0))
l3_5 = self.add_bn_conv_layer(name=base_name+'_l3_5', input=l3_4, nb_filter=list_nb_filter[2][4], nb_row=1, nb_col=7, padding=(0,3))
l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1,1)), name=base_name+'_14_1', input=input_layer)
l4_2 = self.add_to_graph(AveragePooling2D(pool_size=(3,3), strides=(1,1)), name=base_name+'_14_2', input=l4_1)
l4_3 = self.add_bn_conv_layer(name=base_name+'_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1)
self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_3, l3_5, l4_3], merge_mode="concat", concat_axis=1)
self.io.print_info('Added Inception-C {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def add_inceptionD(self, input_layer, list_nb_filter, base_name):
l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1)
l1_2 = self.add_bn_conv_layer(name=base_name+'_l1_2', input=l1_1, nb_filter=list_nb_filter[0][1], nb_row=3, nb_col=3, subsample=(2,2))
l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1)
l2_2 = self.add_bn_conv_layer(name=base_name+'_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=7, padding=(0,3))
l2_3 = self.add_bn_conv_layer(name=base_name+'_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=7, nb_col=1, padding=(3,0))
l2_4 = self.add_bn_conv_layer(name=base_name+'_l2_4', input=l2_3, nb_filter=list_nb_filter[1][2], nb_row=3, nb_col=3, subsample=(2,2))
l3_1 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name=base_name+'_13_1', input=input_layer)
self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_2, l2_4, l3_1], merge_mode="concat", concat_axis=1)
self.io.print_info('Added Inception-D {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def add_inceptionE(self, input_layer, list_nb_filter, base_name, pool_mode):
l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1)
l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1)
l2_2a = self.add_bn_conv_layer(name=base_name+'_l2_2a', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=3, padding=(0,1))
l2_2b = self.add_bn_conv_layer(name=base_name+'_l2_2b', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=3, nb_col=1, padding=(1,0))
l3_1 = self.add_bn_conv_layer(name=base_name+'_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1)
l3_2 = self.add_bn_conv_layer(name=base_name+'_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=3, nb_col=3, padding=(1,1))
l3_3a = self.add_bn_conv_layer(name=base_name+'_l3_3a', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=1, nb_col=3, padding=(0,1))
l3_3b = self.add_bn_conv_layer(name=base_name+'_l3_3b', input=l3_2, nb_filter=list_nb_filter[2][3], nb_row=3, nb_col=1, padding=(1,0))
l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1,1)), name=base_name+'_14_1', input=input_layer)
l4_2 = self.add_to_graph(pooling_dict[pool_mode](pool_size=(3,3), strides=(1,1)), name=base_name+'_14_2', input=l4_1)
l4_3 = self.add_bn_conv_layer(name=base_name+'_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1)
self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_2a, l2_2b, l3_3a, l3_3b, l4_3], merge_mode="concat", concat_axis=1)
self.io.print_info('Added Inception-E {0}'.format(base_name))
# https://github.com/fchollet/keras/issues/391
def define(self):
try:
self.model = Graph()
self.model.add_input(name='input_image', input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width']))
self.model.add_input(name='input_label', input_shape=(1, self.cfgs['nb_classes']))
# CONVNET MODEL STARTS HERE
#
# Part of the network which is defined in the config file should move here
#
cgfs_nodes = self.cfgs['nodes']
for node in cgfs_nodes:
if not node['type'] == 'Activation':
self.add_to_graph(layer_dict[node['type']](**node['parameter']), name=node['name'] , input=node['input_image'])
else:
self.add_to_graph(layer_dict[node['type']](node['parameter']['mode']), name=node['name'] , input=node['input_image'])
#if
self.io.print_info('Added {1}:{0}'.format(node['type'],node['name']))
#for
self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64,), (48, 64), (64, 96, 96), (32,)), base_name='mixed')
self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64,), (48, 64), (64, 96, 96), (64,)), base_name='mixed_1')
self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64,), (48, 64), (64, 96, 96), (64,)), base_name='mixed_2')
self.add_inceptionB(input_layer=self.last_added_node, list_nb_filter=((384,), (64, 96, 96)), base_name='mixed_3')
self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192,), (128, 128, 192), (128, 128, 128, 128, 192), (192,)), base_name='mixed_4')
self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192,), (160, 160, 192), (160, 160, 160, 160, 192), (192,)), base_name='mixed_5')
self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192,), (160, 160, 192), (160, 160, 160, 160, 192), (192,)), base_name='mixed_6')
self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192,), (192, 192, 192), (192, 192, 192, 192, 192), (192,)), base_name='mixed_7')
self.add_inceptionD(input_layer=self.last_added_node, list_nb_filter=((192, 320), (192, 192, 192, 192)), base_name='mixed_8')
self.add_inceptionE(input_layer=self.last_added_node, list_nb_filter=((320,), (384, 384, 384), (448, 384, 384, 384), (192,)), pool_mode='average', base_name='mixed_9')
self.add_inceptionE(input_layer=self.last_added_node, list_nb_filter=((320,), (384, 384, 384), (448, 384, 384, 384), (192,)), pool_mode='max', base_name='mixed_10')
self.add_to_graph(AveragePooling2D(pool_size=(5,5)), name='pool3', input=self.last_added_node)
self.io.print_info('Added {1}:{0}'.format('AveragePooling',self.last_added_node))
self.add_to_graph(Flatten(), name='flatten', input='pool3')
self.io.print_info('Added {1}:{0}'.format('Flatten',self.last_added_node))
self.add_to_graph(Dense(self.cfgs['nb_classes']), name='softmax', input='flatten')
self.io.print_info('Added {1}:{0}'.format('Dense',self.last_added_node))
# CONVNET MODEL ENDS HERE
# RNN MODEL STARTS HERE
self.add_to_graph(Embedding(self.cfgs['nb_classes'],self.cfgs['label_embedding_matrix'],self.cfgs['time_stamps']),name='label_embedding',input='input_label'))
# RNN MODEL ENDS HERE
if not self.cfgs['model_weights_file'] == None:
self.init_from_this()
#if
except Exception as err:
self.io.print_error('Error configuring the model, {0}'.format(err))
self.init = False
return
#try
self.init = True
#def
def init_from_this(self):
weights_file = self.cfgs['model_weights_file']
if not weights_file == 'None':
self.load_weights(weights_file)
self.io.print_info('Weights Initalized from {0}'.format(weights_file))
#if
#def
def load_weights(self, filepath):
if filepath.endswith('.npz'):
pfile = open(filepath,'r')
graph = np.load(pfile)['graph'].item()
for node_name,weights in graph.items():
if node_name in self.cfgs['ignore_while_loading']:
self.io.print_warning('Ignoring weights from {0}'.format(node_name))
continue
#if
self.io.print_info('Transfering parameters from {0}'.format(node_name))
self.model.nodes[node_name].set_weights(weights)
#for
pfile.close()
elif filepath.endswith('.hdf5'):
self.model.load_weights(filepath)
else:
self.io.print_error('Unknown model weights file {}'.format(filepath))
#if
self.io.print_info(self.model.nodes['softmax'].get_config())
def compile(self,compile_cfgs):
try:
opt = optimizer_dict[compile_cfgs['optimizer']](lr=compile_cfgs['lr'], epsilon=compile_cfgs['epsilon'])
self.model.compile(loss={'output':loss_functions_dict[compile_cfgs['loss']]}, optimizer=opt)
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
#def
class LMDBParser():
def __init__(self, lmdb_dir_path, log_dir='/tmp', keep=32):
# 1. Path to directory containing data.mdb
# 2. keep which will take 1000 records from lmdb file into CPU memory
self.lmdb_dir_path = lmdb_dir_path
self.keep = keep
self.passes = 0
self.datum_seen = 0
self.mean_weight = 1.4
self.label_distribution = None
self.label_statistics = None
self.label_statistics_train = None
self.io = EmbeddingIO(log_dir,
log_type='lmdb-parsing-{0}'.format(
lmdb_dir_path.split('/')[-1]))
if not os.path.exists(self.lmdb_dir_path):
self.io.print_error('Could not find LMDB path {0}'.format(
self.lmdb_dir_path))
self.init = False
return
try:
self.lmdb_env = lmdb.open(self.lmdb_dir_path,
readonly=True,
lock=False)
self.lmdb_txn = self.lmdb_env.begin()
self.lmdb_cursor = self.lmdb_txn.cursor()
self.read_label = self.lmdb_txn.get
self.lmdb_cursor.first()
self.datum = Datum()
self.lmdb_entries = int(self.lmdb_txn.stat()['entries'])
except Exception as e:
self.io.print_error('Error reading LDMB file {0},{1}'.format(
self.lmdb_dir_path, e))
self.init = False
return
self.io.print_info('Train LMDB file : {0} '.format(self.lmdb_dir_path))
self.init = True
self.get_batch = {
'None': self.get_next_slice,
'drop_out': self.get_next_slice,
'mikolov': self.get_next_slice_sampled,
'mikolov_original': self.get_next_slice_sampled,
'mikolov_drop_out': self.get_next_slice_sampled,
'mikolov_em_drop_out': self.get_next_slice_sampled,
'mikolov_tetris_drop_out': self.get_next_slice_sampled,
'smote': self.get_next_slice_smote,
'captions': self.get_next_slice_captions,
'captions_mikolov': self.get_next_slice_captions_mikolov
}
def reset_data_stats(self):
self.io.print_info('Mean weight for sampling {}'.format(
self.mean_weight))
self.label_distribution = np.ones(
(len(self.vocabulary), )) / (len(self.vocabulary) + 0.0)
self.label_statistics = np.zeros((len(self.vocabulary), ))
self.label_statistics_train = np.zeros((len(self.vocabulary), ))
def set_params_test(self, vocabulary):
self.vocabulary = vocabulary
def set_params(self,
crop_size,
mean_pixel_value,
keep,
channel_swap,
pixel_scaling,
label_sampling,
vocabulary,
label_mapping,
max_overlap=4,
image_shuffle=[2, 0, 1],
label_reduce=None,
label_embedding=None):
self.keep = keep
self.crop_size = crop_size
self.mean_pixel_value = mean_pixel_value
self.vocabulary = vocabulary
self.label_mapping = label_mapping
self.channel_swap = channel_swap
self.pixel_scaling = pixel_scaling
self.label_sampling = label_sampling
self.image_shuffle = image_shuffle
self.label_reduce = label_reduce
self.max_overlap = max_overlap
self.label_embedding = label_embedding
self.reset_data_stats()
self.build_dataset_cdf()
def set_captions_params(self,
LUT=None,
max_caption_len=None,
caption_vocabulary=None,
label_sampling=None):
self.LUT = LUT
self.max_caption_len = max_caption_len
self.caption_vocabulary = caption_vocabulary
self.label_sampling = label_sampling
self.START = self.LUT['BOS']
self.END = self.LUT['EOS']
self.PAD = self.LUT['PAD']
def build_dataset_cdf(self):
if self.label_mapping is None:
return
dist = [(k, len(v)) for k, v in self.label_mapping.items()
if k in self.vocabulary]
concepts = self.label_mapping['vocabulary']
concepts_count = [len(d) for d in self.label_mapping['mapping']]
sort_idx = np.argsort(-np.array(concepts_count))
sorted_concepts = [concepts[idx] for idx in sort_idx]
concepts_pdf = np.array(concepts_count) / (np.sum(concepts_count) +
0.0)
concepts_cdf = np.cumsum(concepts_pdf)
self.concepts_pdf = np.array(
[concepts_pdf[sorted_concepts.index(c)] for c in self.vocabulary])
self.concepts_cdf = np.array(
[concepts_cdf[sorted_concepts.index(c)] for c in self.vocabulary])
self.concepts_cdf_norm = self.concepts_cdf / np.sum(self.concepts_cdf)
self.io.print_info('Training set statistics set up')
# def build_dataset_cdf(self):
# if self.label_mapping is None:
# return
# dist = [(k, len(v)) for k, v in self.label_mapping.items() if k in self.vocabulary]
# concepts = [d[0] for d in dist]
# concepts_count = [d[1] for d in dist]
# sort_idx = np.argsort(-np.array(concepts_count))
# sorted_concepts = [concepts[idx] for idx in sort_idx]
# concepts_pdf = np.array(concepts_count) / (np.sum(concepts_count) + 0.0)
# concepts_cdf = np.cumsum(concepts_pdf)
# self.concepts_pdf = np.array([concepts_pdf[sorted_concepts.index(c)] for c in self.vocabulary])
# self.concepts_cdf = np.array([concepts_cdf[sorted_concepts.index(c)] for c in self.vocabulary])
# self.concepts_cdf_norm = self.concepts_cdf / np.sum(self.concepts_cdf)
# self.io.print_info('Training set statistics set up')
def get_label_Frimgfile(self, filename):
dirname = '/nas/datasets/getty-collection/getty-all/'
key = dirname + filename
value = self.lmdb_txn.get(key)
#import ipdb; ipdb.set_trace()
if not (value == None):
self.datum.ParseFromString(value)
lab = self.datum.label
else:
lab = []
return lab
def get_labels(self, f, ID):
#import ipdb; ipdb.set_trace()
stream = f(ID)
return pickle.loads(stream)
def create_lmdb_keys_labels(self):
import ipdb
ipdb.set_trace()
l = 0
env = lmdb.open("./keywording_test_images_labels", map_size=1e12)
try:
with env.begin(write=True) as txn:
while 1:
keys, labels = self.get_next_slice_img_lab()
pickled_labels = [
pickle.dumps(label, protocol=1) for label in labels
]
import ipdb
ipdb.set_trace()
if self.passes < 1:
for i, key in enumerate(keys):
ID, _ = os.path.splitext(os.path.basename(key))
if pickled_labels[i] != None:
txn.put(ID.encode('ascii'), pickled_labels[i])
l = l + 1
print "Processes %d" % l
#break
else:
break
except Exception as e:
print 'problem processing image %s' % ID
import ipdb
ipdb.set_trace()
def create_keys_labels_dict(self):
import ipdb
ipdb.set_trace()
keys_labels = dict()
l = 0
import ipdb
ipdb.set_trace()
while 1:
keys, labels = self.get_next_slice_img_lab()
#pickled_labels = [pickle.dumps(label, protocol=1) for label in labels]
if self.passes < 1:
for i, key in enumerate(keys):
ID, _ = os.path.splitext(os.path.basename(key))
if len(labels[i]) == len(self.vocabulary):
keys_labels[ID] = labels[i]
print keys_labels[ID].sum()
l = l + 1
#print "Processes %d" % l
#break
else:
print "Not Processes image %s" % ID
break
else:
break
import ipdb
ipdb.set_trace()
pickle.dump(
keys_labels,
open('/nas2/praveen/home/rnd-libs/rnd_libs/lib/keras/key_labels',
'wb'))
import ipdb
ipdb.set_trace()
def get_next_slice_img_lab(self):
# Warning : This is hard coded for inception_v3 model should be set from config file
labels = np.zeros((self.keep, len(self.vocabulary)), dtype='u1')
keys = []
# TODO : HSS - setting fields from YAML
# [HSS] get rid of while loops with cleaner sampling file
i = 0
while i < self.keep:
# if End of LMDB file start from the begining
if not self.lmdb_cursor.next():
self.lmdb_cursor.first()
self.passes += 1
self.io.print_info(
'Start of next pass over the test set {0}'.format(
self.passes))
# Read a key-value record from the queue
try:
key, value = self.lmdb_cursor.item()
self.datum.ParseFromString(value)
lab = self.datum.label
if lab is None:
self.io.print_warning(
'{1}/{2} Skipping {0}, no useful label found'.format(
key, i, self.keep))
continue
labels[i, :] = lab
keys.append(key)
i += 1
except Exception as e:
self.io.print_info('Skipping corrupted images, {0}'.format(e))
# update label statistics and sampling distribution
#self.update_label_stats(labels)
test_labels_full = labels
#test_labels_semi = self.label_drop(keys, test_labels_full) if 'drop_out' in self.label_sampling else test_labels_full
# if self.label_embedding is not None:
# train_labels_full = np.dot(test_labels_full, self.label_embedding)
# train_labels_semi = np.dot(test_labels_semi, self.label_embedding)
# else:
# train_labels_full = test_labels_full
# train_labels_semi = test_labels_semi
return keys, test_labels_full
def get_next_slice(self):
# Warning : This is hard coded for inception_v3 model should be set from config file
images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1]))
labels = np.zeros((self.keep, len(self.vocabulary)))
keys = []
# TODO : HSS - setting fields from YAML
# [HSS] get rid of while loops with cleaner sampling file
i = 0
while i < self.keep:
# if End of LMDB file start from the begining
if not self.lmdb_cursor.next():
self.lmdb_cursor.first()
self.passes += 1
self.io.print_info(
'Start of next pass over the training set {0}'.format(
self.passes))
# Read a key-value record from the queue
try:
key, value = self.lmdb_cursor.item()
arr, lab = self.get_image_label(value)
if lab is None:
self.io.print_warning(
'{1}/{2} Skipping {0}, no useful label found'.format(
key, i, self.keep))
continue
images[i, :, :, :] = arr
labels[i, :] = lab
keys.append(key)
i += 1
except Exception as e:
self.io.print_info('Skipping corrupted images, {0}'.format(e))
# update label statistics and sampling distribution
self.update_label_stats(labels)
test_labels_full = labels
test_labels_semi = self.label_drop(
keys, test_labels_full
) if 'drop_out' in self.label_sampling else test_labels_full
if self.label_embedding is not None:
train_labels_full = np.dot(test_labels_full, self.label_embedding)
train_labels_semi = np.dot(test_labels_semi, self.label_embedding)
else:
train_labels_full = test_labels_full
train_labels_semi = test_labels_semi
return keys, images, train_labels_semi, test_labels_full
def get_next_slice_sampled(self):
# Warning : This is hard coded for inception_v3 model should be set from config file
images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1]))
labels = np.zeros((self.keep, len(self.vocabulary)))
pseudo_labels = np.zeros((self.keep, len(self.vocabulary)))
keys = []
#
# while since some smaples might not be in this lmdb
#
i = 0
#
# [HSS] get rid of while loops with cleaner sampling file
#
while i < self.keep:
# sample random class
if 'tetris' in self.label_sampling:
# sample classes which havent been seen
if self.datum_seen == 0:
random_class = np.random.choice(self.vocabulary, 1)[0]
else:
prob = 1.0 - self.label_statistics_train / (
np.max(self.label_statistics_train) + 0.0)
prob /= np.sum(prob)
random_class = np.random.choice(self.vocabulary, 1,
p=prob)[0]
else:
random_class = np.random.choice(
self.label_mapping['vocabulary'], 1)[0]
# get the class index to build the vector later
random_class_index = self.label_mapping['vocabulary_mapping'][
random_class]
# select random image sample for the sampled class
image_index = np.random.choice(
self.label_mapping['mapping'][random_class_index], 1)[0]
# fetching actual image string from integer to image mapping
key = str(self.label_mapping['images'][image_index])
# casting to str is important, because sometimes the key is unicode
# look up in LMDB for image-data
value = self.lmdb_txn.get(key)
# skip if not in current lmdb
if value is None:
continue
self.datum_seen += 1
if self.datum_seen >= self.lmdb_entries:
self.datum_seen = 0
self.passes += 1
self.io.print_info(
'Start of next pass over the training set {0}'.format(
self.passes))
self.reset_data_stats()
try:
arr, lab = self.get_image_label(value)
if lab is None:
self.io.print_warning(
'{1}/{2} Skipping {0}, no useful label found'.format(
key, i, self.keep))
continue
nlab = np.zeros_like(lab)
#
# set random class index to 1 rest to 0
#
nlab[random_class_index] = 1.0
if len(lab) == 0:
continue
images[i, :, :, :] = arr
labels[i, :] = lab
pseudo_labels[i, :] = nlab
keys.append(key)
i += 1
except Exception as e:
self.io.print_info('Skipping corrupted images, {0}'.format(e))
# update label statistics and sampling distribution
self.update_label_stats(labels)
base_labels = pseudo_labels if self.label_sampling == 'mikolov' else labels
base_labels = self.label_drop(
keys,
base_labels) if 'drop_out' in self.label_sampling else base_labels
test_labels_full = labels
self.label_statistics_train += np.sum(base_labels, axis=0)
if self.label_embedding is not None:
train_labels_semi = np.dot(base_labels, self.label_embedding)
else:
train_labels_semi = base_labels
return keys, images, train_labels_semi, test_labels_full
def update_label_stats(self, labels):
self.label_statistics += np.sum(labels, axis=0)
self.label_distribution[np.where(
self.label_statistics)] /= self.label_statistics[np.where(
self.label_statistics)]
self.label_distribution /= np.sum(self.label_distribution)
def get_next_slice_smote(self):
# Warning : This is hard coded for inception_v3 model should be set from config file
images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1]))
labels = np.zeros((self.keep, len(self.vocabulary)))
all_labels = np.zeros((self.keep, len(self.vocabulary)))
keys = []
#
# while since some smaples might not be in this lmdb
#
i = 0
#
# [HSS] get rid of while loops with cleaner sampling file
#
current_labels = None
while i < self.keep:
# sample random class
random_class = np.random.choice(self.vocabulary, 1)[0]
# self.io.print_info('Sampled {0}'.format(random_class))
# get the class index to build the vector later
random_class_index = self.vocabulary.index(random_class)
# select random image sample for the sampled class
key = str(np.random.choice(self.label_mapping[random_class], 1)[0])
# casting to str is important, because sometimes the key is unicode
# look up in LMDB for image-data
value = self.lmdb_txn.get(key)
# skip if not in current lmdb
if value is None:
continue
self.datum_seen += 1
if self.datum_seen >= self.lmdb_entries:
self.datum_seen = 0
self.passes += 1
self.io.print_info(
'Start of next pass over the training set {0}'.format(
self.passes))
try:
arr, lab = self.get_image_label(value)
if lab is None:
self.io.print_warning(
'{1}/{2} Skipping {0}, no useful label found'.format(
key, i, self.keep))
continue
# current_labels, flag = self.is_good_to_add(current_labels,lab)
current_labels, lab, true_labels, flag = self.get_good_labels(
current_labels, lab)
if not flag:
continue
images[i, :, :, :] = arr
labels[i, :] = lab
all_labels[i, :] = true_labels
keys.append(key)
i += 1
# self.io.print_info('Current batch size {}'.format(len(images)))
except Exception as e:
self.io.print_info('Skipping corrupted images, {0}'.format(e))
return keys, images, labels, all_labels
def is_good_to_add(self, current, lab):
if current is None:
return np.copy(lab), True
dot = np.dot(current, lab)
# self.io.print_info('Dot product {0}'.format(dot))
if dot >= self.max_overlap:
return np.copy(current), False
else:
current = np.bitwise_or(current, lab)
# self.io.print_info('Used labels {0}'.format(np.sum(current)))
return np.copy(current), True
def get_good_labels(self, current, lab):
all_labels = np.copy(lab)
if current is None:
return np.copy(lab), np.copy(lab), all_labels, True
new_lab = np.bitwise_and(lab, np.invert(np.bitwise_and(current, lab)))
if np.sum(new_lab) == 0:
return np.copy(current), np.copy(lab), all_labels, False
else:
current = np.bitwise_or(current, lab)
return np.copy(current), np.copy(new_lab), all_labels, True
def get_image_label(self, value):
self.datum.ParseFromString(value)
arr = np.fromstring(self.datum.data,
dtype=np.uint8).reshape(self.datum.channels,
self.datum.height,
self.datum.width)
arr = arr.transpose(1, 2, 0)
# This casting is required since uint8 and float32 cause an under-flow while mean subtraction below
arr = np.asarray(arr, dtype=np.float32)
if self.crop_size is not None:
arr = cv2.resize(arr, self.crop_size).astype(np.float32)
if self.mean_pixel_value is not None:
arr -= self.mean_pixel_value
if self.channel_swap is not None:
arr = arr[:, :, self.channel_swap]
if self.pixel_scaling is not None:
arr /= self.pixel_scaling
# As theano likes it N_channels x image_H x image_w
if self.image_shuffle is not None:
arr = arr.transpose(self.image_shuffle)
# Label is a single field but could be an array
lab = self.datum.label
if self.label_reduce is not None:
dlab = np.dot(self.label_reduce, lab)
lab = np.zeros_like(dlab)
lab[np.where(dlab)[0]] = 1
if np.sum(lab) == 0:
lab = None
return arr, lab
def label_drop(self, keys, labels):
if 'em' in self.label_sampling:
# sample inversly to what network has seen in the entire history
freq = self.label_statistics_train + 0.0
non_empty_inices = np.where(self.label_statistics_train)
if not non_empty_inices == []:
freq[np.where(
self.label_statistics_train > self.mean_weight *
np.mean(self.label_statistics_train[non_empty_inices])
)] = self.datum_seen
freq /= self.datum_seen
elif 'tetris' in self.label_sampling:
residue = np.max(
self.label_statistics_train) - self.label_statistics_train
# reset if everything has been seen
if np.sum(residue) == 0:
return labels
drop_labels = np.zeros_like(labels)
# keep at most if available residue amount of samples for the label
for idx, f in enumerate(residue):
for idy in np.random.choice(np.where(labels[:, idx]), f):
drop_labels[idy][idx] = 1.0
return np.multiply(labels, drop_labels)
else:
# sample inversly to batch frequency
freq = np.sum(labels, axis=0) / (len(keys) + 0.0)
# freq = 1 - self.concepts_cdf
freq = [max(0, min(1, f)) for f in freq]
drop_labels = np.array([
np.random.binomial(1, 1 - f, len(keys)) for f in freq
]).transpose()
return np.multiply(labels, drop_labels)
def get_next_slice_captions(self):
images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1]))
partial_captions = np.zeros((self.keep, self.max_caption_len))
next_words = np.zeros((self.keep, self.max_caption_len + 1,
len(self.caption_vocabulary)))
keys = []
caption_range = range(self.max_caption_len + 1)
for i in range(self.keep):
# if End of LMDB file start from the begining
if not self.lmdb_cursor.next():
self.lmdb_cursor.first()
self.passes += 1
self.io.print_info(
'Start of next pass over the training set {0}'.format(
self.passes))
# Read a key-value record from the queue
try:
key, value = self.lmdb_cursor.item()
arr, cap = self.get_image_label(value)
caption = [self.START] + list(cap) + [self.END]
caption = pad_sequences([caption],
maxlen=self.max_caption_len,
padding='post',
value=self.PAD)[0]
images[i, :, :, :] = arr
partial_captions[i, :] = caption
next_words[i, caption_range,
np.hstack((caption, [self.PAD]))] = 1
keys.append(key)
except Exception as e:
self.io.print_info('Skipping corrupted images, {0}'.format(e))
return keys, images, partial_captions, next_words
def get_next_slice_captions_mikolov(self):
images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1]))
partial_captions = np.zeros((self.keep, self.max_caption_len))
next_words = np.zeros((self.keep, self.max_caption_len + 1,
len(self.caption_vocabulary)))
keys = []
i = 0
caption_range = range(self.max_caption_len + 1)
while i < self.keep:
# if End of LMDB file start from the begining
if not self.lmdb_cursor.next():
self.lmdb_cursor.first()
self.passes += 1
self.io.print_info(
'Start of next pass over the training set {0}'.format(
self.passes))
# Read a key-value record from the queue
try:
random_class = np.random.choice(self.vocabulary, 1)[0]
# get the class index to build the vector later
random_class_index = self.vocabulary.index(random_class)
# select random image sample for the sampled class
key = str(
np.random.choice(self.label_mapping[random_class], 1)[0])
# casting to str is important, because sometimes the key is unicode
# look up in LMDB for image-data
value = self.lmdb_txn.get(key)
arr, cap = self.get_image_label(value)
if value is None:
continue
if len(cap) + 2 > self.max_caption_len:
continue
caption = [self.START] + list(cap) + [self.END]
caption = pad_sequences([caption],
maxlen=self.max_caption_len,
padding='post',
value=self.PAD)
images[i, :, :, :] = arr
partial_captions[i, :] = caption
next_words[i, caption_range,
np.hstack((caption, [self.PAD]))] = 1
keys.append(key)
i += 1
except Exception as e:
self.io.print_info(
'Skipping corrupted invalid images, {0}'.format(e))
return keys, images, partial_captions, next_words
def get_triplets(self,
queries,
crop_size=None,
mean_value=None,
keep=None):
pass
def check_passes(self):
self.io.print_info('Done {0} passes of lmdb'.format(self.passes))
return self.passes
def close(self):
self.lmdb_env.close()
class TheanoTester():
def __init__(self, config_file, verbose=False, raw_predictions=False):
if not os.path.exists(config_file):
print 'Error : could not find config file'.format(config_file)
self.init = False
return
self.tester_config_file = config_file
self.verbose = verbose
self.raw_predictions = raw_predictions
self.read_config()
self.io = EmbeddingIO(self.cfgs['log_dir'], 'testing')
self.tester_name = self.cfgs['model_name']
self.model_config_file = self.cfgs['model_config_file']
self.init = False
self.thresholds = None
mapping_keys = [d[0] for d in self.cfgs['mapping_list']]
mapping_values = [d[1] for d in self.cfgs['mapping_list']]
self.mapping = dict(zip(mapping_keys, mapping_values))
self.softpooling_func = f
if self.tester_name not in model_dict.keys():
self.io.print_error(
'Not a valid model type {0} chose among {1}'.format(
self.tester_name, model_dict.keys()))
self.init = False
return
if not os.path.exists(self.tester_config_file):
self.io.print_error('Could not find configuration file {0}'.format(
self.tester_config_file))
self.init = False
return
self.tester = None
self.action_items = {'test': self.run, 'eval': self.eval}
self.test_lmdb = LMDBParser_test(self.cfgs['test_file'],
log_dir=self.cfgs['log_dir'])
self.resize_test = self.cfgs['resize_test']
# self.load_synset_file()
# self.test_lmdb = LMDBParser(self.cfgs['test_file'], log_dir=self.cfgs['log_dir'])
# self.test_lmdb.set_params_test(self.concepts)
def _read_threshold_file(self):
if 'threshold_file' in self.cfgs.keys():
try:
pfile = open(self.cfgs['threshold_file'], 'r')
thresholds = json.load(pfile)
pfile.close()
self.thresholds = {
c: thresholds[c] if c in thresholds.keys() else
self.cfgs['global_threshold']
for c in self.concepts
}
except Exception as err:
self.io.print_error(
'Error parsing threshold file {0},{1}'.format(
self.cfgs['threshold_file'], err))
def read_config(self):
pfile = open(self.tester_config_file)
self.cfgs = yaml.load(pfile)
pfile.close()
def setup(self):
self.tester = model_dict[self.tester_name]()
self.tester.configure(self.model_config_file)
self.tester.define()
#import ipdb; ipdb.set_trace()
self.model_type = self.tester.model.get_config()['name']
self.im_h, self.im_w = self.tester.cfgs[
'image_height'], self.tester.cfgs['image_width']
if not self.tester.init:
self.io.print_error('Error with model definition')
self.init = False
return
self.io.print_info('{0} Model defined from {1}'.format(
self.tester_name, self.model_config_file))
self.compile()
if not self.tester.init:
self.io.print_error('Error with model compilation')
self.init = False
return
self.io.print_info('{0} Model compiled'.format(self.tester_name))
self.init = True
self.load_synset_file()
self.load_mean_file()
self._read_threshold_file()
self.load_confidence_model()
self.read_label_embedding()
def compile(self):
"""
This compile version is to be used for testing only. Since its optimized for predictions
"""
self.output_nodes = self.cfgs['output_node_name']['test']
if self.model_type == 'Sequential':
self.inference = K.function(
[self.tester.model.get_input(train=False)],
[self.tester.model.layers[-1].get_output(train=False)])
elif self.model_type == 'Graph':
#import ipdb; ipdb.set_trace()
self.inference = K.function(
[self.tester.model.get_input(train=False)], [
self.tester.model.nodes[node].get_output(train=False)
for node in self.output_nodes
])
else:
self.io.print_error(
'Say whaaaat ?! This model type is not supported : {}'.format(
self.model_type))
sel.init = False
def read_label_embedding(self):
self.label_embedding = None
self.label_vectors = None
if 'label_embedding_file' in self.cfgs.keys():
self.label_embedding = self.io.read_npz_file(
self.cfgs['label_embedding_file'], 'matrix')
self.label_vectors = self.io.read_npz_file(
self.cfgs['label_embedding_file'], 'vectors')
self.io.print_info('Read label embedding file {0}'.format(
self.cfgs['label_embedding_file']))
def load_confidence_model(self):
self.confidence_model = None
if 'confidence_model' in self.cfgs.keys():
self.confidence_model = joblib.load(self.cfgs['confidence_model'])
#to be used in
def get_images_frVisualization(self):
self.load_synset_file()
GndTr = dict()
if not self.init:
return
indexes = dict()
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
#filterout images from getty
self.io.print_info(
'Removing getty scrapped images from validation data')
filter_getty = [
contains(img, 'getty-concept-sourcing') for img in self.test_images
]
ind = np.where(np.array(filter_getty) == True)[0]
test_images = np.array(self.test_images)[ind]
#---------------
for image in test_images:
im_basename = os.path.basename(image)
im_basename, img_ext = os.path.splitext(im_basename)
g = self.test_lmdb.get_labels(im_basename)
concepts = np.array(self.concepts)
index = np.where(g == 1)[0]
GndTr[im_basename] = concepts[index]
indexes[im_basename] = index
return (test_images, GndTr, indexes)
def visualize_image(self, beta, pixels):
self.beta = beta
self.load_synset_file()
flag, predictions, confidence, t = self.classify_images(pixels)
#file_paths = [os.path.join(self.cfgs['results_dir'], 'visualization_',str(self.beta)+'_'+im + save_ext) for im in basenames]
def run(self):
self.load_synset_file()
if not self.init:
return
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
image_batches = [
self.test_images[i:i + self.cfgs['batch_size']]
for i in range(0, len(self.test_images), self.cfgs['batch_size'])
]
self.io.print_info('{0} images split into {1} Batches of {2}'.format(
len(self.test_images), len(image_batches),
self.cfgs['batch_size']))
for beta in self.cfgs['betas']:
self.beta = beta
for idx, images in enumerate(image_batches):
if not self.cfgs['force']:
flag = self.check_jsons(images)
if flag:
self.io.print_warning(
'Skipping batch {0} of {1}'.format(
idx, len(image_batches)))
continue
pixels, basenames, dummy = self.fetch_images(images)
dummy = []
flag, predictions, confidence, t = self.classify_images(pixels)
if self.raw_predictions is False:
getty_like_filter = [
self.suppress_stop_list(g) for g in predictions
]
getty_like_safe = [
self.run_thresholds(g) for g in getty_like_filter
]
getty_like_pc = [
self.map_concepts(g) for g in getty_like_safe
]
getty_like_public = [
self.resolve_antonyms(g) for g in getty_like_pc
]
getty_like_unique = [
self.resolve_duplicates(g) for g in getty_like_public
]
predictions_pc = getty_like_unique
save_ext = '.json'
else:
predictions_pc = predictions
save_ext = '.npz'
if not flag:
continue
file_paths = [
os.path.join(self.cfgs['results_dir'],
str(self.beta) + '_' + im + save_ext)
for im in basenames
]
if not self.raw_predictions:
to_save = [dict({'external': p}) for p in predictions_pc]
else:
to_save = predictions_pc
#import ipdb; ipdb.set_trace()
#to_save = [dict({'external': p}) for p in predictions_pc]
#to_save=predictions_pc
#to_save_id = [dict({'external': id_}) for id_ in ids]
assert (len(file_paths) == len(to_save))
self.io.print_info('Done batch {0} of {1} in {2} secs'.format(
idx, len(image_batches), t))
#import ipdb; ipdb.set_trace()
#import ipdb; ipdb.set_trace()
f = map(self.io.save_good_labels, file_paths, to_save)
#f = map(self.io.save_good_labels, file_paths, to_save_id)
def check_jsons(self, images):
basenames = []
for image in images:
im_basename = os.path.basename(image)
im_basename, _ = os.path.splitext(im_basename)
basenames.append(im_basename)
flags = [
os.path.exists(os.path.join(self.cfgs['results_dir'], b + '.json'))
for b in basenames
]
return np.all(flags)
def compare_various_betas(self):
betas = self.cfgs['betas']
Area_PR = dict()
dir_path = os.path.dirname(os.path.realpath(__file__))
#print dir_path
self.limit_NoofClasses = None
self.load_synset_file()
self.load_synset_file_training()
self.imgHist_Concepts_inTraindata = [
(np.where(np.array(self.concepts) == i)[0][0])
for i in self.trained_concepts
]
#import lmdb
#lmdb_env = lmdb.open("./keywording_test_images_labels", readonly=True, lock=False)
# lmdb_txn = lmdb_env.begin()
# read_label = lmdb_txn.get
# label1test=self.test_lmdb.get_labels(read_label,'10085536')
# label2test=self.test_lmdb.get_labels(read_label,'10085536')
#self.imgname_labels=self.test_lmdb.create_keys_labels_dict()
var = dict()
for beta in betas:
var.update(self.eval_usinglmdb(str(beta)))
#import ipdb; ipdb.set_trace()
if self.limit_NoofClasses == None:
df2 = pd.DataFrame(var, index=self.trained_concepts)
else:
df2 = pd.DataFrame(
var, index=self.trained_concepts[0:self.limit_NoofClasses])
df2.to_csv(dir_path + '/results/' +
self.cfgs['results_dir'].split('/')[-2] + '.csv')
#import ipdb; ipdb.set_trace()
# mAP=[]
# mRec90=[]
# mRec80=[]
# max_f1=[]
# for beta in betas:
# vAP,v90,v80,vf1=Area_PR[beta]
# mAP.append(vAP)
# mRec90.append(v90)
# mRec80.append(v80)
# max_f1.append(vf1)
# #print ("mAP:%.4f,mean Recall:%.4f,beta:%s" % (mAP,mRec,str(beta)))
# #import ipdb; ipdb.set_trace()
# #print(*betas, sep='\t')
# print('\t'.join([str(x) for x in betas]))
# #print(*mRec80, sep='\t')
# print('\t'.join([str(round(x*100,3)) for x in mRec80]))
# #print(*mRec90, sep='\t')
# print('\t'.join([str(round(x*100,3)) for x in mRec90]))
# print('\t'.join([str(round(x*100,3)) for x in mAP]))
# print('\t'.join([str(round(x*100,3)) for x in max_f1]))
#print(*mAP, sep='\t')
#pickle.dump(Area_PR,open('/nas2/praveen/home/rnd-libs/rnd_libs/lib/keras/result_Area_PR','wb'))
# def generate_csv_file(self):
# betas=self.cfgs['betas']
# Area_PR=dict()
# #import lmdb
# #lmdb_env = lmdb.open("./keywording_test_images_labels", readonly=True, lock=False)
# #import ipdb; ipdb.set_trace()
# # lmdb_txn = lmdb_env.begin()
# # read_label = lmdb_txn.get
# # label1test=self.test_lmdb.get_labels(read_label,'10085536')
# # label2test=self.test_lmdb.get_labels(read_label,'10085536')
# #self.imgname_labels=self.test_lmdb.create_keys_labels_dict()
# for beta in betas:
# Area_PR[beta]=self.eval_usinglmdb(str(beta))
# mAP=[]
# mRec90=[]
# mRec80=[]
# max_f1=[]
# detections=[]
# for beta in betas:
# vAP,v90,v80,max_f1,detections=Area_PR[beta]
# mAP.append(vAP)
# mRec90.append(v90)
# mRec80.append(v80)
#print ("mAP:%.4f,mean Recall:%.4f,beta:%s" % (mAP,mRec,str(beta)))
#import ipdb; ipdb.set_trace()
#print(*betas, sep='\t')
#import ipdb; ipdb.set_trace()
# def compare_various_betas(self):
# import ipdb; ipdb.set_trace()
# betas=self.cfgs['betas']
# for beta in betas:
# self.eval_usinglmdb(str(beta)
def eval_usinglmdb(self, beta):
#import ipdb; ipdb.set_trace()
#import ipdb; ipdb.set_trace()
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
self.predictions = {}
self.gt = {}
valid_keys = []
notgoodGTimgs = []
ImgsSmallNumofTags = []
#import ipdb; ipdb.set_trace()
for idx, image in enumerate(self.test_images):
if idx % 1000 == 0:
self.io.print_info('Reading {0}/{1}'.format(
idx, len(self.test_images)))
im_basename = os.path.basename(image)
im_basename, img_ext = os.path.splitext(im_basename)
#result_json = os.path.join(self.cfgs['results_dir'], im_basename + '.json')
result_json = os.path.join(self.cfgs['results_dir'],
beta + '_' + im_basename + '.npz')
#gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json')
if os.path.exists(result_json):
#p, s,ids = self.io.read_prediction_json_withIDs(result_json)
p = np.load(result_json)['features']
else:
p = []
#g = self.io.read_vector(gt_json)
#import ipdb; ipdb.set_trace()
g = self.test_lmdb.get_labels(im_basename)
#print np.sum(g)
#print p.shape
if len(p) == 0:
print("images with no prediction generated: %s") % im_basename
continue
if len(g) == 0:
#import ipdb; ipdb.set_trace()
print("images with no GT data: %s") % im_basename
notgoodGTimgs.append(im_basename)
continue
if len(g) < 7735 and len(g) != 0:
print("images with small number of tags: %s") % im_basename
ImgsSmallNumofTags.append(im_basename)
continue
#g_filtered=self.parse_gndTruth_withFilter(ids,g)
# if p is [] or g is []:
# continue
self.predictions[im_basename] = p
self.gt[im_basename] = g
valid_keys.append(im_basename)
# if idx>1000:
# break
print 'Number of images: with no GT %d and with small no of tags %d' % (
len(notgoodGTimgs), len(ImgsSmallNumofTags))
#import ipdb; ipdb.set_trace()
y_test = np.zeros((len(valid_keys), len(self.concepts)), dtype='u1')
y_score = np.zeros((len(valid_keys), len(self.concepts)))
for i, key in enumerate(valid_keys):
y_test[i] = self.gt[key]
y_score[i] = self.predictions[key]
gt = dict()
pred = defaultdict(dict)
for i, concept in enumerate(self.concepts):
imgname_ind = np.where(y_test[i] == 1)
np_array_photo_ids = np.array(valid_keys)[imgname_ind]
gt[concept] = list(np_array_photo_ids)
#pred[concept]={key: value for key, value in self.predictions.items() if key in list(np_array_photo_ids)}
# for key in list(np_array_photo_ids):
# pred[concept].update({key: self.predictions[key]})
# pairof_photoid_predictions=self.predictions.items()
# for photo_id, predictions in pairof_photoid_predictions:
# for concept, score in zip(self.concepts,predictions):
# pred[concept][photo_id] = score
# #import ipdb; ipdb.set_trace()
# thresholds=get_thresholds(pred, self.concepts, gt)
# detections=get_detections_per_concept(pred, self.concepts, gt, thresholds)
#class_histogram=y_test.sum(axis=0)
#import ipdb; ipdb.set_trace()
if self.cfgs['percentate_of_classesFrEval'] != None or self.cfgs[
'percentate_of_classesFrEval'] < 1.0:
use_subsetofClsFrEval = True
n_classes = len(self.concepts)
limit_NoofClasses = int(self.cfgs['percentate_of_classesFrEval'] *
n_classes)
subset_indices_FrEval = self.imgHist_Concepts_inTraindata[:
limit_NoofClasses]
self.limit_NoofClasses = limit_NoofClasses
#self.concepts=np.array(self.concepts)[indices]
#y_test_subset=np.zeros((len(valid_keys),len(self.concepts)),dtype='u1')
#y_score_subset=np.zeros((len(valid_keys),len(self.concepts)))
#for i,key in enumerate(valid_keys):
# y_test_subset[i]=self.gt[key][indices]
# y_score_subset[i]=self.predictions[key][indices]
#print "Using %d out of %d rare number of classes for evaluations" % (limit_NoofClasses,n_classes)
else:
use_subsetofClsFrEval = False
subset_indices_FrEval = self.imgHist_Concepts_inTraindata
#subset_indices_FrEval=range(len(self.concepts))
#if use_subsetofClsFrEval:
# y_test=y_test_subset
# y_score=y_score_subset
#plt.plot(np.sort(temp)[::-1])
#plt.savefig('/nas2/praveen/home/rnd-libs/rnd_libs/lib/keras/intermediate/class_histogram')
#fpr = dict()
#tpr = dict()
recAt90 = []
recAt70 = []
ap = []
names = ['ap', 'recAt70', 'recAt90']
#max_f1=[]
for ind in subset_indices_FrEval:
#import ipdb; ipdb.set_trace()
# if y_test[:, i].sum()==0.0:
# continue
#fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
ap.append(
roundoff(
average_precision_score(y_test[:, ind], y_score[:, ind])))
recAt90.append(
roundoff(
recall_at_specified_precision(y_test[:, ind],
y_score[:, ind],
specified_precision=0.9)))
recAt70.append(
roundoff(
recall_at_specified_precision(y_test[:, ind],
y_score[:, ind],
specified_precision=0.7)))
#max_f1[self.concepts[ind]] = max_f1_score(y_test[:, ind], y_score[:, ind])
#import ipdb; ipdb.set_trace()
#return (np.array(ap.values()).mean(), np.array(recAt90.values()).mean(), np.array(recAt80.values()).mean())
#max_f1=[0.0]
#import ipdb; ipdb.set_trace()
returndict = dict()
for name in names:
returndict[beta + '_' + name] = locals()[name]
return (returndict)
#import ipdb; ipdb.set_trace()
# import ipdb; ipdb.set_trace()
# list_thres=[self.cfgs['global_threshold']]
# for t in list_thres:
# self.compute_precision_recall(t)
def eval(self):
self.load_synset_file()
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
self.predictions = {}
self.gt = {}
for idx, image in enumerate(self.test_images):
if idx % 1000 == 0:
self.io.print_info('Reading {0}/{1}'.format(
idx, len(self.test_images)))
im_basename = os.path.basename(image)
im_basename, _ = os.path.splitext(im_basename)
result_json = os.path.join(self.cfgs['results_dir'],
im_basename + '.json')
gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json')
p, s = self.io.read_prediction_json(result_json)
g = self.io.read_vector(gt_json)
if p is [] or g is []:
continue
self.predictions[im_basename] = zip(p, s)
self.gt[im_basename] = list(set(g).intersection(self.concepts))
for t in [self.cfgs['thresholds']]:
self.compute_precision_recall(t)
def compute_precision_recall(self, thresh):
fps = {}
tps = {}
fns = {}
gt_dict = {}
for c in self.concepts:
fps[c], tps[c], fns[c], gt_dict[c] = [], [], [], []
for key, value in self.predictions.items():
detections = [v[0] for v in value if v[1] >= thresh]
gt = self.gt[key]
for g in gt:
gt_dict[g].append(key)
tp = list(set(gt).intersection(detections))
fp = list(set(detections) - set(gt))
fn = list(set(gt) - set(detections))
for t in tp:
tps[t].append(key)
for f in fp:
fps[f].append(key)
for n in fn:
fns[n].append(key)
for c in self.concepts:
if gt_dict[c] == []:
self.io.print_info(
'Skipping {0}, no samples in ground-truth'.format(c))
continue
p = len(tps[c]) / (len(tps[c]) + len(fps[c]) + 0.00000001)
r = len(tps[c]) / (len(gt_dict[c]) + 0.0)
f1 = self.compute_f1(p, r)
self.io.print_info(
'At {0} : {1} precision {2}, recall {3}, f1-score {4}'.format(
c, thresh, p, r, f1))
def compute_f1(self, p, r):
return 2 * (p * r) / (p + r + 0.000001)
def load_synset_file(self):
pfile = open(self.cfgs['synset_file'], 'r')
concepts = pfile.readlines()
pfile.close()
self.concepts = [p.strip() for p in concepts]
def load_synset_file_training(self):
pfile = open(self.cfgs['synset_file_training'], 'r')
concepts = pfile.readlines()
pfile.close()
self.trained_concepts = [p.strip() for p in concepts]
def load_mean_file(self):
if not self.cfgs['mean_file'] == 'None':
pfile = open(self.cfgs['mean_file'], 'r')
m = np.load(pfile)
pfile.close()
else:
m = [[[128.0]], [[128.0]], [[128.0]]]
m = np.mean(np.mean(m, axis=1), axis=1)
self.mean_pixel_value = np.array([m[0], m[1], m[2]])
self.pixel_scaling = self.cfgs['pixel_scaling'][
'test'] if 'pixel_scaling' in self.cfgs.keys() else None
self.channel_swap = self.cfgs['channel_swap'][
'test'] if 'channel_swap' in self.cfgs.keys() else None
self.image_shuffle = self.cfgs['image_shuffle'][
'test'] if 'image_shuffle' in self.cfgs.keys() else None
def jsonify(self, results):
return [[r[0], str(r[1])] for r in results]
def map_to_synsets(self, results):
results = {r[0]: r[1] for r in results}
return [results[c] for c in self.concepts]
def parse_predictions(self, p):
sort_idx = np.argsort(-p)
concepts = [(self.concepts[idx], float(p[idx]), idx)
for idx in sort_idx[:self.cfgs['top_k']['test']]]
#import ipdb; ipdb.set_trace()
return concepts
def parse_gndTruth_withFilter(self, ids, p):
p = np.array(p)
concepts = np.array(self.concepts)
filtered_gt = p[ids]
filtered_concepts = concepts[ids]
gt_ids = np.where(filtered_gt == 1)[0]
#sort_idx = np.argsort(-p)
#import ipdb; ipdb.set_trace()
return filtered_concepts[gt_ids]
def parse_gndTruth(self, p):
#import ipdb; ipdb.set_trace()
gt = np.array(p)
concepts = np.array(self.concepts)
#filtered_gt=p[ids]
#filtered_concepts=concepts[ids]
gt_ids = np.where(gt == 1)[0]
#sort_idx = np.argsort(-p)
#import ipdb; ipdb.set_trace()
return concepts[gt_ids]
def classify_images(self, images):
confidence = [None]
if images == []:
return False, [(), ()], 0.0
try:
tstart = time.time()
(predictions, features) = self.inference(images)
#import ipdb; ipdb.set_trace()
predictions = f(predictions, self.beta)[0]
#import ipdb; ipdb.set_trace()
#pickle.dump(predictions,open('predictions','wb'))
#test=f(predictions,self.beta)[0]
#predictions=numpy_tensor_mean(predictions,self.beta,1)[0]
#import ipdb; ipdb.set_trace()
# if len(predictions.shape) == 4:
# predictions=predictions[:,:,0,0]
#import ipdb; ipdb.set_trace()
if self.label_vectors is not None:
predictions /= np.linalg.norm(predictions, axis=1)[:,
np.newaxis]
predictions = np.dot(predictions, self.label_vectors)
if self.confidence_model is not None:
confidence = [('Confidence', p)
for p in self.confidence_model.predict(features)]
tend = time.time()
except Exception as err:
self.io.print_error('Error processing image, {0}'.format(err))
return False, [(), ()], None, 0.0
# if not self.raw_predictions:
# results = [self.parse_predictions(p)[1] for p in predictions] , True, [self.parse_predictions(p)[0] for p in predictions], confidence, '{0:.4f}'.format((tend - tstart))
# else:
# results = True, predictions, confidence, '{0:.4f}'.format((tend - tstart))
#results = True, [self.parse_predictions(p) for p in predictions], confidence, '{0:.4f}'.format((tend - tstart))
results = True, predictions, confidence, '{0:.4f}'.format(
(tend - tstart))
return results
def predict(self, image):
image = self.normalize_images([image])
return self.classify_images(image)
def normalize_images(self, images):
normalize_images = []
for image in images:
image *= 255.0
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info(
'Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
try:
image = cv2.resize(image, (self.im_h, self.im_w))
except Exception as err:
self.io.print_error('Could not parse test image {0}'.format(e))
return False, [], 0.0
image -= self.mean_pixel_value
image = image.transpose(2, 0, 1)
# Channel swap since caffe.io.load_image returns RGB image and training was done with BGR
image = image[(2, 1, 0), :, :]
normalize_images.append(image)
return [np.array(normalize_images)]
def scale_image(self, image):
#
# caffe scales image 0-1 in pixel values we bring it back to 255 as keras likes it that way
#
# No more caffe no more scaling down to 0-1 in caffe
# image *= 255.0
tstart = time.time()
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info(
'Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
tend = time.time()
tstart = time.time()
try:
if self.resize_test:
#Tracer()()
#print image.size
image = cv2.resize(image,
(self.im_h, self.im_w)).astype(np.float32)
else:
print 'no resize'
image = image.astype(np.float32)
except Exception as e:
self.io.print_error('Could not parse {0}'.format(e))
return None
tend = time.time()
tstart = time.time()
if self.mean_pixel_value is not None:
image -= self.mean_pixel_value
if self.pixel_scaling is not None:
image /= self.pixel_scaling
if self.channel_swap is not None:
image = image[:, :, self.channel_swap]
# H,W,C to C,H,W
if self.image_shuffle is not None:
image = image.transpose(self.image_shuffle)
tend = time.time()
return image
def fetch_images(self, image_file_names):
images = []
basenames = []
tstart = time.time()
for idx, image_file_name in enumerate(image_file_names):
image_file_name = image_file_name.replace('/nas/', '/nas2/')
im_basename = os.path.basename(image_file_name)
im_basename, _ = os.path.splitext(im_basename)
if not urlparse.urlparse(image_file_name).scheme == "":
url_response = urllib.urlopen(image_file_name)
if url_response.code == 404:
print self.io.print_error(
'[Testing] URL error code : {1} for {0}'.format(
image_file_name, url_response.code))
continue
try:
string_buffer = StringIO.StringIO(url_response.read())
image = np.asarray(bytearray(string_buffer.read()),
dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
image = image[:, :, (2, 1, 0)]
image = image.astype(np.float32)
except Exception as err:
print self.io.print_error(
'[Testing] Error with URL {0} {1}'.format(
image_file_name, err))
continue
elif os.path.exists(image_file_name):
try:
fid = open(image_file_name, 'r')
stream = fid.read()
fid.close()
image = Image.open(cStringIO.StringIO(stream))
image = np.array(image)
image = image.astype(np.float32)
# image = cv2.imread(image_file_name)
# image = image[:, :, (2, 1, 0)]
# image = image.astype(np.float32)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error(
'[Testing] Error with image file {0} {1}'.format(
image_file_name, err))
continue
else:
try:
image = self.conn.read_image(image_file_name)
if image is None:
raise self.io.print_error(
"[Testing] Image data could not be loaded - %s" %
image_file_name)
continue
image = np.array(image)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error(
'[Testing] Error with S3 Bucket hash {0} {1}'.format(
image_file_name, err))
continue
# try : do or do not there is not try
#orig_image=image.astype('u1')
image = self.scale_image(image)
if image is None:
continue
if self.verbose:
self.io.print_info('Processing {0} {1}'.format(
image_file_name, image.shape))
images.append(image)
basenames.append(im_basename)
tend = time.time()
self.io.print_info('Fetching {0} images took {1:.4f} secs'.format(
len(image_file_names), tend - tstart))
#return ([np.array(images)], basenames,orig_images)
return ([np.array(images)], basenames)
def fetch_gif_frames(self, inGif):
# Only works for URLs for now
# write frames to disk and pass filenames to fetch_images
outFolder = '/tmp'
frame = Image.open(BytesIO(urllib.urlopen(inGif).read()))
nframes = 0
file_names = []
while frame:
frame_name = '{0}/{1}-{2}.png'.format(outFolder,
os.path.basename(inGif),
nframes)
frame.save(frame_name, 'png')
file_names.append(frame_name)
nframes += 1
try:
frame.seek(nframes)
except EOFError:
break
file_names = [file_names[i] for i in range(0, len(file_names), 10)]
return file_names
def non_max(self, predictions):
concepts = {}
for pred in predictions:
for p in pred:
if p[0] in concepts.keys():
concepts[p[0]].append(float(p[1]))
else:
concepts[p[0]] = [float(p[1])]
for key, value in concepts.items():
concepts[key] = np.max(np.array(value))
labels = concepts.keys()
scores = [concepts[l] for l in labels]
sort_idx = np.argsort(-np.array(scores))
results = [(labels[idx], str(scores[idx])) for idx in sort_idx]
return [results]
def square_it_up(self, image):
edge_size = np.min(image.shape[:2])
cy, cx = np.array(image.shape[:2]) // 2
r_img = image[np.max([0, cy - edge_size //
2]):np.min([cy +
edge_size // 2, image.shape[0]]),
np.max([0, cx - edge_size //
2]):np.min([cx +
edge_size // 2, image.shape[1]]), :]
return r_img
def resolve_duplicates(self, concepts):
seen = set()
unique = []
for c in concepts:
if c[0] in seen:
logger.debug("Suppressing duplicate {}:{}".format(c[0], c[1]))
continue
seen.add(c[0])
unique.append(c)
return unique
def filter_concepts(self, predictions):
return [p[:self.cfgs['store_top_k']] for p in predictions]
def suppress_stop_list(self, predictions):
return [c for c in predictions if c[0] not in self.cfgs['stop_list_']]
def run_thresholds(self, predictions):
return self.threshold_concepts(predictions)
def resolve_antonyms(self, human_predictions):
conflicting_predictions = []
human_dict = {d[0]: d[1] for d in human_predictions}
human_labels = [d[0] for d in human_predictions]
human_scores = [d[1] for d in human_predictions]
for c1, c2 in self.cfgs['antonym_list']:
if not (c1 in human_labels and c2 in human_labels):
continue
# if
s1, s2 = human_dict[c1], human_dict[c2]
idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1)
logger.debug('Suppressing {0}:{1}'.format(human_labels[idx],
human_scores[idx]))
del human_labels[idx]
del human_scores[idx]
# for
remove_flag = -1
for idx, group in enumerate(self.cfgs['count_order_list']):
_this = np.intersect1d(human_dict.keys(), group)
if len(_this) > 0:
remove_flag = idx + 1
break
# if
# for
if not remove_flag == len(self.cfgs['count_order_list']):
remove_tags = []
for g in self.cfgs['count_order_list'][remove_flag:]:
remove_tags.extend(g)
# for
for t in remove_tags:
if t not in human_labels:
continue
# if
ridx = human_labels.index(t)
del human_labels[ridx]
del human_scores[ridx]
# for
# if
return [(g, s) for g, s in zip(human_labels, human_scores)]
# def
def map_concepts(self, concepts):
mapped_predictions = [(self.mapping[c[0]],
c[1]) if c[0] in self.mapping.keys() else c
for c in concepts]
mapped_predictions = self.conditional_mapping(mapped_predictions)
return mapped_predictions
# def
def conditional_mapping(self, predictions):
if 'conditional_list' in self.cfgs.keys():
for condition, mapping in self.cfgs['conditional_list']:
concepts = [p[0] for p in predictions]
holds = list(set(concepts).intersection(set(condition)))
if not len(holds) == 0:
predictions = [
p for p in predictions if not p[0] in mapping
]
# if
# for
# if
return predictions
# def
def threshold_concepts(self, predictions):
if self.thresholds is not None:
predictions = [
p for p in predictions if p[1] >= self.thresholds[p[0]]
]
# if
return predictions
class LabelEmbedding(object):
"""
The main/useful class to implement some of the ideas in : https://hal.inria.fr/hal-00815747v1/document
Given reasonable amout of data with dense predictions and getty tags we learn a mapping that can align them.
arg_max( dense_predictions*W*getty_labels^T )
"""
def __init__(self, config_file, force=False, reduced=False):
self.config_file = config_file
self.init = True
self.force = force
self._read_configuration()
self.init = self._verify_files()
self.is_mat = True
if not reduced:
self.io = EmbeddingIO(self.cfgs['path_logging_dir'])
else:
self.io = EmbeddingIO(None, True, True)
print 'Reduced functionality of Label Embedding'
#if
self.working_order = [(0, 'run_preprocess', self._run_preprocess),
(1, 'run_train', self._run_training),
(2, 'build_embedding', self._build_embedding),
(3, 'run_thresholds', self._run_thresholds),
(4, 'run_test', self._run_testing)]
self.train_method = {
'CFM': self._solve_closed_form,
'CFM_PC': self._solve_closed_form_pc,
'SGD': self._solve_sgd,
'WSABIE': self._solve_wsabie,
'CCA': self._solve_cca
}
lemb_mapping_keys = [d[0] for d in self.cfgs['lemb_mapping_list']]
lemb_mapping_values = [d[1] for d in self.cfgs['lemb_mapping_list']]
self.lemb_mapping = dict(zip(lemb_mapping_keys, lemb_mapping_values))
#def
"""
Check if all files and path exists.
From yaml config file there are variables beginning with path_
"""
def prepare_for_testing(self):
self._read_all_labels()
if not '_PC' in self.cfgs['train_method']:
self.W = self.io.read_weight_matrix(
self.cfgs['npz_files_loc']['weights'])
self.is_mat = True
self.threshold_map, self.black_list = self.io.read_threshold_file(
self.cfgs['npz_files_loc']['thresholds'])
self.threshold_map = self.threshold_map.tolist()
self.io.print_info(
'{0}/{1} concepts are black-listed (not enough validation data)'
.format(len(self.black_list), len(self.getty_labels)))
else:
self.W = self.io.read_all_emb(
self.getty_labels, self.cfgs['train_method'],
self.cfgs['npz_files_loc']['weights'])
self.is_mat = False
if self.cfgs['use_global_threshold']:
self.threshold_map, self.black_list = {
c: 0.5
for c in self.W.keys()
}, []
else:
self.threshold_map, self.black_list = self.io.read_threshold_file(
self.cfgs['npz_files_loc']['thresholds'])
#if
self.not_black_list = self.io.read_getty_labels(
self.cfgs['path_getty_label_file'],
self.cfgs['minimum_samples_for_validation'],
self.cfgs['getty_key_words_types'])
#if
if self.is_mat:
assert self.W.shape[1] == len(self.getty_labels)
self.io.print_info('Found weight {0}x{1} matrix'.format(
self.W.shape[0], self.W.shape[1]))
#if
def init_embedding(self, hedger_file, synset_names, getty_file,
emb_weights_file, emb_threshold_file):
self.all_wn_labels = self.io.read_hedged_labels(hedger_file)
self.leaf_wn_labels = synset_names
self.getty_labels = self.io.read_getty_labels(
getty_file, self.cfgs['getty_stop_list'],
self.cfgs['getty_key_words_types'])
print('Reading label embedding from {0}'.format(emb_weights_file))
if not '_PC' in self.cfgs['train_method']:
self.W = self.io.read_weight_matrix(emb_weights_file)
self.is_mat = True
self.threshold_map, self.black_list = self.io.read_threshold_file(
emb_threshold_file)
self.threshold_map = self.threshold_map.tolist()
self.io.print_info(
'{0}/{1} concepts are black-listed (not enough validation data)'
.format(len(self.black_list), len(self.getty_labels)))
else:
self.W = self.io.read_all_emb(self.getty_labels,
self.cfgs['train_method'],
emb_weights_file)
self.is_mat = False
if self.cfgs['use_global_threshold']:
self.threshold_map, self.black_list = {
c: 0.5
for c in self.W.keys()
}, []
else:
self.threshold_map, self.black_list = self.io.read_threshold_file(
self.cfgs['npz_files_loc']['thresholds'])
#if
self.not_black_list = self.io.read_getty_labels(
getty_file, self.cfgs['minimum_samples_for_validation'],
self.cfgs['getty_key_words_types'])
#if
if self.is_mat:
assert self.W.shape[1] == len(self.getty_labels)
self.io.print_info('Found weight {0}x{1} matrix'.format(
self.W.shape[0], self.W.shape[1]))
#if
self.io.print_info('Testing set-up for label embedding')
#if
def _verify_files(self):
flag = True
paths_to_check = [(k, v) for k, v in self.cfgs.items() if 'path_' in k]
for var, file_path in paths_to_check:
if type(file_path) == dict:
check_files = [(var + '+' + r, j)
for r, j in file_path.items()]
else:
check_files = [(var, file_path)]
#if
file_flag = [os.path.exists(f[1]) for f in check_files]
flag = flag and np.all(file_flag)
for idx, f in enumerate(file_flag):
if not f:
print colored(
'[WARN] Could not find {0} for {1}, no such file or directory'
.format(check_files[idx][1],
check_files[idx][0]), 'blue')
#if
#if
#def
return flag
#def
"""
Simple yaml config file reader
"""
def _read_configuration(self):
pfile = open(self.config_file)
self.cfgs = yaml.load(pfile)
pfile.close()
#def
"""
Read wordnet synset file and getty labels.
"""
def _read_all_labels(self):
self.leaf_wn_labels, self.all_wn_labels = self.io.read_wn_labels(
self.cfgs['path_bet_file'], self.cfgs['path_wn_labels_file'])
self.getty_labels = self.io.read_getty_labels(
self.cfgs['path_getty_label_file'], self.cfgs['getty_stop_list'],
self.cfgs['getty_key_words_types'])
#def
def get_to_work(self, tasks):
for idx, action, action_func in self.working_order:
if tasks[action]:
action_func()
#if
#for
#def
"""
Read all possible labels wordnet and getty
Run converting dense prediction vectors from DL into histogram features.
Run converting getty labels into binary vectors (presense or absence)
Save processed vectors.
"""
def _run_preprocess(self):
self._read_all_labels()
self._read_training_samples()
self._process_vectors()
#def
"""
Second step after pre-processing vectors, depending on training method selected used closed form solution or use SGD
"""
def _run_training(self):
if self.cfgs['train_method'] in self.train_method.keys():
self.train_method[self.cfgs['train_method']]()
else:
self.io.print_error(
'Could not find the specified training method {0}'.format(
self.cfgs['train_method']))
#if
#def
def _build_embedding(self):
self._read_all_labels()
W = self.io.read_all_emb(self.getty_labels, self.cfgs['train_method'],
self.cfgs['path_weight_matrix'])
self.io.write_all_emb(self.cfgs['npz_files_loc']['weights'], W)
self.io.print_info('Done writing {1} embeddings to {0}'.format(
self.cfgs['npz_files_loc']['weights'], len(W.keys())))
#def
"""
Run testing woudl included and instance of FullConvNet and dense predictions.
Mapping predictions using learned weight matrix.
Save Mapped predictions.
TODO : Check code to reflect changes for individual embeddings
"""
def _run_thresholds(self):
self.prepare_for_testing()
self.validation_samples = self.io.read_images(
self.cfgs['path_file_list']['val'])
D_val, G_val, _ = self._read_all_data(self.validation_samples, 'val')
G_val_pred = [
v.predict_proba(D_val)[:,
np.where(v.named_steps['train'].classes_ ==
1)[0][0]]
if type(v) == sklearn.pipeline.Pipeline else
v.predict_proba(D_val)[:, np.where(v.classes_ == 1)[0][0]]
for k, v in self.W.items()
]
G_val_pred = np.array(G_val_pred).transpose()
sample_range = np.array(range(1, D_val.shape[0] + 1), dtype=float)
concepts_threshold = {}
non_trained_concepts = []
for l in range(D_val.shape[1]):
gt_l = np.where(G_val[:, l] > 0)[0].tolist()
if len(gt_l) < self.cfgs['minimum_samples_for_validation']:
self.io.print_warning(
'Skipping {0}, not enough samples {1}'.format(
self.getty_labels[l], len(gt_l)))
non_trained_concepts.append(self.getty_labels[l])
continue
#if
sorted_idx = np.argsort(-G_val_pred[:, l])
good_idx = [1 if idx in gt_l else 0 for idx in sorted_idx.tolist()]
c_sum = np.cumsum(np.array(good_idx)) / sample_range
here = np.where(c_sum > self.cfgs['concept_precision'])[0].tolist()
fhere = here[-1] if not here == [] else np.argmax(c_sum)
concepts_threshold[self.getty_labels[l]] = G_val_pred[
sorted_idx[fhere], l]
self.io.print_info('Concept {0} has threshold {1}'.format(
self.getty_labels[l],
concepts_threshold[self.getty_labels[l]]))
#for
self.io.save_threshold_file(self.cfgs['npz_files_loc']['thresholds'],
concepts_threshold, non_trained_concepts)
#def
def predict(self, dense_pred):
vec_dense = []
for pred in dense_pred:
vec_dense.extend(pred)
#for
predictions = self.dense_to_labels(vec_dense)
predictions = [(p) for p in predictions]
return predictions
#def
def resolve_label_embedding(self, emb_pred):
filtered_pred = [
d[0] for d in emb_pred if d[0] not in self.cfgs['stop_list_lemb']
]
mapped_pred = self.map_lemb_concepts(filtered_pred)
return mapped_pred
#def
def map_lemb_concepts(self, concepts):
return [
self.lemb_mapping[c] if c in self.lemb_mapping.keys() else c
for c in concepts
]
#def
def resolve_antonyms(self, human_predictions):
conflicting_predictions = []
human_dict = {d[0]: d[1] for d in human_predictions}
human_labels = [d[0] for d in human_predictions]
human_scores = [d[1] for d in human_predictions]
for c1, c2 in self.cfgs['antonym_list']:
if not (c1 in human_labels and c2 in human_labels):
continue
#if
s1, s2 = human_dict[c1], human_dict[c2]
idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1)
print 'Supressing {0}:{1}'.format(human_labels[idx],
human_scores[idx])
del human_labels[idx]
del human_scores[idx]
#for
remove_flag = -1
for idx, group in enumerate(self.cfgs['count_order_list']):
_this = np.intersect1d(human_dict.keys(), group)
if len(_this) > 0:
remove_flag = idx + 1
break
#if
#for
if not remove_flag == len(self.cfgs['count_order_list']):
remove_tags = []
for g in self.cfgs['count_order_list'][remove_flag:]:
remove_tags.extend(g)
#for
for t in remove_tags:
if not t in human_labels:
continue
#if
ridx = human_labels.index(t)
del human_labels[ridx]
del human_scores[ridx]
#for
#if
return [(g, s) for g, s in zip(human_labels, human_scores)]
#def
def softmax(self, w, t=1.0):
e = np.exp(w / t)
dist = e / np.sum(e)
return dist
#def
def dense_to_labels(self, vec_dense):
if vec_dense == []:
return []
#def
f_vec_dense = self._build_wn_label_vectors(vec_dense)
if self.is_mat:
getty_like_vec = np.dot(f_vec_dense, self.W)
good = np.argsort(-getty_like_vec)
good_labels = [
(self.getty_labels[g], getty_like_vec[g]) for g in good
if (not self.getty_labels[g] in self.black_list
and self.getty_labels[g] in self.threshold_map.keys())
]
good_labels = [(g, s) for g, s in good_labels
if s >= 0][:self.cfgs['top_k_tags']]
else:
getty_like_vec = [(k, v.predict_proba(f_vec_dense)[0][np.where(
v.named_steps['train'].classes_ == 1)[0][0]])
if type(v) == sklearn.pipeline.Pipeline else
(k, v.predict_proba(f_vec_dense)[0][np.where(
v.classes_ == 1)[0][0]])
for k, v in self.W.items()]
good = np.argsort(-np.array([d[1] for d in getty_like_vec]))
good_labels = [
getty_like_vec[g] for g in good
if getty_like_vec[g][0] in self.not_black_list
]
good_labels = [
(g, s) for g, s in good_labels if s > self.threshold_map[g]
][:self.cfgs['top_k_tags']]
#if
return good_labels
#for
def _run_testing(self):
if not os.path.exists(self.cfgs['npz_files_loc']['weights']):
self.io.print_error(
'Could not find label emdbedding matrix file {}'.format(
self.cfgs['npz_files_loc']['weights']))
return
#if
self.prepare_for_testing()
self.testing_samples = self.io.read_images(
self.cfgs['path_file_list']['test'])
self.io.print_info('Will process {0} test images'.format(
len(self.testing_samples)))
shuffle(self.testing_samples)
#self.testing_samples = self.testing_samples[110:120]
for idx, sample in enumerate(self.testing_samples):
if idx % 1000 == 0:
self.io.print_info('Processing {0}/{1} image'.format(
idx, len(self.testing_samples)))
#if
basename = os.path.basename(sample)
basename, ext = os.path.splitext(basename)
json_file = os.path.join(self.cfgs['path_results_dir']['test'],
basename + '.json')
if os.path.exists(json_file):
if not self.force:
self.io.print_warning(
'Skipping {0}, already exists'.format(json_file))
continue
#if
#if
vec_dense = self.io.read_vector(
os.path.join(self.cfgs['path_dense_pred_loc']['test'],
basename) + self.cfgs['wn_suffix'])
vec_getty = self.io.read_vector(
os.path.join(self.cfgs['path_getty_labels_loc']['test'],
basename) + self.cfgs['getty_suffix'])
good_labels = self.dense_to_labels(vec_dense)
self.io.save_good_labels(json_file, dict(good_labels))
if self.cfgs['show_predicted_labels']:
self.io.print_info('Photo-id {0}'.format(basename))
self.io.print_info('Predicted {0}'.format(good_labels))
self.io.print_warning('GT {0}'.format(vec_getty))
#if
#for
#def
"""
Read images pointed to by training and validation file lists
"""
def _read_training_samples(self):
self.training_data = {}
self.validation_data = {}
self.training_data['samples'] = self.io.read_images(
self.cfgs['path_file_list']['train'])
self.validation_data['samples'] = self.io.read_images(
self.cfgs['path_file_list']['val'])
self.io.print_info(
'Found {0} training samples {1} validation samples'.format(
len(self.training_data['samples']),
len(self.validation_data['samples'])))
#def
"""
Convert dense predictions into vectors
Convert getty labels into binary vectors
"""
def _process_vectors(self):
self._label_to_vectors(self.training_data['samples'], 'train')
self._label_to_vectors(self.validation_data['samples'], 'val')
#def
def _label_to_vectors(self, training_images, tag='train'):
for idx, sample in enumerate(training_images):
if idx % 1000 == 0:
self.io.print_info(
'Processing {0}/{1} vectors for {2}, {3}'.format(
idx, len(training_images), tag, sample))
#if
basename = os.path.basename(sample)
basename, ext = os.path.splitext(basename)
npz_file = os.path.join(self.cfgs['path_processed_vectors'][tag],
basename + '.npz')
if (os.path.exists(npz_file) and not self.force):
self.io.print_info(
'Skipping {0}, already exists'.format(npz_file))
continue
#if
vec_dense = self.io.read_vector(
os.path.join(self.cfgs['path_dense_pred_loc']['train'],
basename) + self.cfgs['wn_suffix'])
vec_getty = self.io.read_vector(
os.path.join(self.cfgs['path_getty_labels_loc']['train'],
basename) + self.cfgs['getty_suffix'])
#f_vec_dense = vec_dense
f_vec_dense = self._build_wn_label_vectors(vec_dense)
f_vec_getty = self._build_getty_label_vectors(vec_getty)
self.io.save_npz_file(f_vec_dense, f_vec_getty, npz_file)
#for
self.io.print_info('Done building {0} vectors'.format(tag))
#def
"""
Convert wn dense predictions to sparse vectors and L2 normalization.
"""
def _build_wn_label_vectors(self, svec):
if svec == []:
return []
#if
vec = np.zeros(len(self.leaf_wn_labels), dtype=float)
idx = np.array([d[0] for d in svec])
values = np.array([d[1] for d in svec])
for d in svec:
vec[d[0]] += d[1]
#if
if not self.cfgs['normalization']['wn_norm'] == 'None':
vec /= (np.linalg.norm(vec) + 0.000001)
#if
return vec
#def
"""
Convert getty label vectors to binary vectors of presense of absence and L2 normalization.
"""
def _build_getty_label_vectors(self, svec):
if svec == []:
return []
#if
vec = np.zeros(len(self.getty_labels), dtype=float)
idx = np.array([
self.getty_labels.index(d) for d in svec if d in self.getty_labels
])
if idx.size == 0:
return []
#if
vec[idx] = 1.0
if not self.cfgs['normalization']['getty_norm'] == 'None':
vec /= (np.linalg.norm(vec) + 0.000001)
#if
return vec
#def
def _init_embedding_matrix(self):
self.W = np.zeros((len(self.leaf_wn_labels), len(self.getty_labels)))
self.io.print_info('Weight matrix set-up {0}x{1}'.format(
self.W.shape[0], self.W.shape[1]))
#def
def _init_identity_matrix(self):
self.I = np.eye(len(self.leaf_wn_labels), len(self.leaf_wn_labels))
#def
def _read_all_data(self, image_list, tag):
D_all_vec = []
G_all_vec = []
all_files = []
for idx, sample in enumerate(image_list):
if idx % 1000 == 0:
self.io.print_info(
'Reading {0}/{1} vectors for {3}, {2}'.format(
idx, len(image_list), sample, tag))
#if
basename = os.path.basename(sample)
basename, ext = os.path.splitext(basename)
npz_file = os.path.join(self.cfgs['path_processed_vectors'][tag],
basename + '.npz')
if not os.path.exists(npz_file):
self.io.print_warning(
'Skipping {0} no such file'.format(npz_file))
continue
#if
D, G = self.io.read_npz_file(npz_file)
D = D.flatten()
if (D.size == 0 or G.size == 0
or not len(D) == self.cfgs['expected_dim']):
self.io.print_warning(
'Skipping {0} mis-match of dimensionality'.format(
npz_file))
continue
#if
D_all_vec.append(D)
G_all_vec.append(G)
all_files.append(basename)
#for
return np.array(D_all_vec), np.array(G_all_vec), all_files
#def
"""
Different solutions to label embedding Closed form solution (CFM) / SGD / WSABIE / CCA
"""
def _solve_closed_form(self):
self._read_all_labels()
self._read_training_samples()
#self._init_embedding_matrix()
self._init_identity_matrix()
D_train, G_train, _ = self._read_all_data(
self.training_data['samples'], 'train')
if self.cfgs['balance_data']:
D_train, G_train = self._balance_training_data(D_train, G_train)
#if
D_val, G_val, _ = self._read_all_data(self.validation_data['samples'],
'val')
train_size = (D_train.nbytes + G_train.nbytes) / (1000.0 * 1024**2)
val_size = (D_val.nbytes + G_val.nbytes) / (1000.0 * 1024**2)
# TODO : compute expected size of data in memory before reading it LOL
# TODO : Future harsimrat will fix this
# Yo future Harsimrat here, this is obsolte now.
self.io.print_info('Train : {0:2f} GB, val : {1:2f} GB'.format(
train_size, val_size))
for l in self.cfgs['lambda']:
self.io.print_info(
'Computing closed form solution (CFM) for lambda = {0}'.format(
l))
self.W = np.dot(
np.linalg.inv(
np.dot(D_train.transpose(), D_train) + l * self.I),
np.dot(D_train.transpose(), G_train))
error = 1 - self._compute_validation_error_on_tags(
self.W, D_val, G_val)
self.io.print_info('(CFM) error for lambda = {0} : {1}'.format(
l, error))
self.io.save_weight_matrix(
self.W,
os.path.join(self.cfgs['path_weight_matrix'],
'CFM-{0}.npz'.format(l)))
#for
#def
"""
Closed form solution per concepts
"""
def _solve_closed_form_pc(self):
self._read_all_labels()
self._read_training_samples()
D_train, G_train, train_files = self._read_all_data(
self.training_data['samples'], 'train')
D_val, G_val, val_files = self._read_all_data(
self.validation_data['samples'], 'val')
train_size = (D_train.nbytes + G_train.nbytes) / (1000.0 * 1024**2)
val_size = (D_val.nbytes + G_val.nbytes) / (1000.0 * 1024**2)
self.io.print_info('Train : {0:2f} GB, val : {1:2f} GB'.format(
train_size, val_size))
self.reset_pipe_line()
for label, label_name in len(self.getty_labels):
svm_file_path = os.path.join(self.cfgs['path_weight_matrix'],
'CFM_PC-{0}.pkl'.format(label_name))
if os.path.exists(svm_file_path):
if not self.force:
self.io.print_warning(
'Skipping {}, already exists'.format(svm_file_path))
continue
#if
#if
tidx = np.sum(G_train[:, label])
vidx = np.sum(G_val[:, label])
if tidx == 0 or vidx == 0:
self.io.print_warning(
'Not enough samples for {0}, train:{1} or val :{2}'.format(
label_name, int(tidx), int(vidx)))
continue
#if
self.io.print_info(
'{1}/{2} Learning embedding for {0} with {3} +samples {4} -ive samples'
.format(label_name, label, len(self.getty_labels),
np.sum(G_train[:, label]),
G_train.shape[0] - np.sum(G_train[:, label])))
self.pipe_line.fit(D_train, G_train[:, label])
error = 1 - self.pipe_line.score(D_val, G_val[:, label])
self.io.print_info('(CFM_PC) error for {1} : {0}'.format(
error, label_name))
self.io.save_emb_file(self.pipe_line, svm_file_path)
#for
#def
def reset_pipe_line(self):
action_list = [
('norm', normalizer_dict[self.cfgs['normalization_type']]),
('scale', scaling_dict[self.cfgs['scaling_type']]),
('train', classifier_dict[self.cfgs['classifier_type']])
]
action_list = [(a, b) for a, b in action_list if not b == None]
self.pipe_line = Pipeline(action_list)
#def
def _solve_sgd(self):
self.io.print_error('SGD not implemented')
#def
def _solve_wsabie(self):
self.io.print_error('WSABIE not implemented')
#fef
def _solve_cca(self):
self.io.print_error('CCA not implemented')
#def
"""
Validation needs to change since we optimize for tag coverage we should compute intersecton over union of predicted tags + getty tags on validation set
"""
def _compute_validation_error(self, W, D, G):
n_samples = G.shape[0]
pG = np.dot(D, W)
for i in range(pG.shape[0]):
pG[i, :] /= (np.linalg.norm(pG[i, :]) + 0.000001)
#for
cG = np.dot(pG, G.transpose())
predictions = np.argmax(cG, axis=0)
gt_match = np.array(range(n_samples))
good = np.bitwise_xor(predictions, gt_match)
correct_match = good == 0
return np.sum(correct_match) / (n_samples + 0.0)
#def
"""
This function needs to change towards what we should optimize currently global predictions of tag is optimized.
"""
def _compute_validation_error_on_tags(self, W, D, G):
pG = np.dot(D, W)
predictions = np.zeros(pG.shape)
for i in range(pG.shape[0]):
pidx = np.argsort(-pG[i, :])[:self.cfgs['top_k_tags']]
predictions[i, pidx] = 1
#for
gt = np.zeros(G.shape)
idx = np.where(G > 0)
gt[idx] = 1
good = predictions * gt
n_samples = np.sum(predictions)
#self.io.print_info('{0} correct out of {1}'.format(np.sum(good),n_samples))
return np.sum(good) / (n_samples + 0.0)
#def
def _compute_validation_error_on_tags_pc(self, W, D, G):
pG = np.dot(D, W)
predictions = np.zeros(pG.shape)
pidx = np.where(pG > 0)[0]
predictions[pidx] = 1
gt = G
good = predictions * gt
n_samples = np.sum(predictions)
# error : (fp + tn)/n_samples
return (np.sum(predictions) + np.sum(gt) -
2 * np.sum(good)) / (n_samples + 0.0)
#def
def _restore_snapshot(self, label_idx):
if label_idx == 0:
self._init_embedding_matrix()
return
#if
label = labe_idx - 1
weight_matrix_path = os.path.join(
self.cfgs['path_weight_matrix'],
'CFM-{1}.npz'.format(l, self.getty_labels[label]))
self.W = self.io.read_weight_matrix(weight_matrix_path)
#def
def _balance_training_data(self, DL_train, G_train):
return DL_train_balance, G_train_balance
#def
#class
class TheanoTesterCaptions():
def __init__(self, config_file, verbose=False, raw_predictions=False):
if not os.path.exists(config_file):
print 'Error : could not find config file'.format(config_file)
self.init = False
return
self.tester_config_file = config_file
self.verbose = verbose
self.raw_predictions = raw_predictions
self.read_config()
self.io = EmbeddingIO(self.cfgs['log_dir'], 'testing')
self.tester_name = self.cfgs['model_name']
self.model_config_file = self.cfgs['model_config_file']
self.init = False
self.thresholds = None
mapping_keys = [d[0] for d in self.cfgs['mapping_list']]
mapping_values = [d[1] for d in self.cfgs['mapping_list']]
self.mapping = dict(zip(mapping_keys, mapping_values))
if self.tester_name not in model_dict.keys():
self.io.print_error(
'Not a valid model type {0} chose among {1}'.format(
self.tester_name, model_dict.keys()))
self.init = False
return
if not os.path.exists(self.tester_config_file):
self.io.print_error('Could not find configuration file {0}'.format(
self.tester_config_file))
self.init = False
return
self.tester = None
self.action_items = {'test': self.run, 'eval': self.eval}
def _read_threshold_file(self):
if 'threshold_file' in self.cfgs.keys():
try:
pfile = open(self.cfgs['threshold_file'], 'r')
thresholds = json.load(pfile)
pfile.close()
self.thresholds = {
c: thresholds[c] if c in thresholds.keys() else
self.cfgs['global_threshold']
for c in self.concepts
}
except Exception as err:
self.io.print_error(
'Error parsing threshold file {0},{1}'.format(
self.cfgs['threshold_file'], err))
def read_config(self):
pfile = open(self.tester_config_file)
self.cfgs = yaml.load(pfile)
pfile.close()
def setup(self):
self.tester = model_dict[self.tester_name]()
self.tester.configure(self.model_config_file)
self.tester.define()
self.model_type = self.tester.model.get_config()['name']
self.im_h, self.im_w = self.tester.cfgs[
'image_height'], self.tester.cfgs['image_width']
if not self.tester.init:
self.io.print_error('Error with model definition')
self.init = False
return
self.io.print_info('{0} Model defined from {1}'.format(
self.tester_name, self.model_config_file))
self.compile()
if not self.tester.init:
self.io.print_error('Error with model compilation')
self.init = False
return
self.io.print_info('{0} Model compiled'.format(self.tester_name))
self.init = True
self.load_synset_file()
self.load_mean_file()
self._read_threshold_file()
self.load_confidence_model()
self.read_label_embedding()
def compile(self):
"""
This compile version is to be used for testing only. Since its optimized for predictions
"""
self.output_nodes = self.cfgs['output_node_name']['test']
if self.model_type == 'Sequential':
self.inference = K.function(
[self.tester.model.get_input(train=False)],
[self.tester.model.layers[-1].get_output(train=False)])
elif self.model_type == 'Graph':
self.inference = K.function(
[self.tester.model.get_input(train=False)], [
self.tester.model.nodes[node].get_output(train=False)
for node in self.output_nodes
])
else:
self.io.print_error(
'Say whaaaat ?! This model type is not supported : {}'.format(
self.model_type))
sel.init = False
def read_label_embedding(self):
self.label_embedding = None
self.label_vectors = None
if 'label_embedding_file' in self.cfgs.keys():
self.label_embedding = self.io.read_npz_file(
self.cfgs['label_embedding_file'], 'matrix')
self.label_vectors = self.io.read_npz_file(
self.cfgs['label_embedding_file'], 'vectors')
self.io.print_info('Read label embedding file {0}'.format(
self.cfgs['label_embedding_file']))
def load_confidence_model(self):
self.confidence_model = None
if 'confidence_model' in self.cfgs.keys():
self.confidence_model = joblib.load(self.cfgs['confidence_model'])
def run(self):
self.load_synset_file()
if not self.init:
return
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
image_batches = [
self.test_images[i:i + self.cfgs['batch_size']]
for i in range(0, len(self.test_images), self.cfgs['batch_size'])
]
self.io.print_info('{0} images split into {1} Batches of {2}'.format(
len(self.test_images), len(image_batches),
self.cfgs['batch_size']))
for idx, images in enumerate(image_batches):
if not self.cfgs['force']:
flag = self.check_jsons(images)
if flag:
self.io.print_warning('Skipping batch {0} of {1}'.format(
idx, len(image_batches)))
continue
pixels, basenames = self.fetch_images(images)
flag, predictions, confidence, t = self.classify_images(pixels)
if self.raw_predictions is False:
getty_like_filter = [
self.suppress_stop_list(g) for g in predictions
]
getty_like_safe = [
self.run_thresholds(g) for g in getty_like_filter
]
getty_like_pc = [self.map_concepts(g) for g in getty_like_safe]
getty_like_public = [
self.resolve_antonyms(g) for g in getty_like_pc
]
getty_like_unique = [
self.resolve_duplicates(g) for g in getty_like_public
]
predictions_pc = getty_like_unique
save_ext = '.json'
else:
predictions_pc = predictions
save_ext = '.npz'
if not flag:
continue
file_paths = [
os.path.join(self.cfgs['results_dir'], im + save_ext)
for im in basenames
]
if not self.raw_predictions:
to_save = [dict({'external': p}) for p in predictions_pc]
else:
to_save = predictions_pc
assert (len(file_paths) == len(to_save))
self.io.print_info('Done batch {0} of {1} in {2} secs'.format(
idx, len(image_batches), t))
f = map(self.io.save_good_labels, file_paths, to_save)
def check_jsons(self, images):
basenames = []
for image in images:
im_basename = os.path.basename(image)
im_basename, _ = os.path.splitext(im_basename)
basenames.append(im_basename)
flags = [
os.path.exists(os.path.join(self.cfgs['results_dir'], b + '.json'))
for b in basenames
]
return np.all(flags)
#
# Which measure to keep
# Precision/Recall/F1 scores
#
def eval(self):
self.load_synset_file()
self.test_images = self.io.read_images(self.cfgs['val_file_list'])
self.predictions = {}
self.gt = {}
for idx, image in enumerate(self.test_images):
if idx % 1000 == 0:
self.io.print_info('Reading {0}/{1}'.format(
idx, len(self.test_images)))
im_basename = os.path.basename(image)
im_basename, _ = os.path.splitext(im_basename)
result_json = os.path.join(self.cfgs['results_dir'],
im_basename + '.json')
gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json')
p, s = self.io.read_prediction_json(result_json)
g = self.io.read_vector(gt_json)
if p is [] or g is []:
continue
self.predictions[im_basename] = zip(p, s)
self.gt[im_basename] = list(set(g).intersection(self.concepts))
for t in self.cfgs['thresholds']:
self.compute_precision_recall(t)
def compute_precision_recall(self, thresh):
fps = {}
tps = {}
fns = {}
gt_dict = {}
for c in self.concepts:
fps[c], tps[c], fns[c], gt_dict[c] = [], [], [], []
for key, value in self.predictions.items():
detections = [v[0] for v in value if v[1] >= thresh]
gt = self.gt[key]
for g in gt:
gt_dict[g].append(key)
tp = list(set(gt).intersection(detections))
fp = list(set(detections) - set(gt))
fn = list(set(gt) - set(detections))
for t in tp:
tps[t].append(key)
for f in fp:
fps[f].append(key)
for n in fn:
fns[n].append(key)
for c in self.concepts:
if gt_dict[c] == []:
self.io.print_info(
'Skipping {0}, no samples in ground-truth'.format(c))
continue
p = len(tps[c]) / (len(tps[c]) + len(fps[c]) + 0.00000001)
r = len(tps[c]) / (len(gt_dict[c]) + 0.0)
f1 = self.compute_f1(p, r)
self.io.print_info(
'At {0} : {1} precision {2}, recall {3}, f1-score {4}'.format(
c, thresh, p, r, f1))
def compute_f1(self, p, r):
return 2 * (p * r) / (p + r + 0.000001)
def load_synset_file(self):
pfile = open(self.cfgs['synset_file'], 'r')
concepts = pfile.readlines()
pfile.close()
self.concepts = [p.strip() for p in concepts]
def load_mean_file(self):
if not self.cfgs['mean_file'] == 'None':
pfile = open(self.cfgs['mean_file'], 'r')
m = np.load(pfile)
pfile.close()
else:
m = [[[128.0]], [[128.0]], [[128.0]]]
m = np.mean(np.mean(m, axis=1), axis=1)
self.mean_pixel_value = np.array([m[0], m[1], m[2]])
self.pixel_scaling = self.cfgs['pixel_scaling'][
'test'] if 'pixel_scaling' in self.cfgs.keys() else None
self.channel_swap = self.cfgs['channel_swap'][
'test'] if 'channel_swap' in self.cfgs.keys() else None
self.image_shuffle = self.cfgs['image_shuffle'][
'test'] if 'image_shuffle' in self.cfgs.keys() else None
def jsonify(self, results):
return [[r[0], str(r[1])] for r in results]
def map_to_synsets(self, results):
results = {r[0]: r[1] for r in results}
return [results[c] for c in self.concepts]
def parse_predictions(self, p):
sort_idx = np.argsort(-p)
concepts = [(self.concepts[idx], float(p[idx]))
for idx in sort_idx[:self.cfgs['top_k']['test']]]
return concepts
def classify_images(self, images):
confidence = [None]
if images == []:
return False, [(), ()], 0.0
try:
tstart = time.time()
(predictions, features) = self.inference(images)
if self.label_vectors is not None:
predictions /= np.linalg.norm(predictions, axis=1)[:,
np.newaxis]
predictions = np.dot(predictions, self.label_vectors)
if self.confidence_model is not None:
confidence = [('Confidence', p)
for p in self.confidence_model.predict(features)]
tend = time.time()
except Exception as err:
self.io.print_error('Error processing image, {0}'.format(err))
return False, [(), ()], None, 0.0
if not self.raw_predictions:
results = True, [self.parse_predictions(p) for p in predictions
], confidence, '{0:.4f}'.format((tend - tstart))
else:
results = True, predictions, confidence, '{0:.4f}'.format(
(tend - tstart))
return results
def predict(self, image):
image = self.normalize_images([image])
return self.classify_images(image)
def normalize_images(self, images):
normalize_images = []
for image in images:
image *= 255.0
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info(
'Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
try:
image = cv2.resize(image, (self.im_h, self.im_w))
except Exception as err:
self.io.print_error('Could not parse test image {0}'.format(e))
return False, [], 0.0
image -= self.mean_pixel_value
image = image.transpose(2, 0, 1)
# Channel swap since caffe.io.load_image returns RGB image and training was done with BGR
image = image[(2, 1, 0), :, :]
normalize_images.append(image)
return [np.array(normalize_images)]
def scale_image(self, image):
#
# caffe scales image 0-1 in pixel values we bring it back to 255 as keras likes it that way
#
# No more caffe no more scaling down to 0-1 in caffe
# image *= 255.0
tstart = time.time()
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info(
'Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
tend = time.time()
tstart = time.time()
try:
image = cv2.resize(image,
(self.im_h, self.im_w)).astype(np.float32)
except Exception as err:
self.io.print_error('Could not parse {0}'.format(e))
return None
tend = time.time()
tstart = time.time()
if self.mean_pixel_value is not None:
image -= self.mean_pixel_value
if self.pixel_scaling is not None:
image /= self.pixel_scaling
if self.channel_swap is not None:
image = image[:, :, self.channel_swap]
# H,W,C to C,H,W
if self.image_shuffle is not None:
image = image.transpose(self.image_shuffle)
tend = time.time()
return image
def fetch_images(self, image_file_names):
images = []
basenames = []
tstart = time.time()
for idx, image_file_name in enumerate(image_file_names):
im_basename = os.path.basename(image_file_name)
im_basename, _ = os.path.splitext(im_basename)
if not urlparse.urlparse(image_file_name).scheme == "":
url_response = urllib.urlopen(image_file_name)
if url_response.code == 404:
print self.io.print_error(
'[Testing] URL error code : {1} for {0}'.format(
image_file_name, url_response.code))
continue
try:
string_buffer = StringIO.StringIO(url_response.read())
image = np.asarray(bytearray(string_buffer.read()),
dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
image = image[:, :, (2, 1, 0)]
image = image.astype(np.float32)
except Exception as err:
print self.io.print_error(
'[Testing] Error with URL {0} {1}'.format(
image_file_name, err))
continue
elif os.path.exists(image_file_name):
try:
fid = open(image_file_name, 'r')
stream = fid.read()
fid.close()
image = Image.open(cStringIO.StringIO(stream))
image = np.array(image)
image = image.astype(np.float32)
# image = cv2.imread(image_file_name)
# image = image[:, :, (2, 1, 0)]
# image = image.astype(np.float32)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error(
'[Testing] Error with image file {0} {1}'.format(
image_file_name, err))
continue
else:
try:
image = self.conn.read_image(image_file_name)
if image is None:
raise self.io.print_error(
"[Testing] Image data could not be loaded - %s" %
image_file_name)
continue
image = np.array(image)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error(
'[Testing] Error with S3 Bucket hash {0} {1}'.format(
image_file_name, err))
continue
# try : do or do not there is not try
image = self.scale_image(image)
if image is None:
continue
if self.verbose:
self.io.print_info('Processing {0} {1}'.format(
image_file_name, image.shape))
images.append(image)
basenames.append(im_basename)
tend = time.time()
self.io.print_info('Fetching {0} images took {1:.4f} secs'.format(
len(image_file_names), tend - tstart))
return [np.array(images)], basenames
def fetch_gif_frames(self, inGif):
# Only works for URLs for now
# write frames to disk and pass filenames to fetch_images
outFolder = '/tmp'
frame = Image.open(BytesIO(urllib.urlopen(inGif).read()))
nframes = 0
file_names = []
while frame:
frame_name = '{0}/{1}-{2}.png'.format(outFolder,
os.path.basename(inGif),
nframes)
frame.save(frame_name, 'png')
file_names.append(frame_name)
nframes += 1
try:
frame.seek(nframes)
except EOFError:
break
file_names = [file_names[i] for i in range(0, len(file_names), 10)]
return file_names
def non_max(self, predictions):
concepts = {}
for pred in predictions:
for p in pred:
if p[0] in concepts.keys():
concepts[p[0]].append(float(p[1]))
else:
concepts[p[0]] = [float(p[1])]
for key, value in concepts.items():
concepts[key] = np.max(np.array(value))
labels = concepts.keys()
scores = [concepts[l] for l in labels]
sort_idx = np.argsort(-np.array(scores))
results = [(labels[idx], str(scores[idx])) for idx in sort_idx]
return [results]
def square_it_up(self, image):
edge_size = np.min(image.shape[:2])
cy, cx = np.array(image.shape[:2]) // 2
r_img = image[np.max([0, cy - edge_size //
2]):np.min([cy +
edge_size // 2, image.shape[0]]),
np.max([0, cx - edge_size //
2]):np.min([cx +
edge_size // 2, image.shape[1]]), :]
return r_img
def resolve_duplicates(self, concepts):
seen = set()
unique = []
for c in concepts:
if c[0] in seen:
logger.debug("Suppressing duplicate {}:{}".format(c[0], c[1]))
continue
seen.add(c[0])
unique.append(c)
return unique
def filter_concepts(self, predictions):
return [p[:self.cfgs['store_top_k']] for p in predictions]
def suppress_stop_list(self, predictions):
return [c for c in predictions if c[0] not in self.cfgs['stop_list_']]
def run_thresholds(self, predictions):
return self.threshold_concepts(predictions)
def resolve_antonyms(self, human_predictions):
conflicting_predictions = []
human_dict = {d[0]: d[1] for d in human_predictions}
human_labels = [d[0] for d in human_predictions]
human_scores = [d[1] for d in human_predictions]
for c1, c2 in self.cfgs['antonym_list']:
if not (c1 in human_labels and c2 in human_labels):
continue
# if
s1, s2 = human_dict[c1], human_dict[c2]
idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1)
logger.debug('Suppressing {0}:{1}'.format(human_labels[idx],
human_scores[idx]))
del human_labels[idx]
del human_scores[idx]
# for
remove_flag = -1
for idx, group in enumerate(self.cfgs['count_order_list']):
_this = np.intersect1d(human_dict.keys(), group)
if len(_this) > 0:
remove_flag = idx + 1
break
# if
# for
if not remove_flag == len(self.cfgs['count_order_list']):
remove_tags = []
for g in self.cfgs['count_order_list'][remove_flag:]:
remove_tags.extend(g)
# for
for t in remove_tags:
if t not in human_labels:
continue
# if
ridx = human_labels.index(t)
del human_labels[ridx]
del human_scores[ridx]
# for
# if
return [(g, s) for g, s in zip(human_labels, human_scores)]
# def
def map_concepts(self, concepts):
mapped_predictions = [(self.mapping[c[0]],
c[1]) if c[0] in self.mapping.keys() else c
for c in concepts]
mapped_predictions = self.conditional_mapping(mapped_predictions)
return mapped_predictions
# def
def conditional_mapping(self, predictions):
if 'conditional_list' in self.cfgs.keys():
for condition, mapping in self.cfgs['conditional_list']:
concepts = [p[0] for p in predictions]
holds = list(set(concepts).intersection(set(condition)))
if not len(holds) == 0:
predictions = [
p for p in predictions if not p[0] in mapping
]
# if
# for
# if
return predictions
# def
def threshold_concepts(self, predictions):
if self.thresholds is not None:
predictions = [
p for p in predictions if p[1] >= self.thresholds[p[0]]
]
# if
return predictions
class AlexHash():
def __init__(self):
self.config_file = None
self.io = EmbeddingIO(None)
self.init = False
self.cfgs = None
#def
def configure(self,config_file):
self.config_file = config_file
self.init = False
if not os.path.exists(self.config_file):
self.io.print_error('Could not find config file for VGG-19 {0}'.format(self.config_file))
self.init = False
return
#if
pfile = open(self.config_file,'r')
self.cfgs = yaml.load(pfile)
pfile.close()
self.init = True
#def
def define(self):
#
# TODO : get the filter size, number of filters, reception field size from yaml file
#
try:
self.model = Graph()
self.model.add_input(name='img', input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width']))
# C1
self.model.add_node(Convolution2D(96, 11, 11,subsample=(4,4)),name='conv1',input='img')
self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu1',input='conv1')
#self.model.add_node(LRN2D(alpha=0.0001, k=1, beta=0.75, n=5),name='norm1', input='relu1')
self.model.add_node(MaxPooling2D(pool_size=(3, 3),stride=(2,2)),name='pool1',input='norm1')
# C1
# C2
self.model.add_node(ZeroPadding2D((2,2)),name='zpad5',input='pool1')
self.model.add_node(Convolution2D(256, 5, 5),name='conv2',input='zpad5')
self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu2',input='conv2')
#self.model.add_node(LRN2D(alpha=0.0001, k=1, beta=0.75, n=5),name='norm2', input='relu2')
self.model.add_node(MaxPooling2D(pool_size=(3, 3),stride=(2,2)),name='pool2',input='norm2')
# C2
# C3
self.model.add_node(ZeroPadding2D((1,1)),name='zpad9',input='pool2')
self.model.add_node(Convolution2D(384, 3, 3),name='conv3',input='zpad9')
self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu3',input='conv3')
# C3
# C4
self.model.add_node(ZeroPadding2D((1,1)),name='zpad11',input='relu3')
self.model.add_node(Convolution2D(384, 3, 3),name='conv4',input='zpad11')
self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu4',input='conv4')
# C4
# C5
self.model.add_node(ZeroPadding2D((1,1)),name='zpad13',input='relu4')
self.model.add_node(Convolution2D(256, 3, 3),name='conv5',input='zpad13')
self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu5',input='conv5')
self.model.add_node(MaxPooling2D(pool_size=(3, 3),stride=(2,2)),name='pool5',input='relu5')
# C5
# Flatten - flat16 since its the 16th layer errr.
self.model.add_node(Flatten(),name='flat16',input='pool5')
# Flatten
# FC6
self.model.add_node(Dense(4096,init=self.cfgs['fc_init']),name='fc6',input='flat16')
self.model.add_node(Activation(self.cfgs['fc_non_linearity']),name='relu6',input='fc6')
self.model.add_node(Dropout(self.cfgs['drop_out']),name='drop6',input='relu6')
# FC6
# FC7
self.model.add_node(Dense(4096,init=self.cfgs['fc_init']),name='fc7',input='drop6')
self.model.add_node(Activation(self.cfgs['fc_non_linearity']),name='relu7',input='fc7')
self.model.add_node(Dropout(self.cfgs['drop_out']),name='drop7',input='relu7')
# FC7
# Contact fc6, fc7
self.model.add_node(Activation('linear'),name='concat',merge_mode='concat',inputs=['drop6','drop7'])
#
# Hash Layer
self.model.add_node(Dense(self.cfgs['n_bits']),name='hash',input='concat')
# Hash layer
# Last Aktivation
self.model.add_node(Activation(self.cfgs['activation']),name='bits',input='hash')
# Totally last Aktivation
self.model.add_output(name='output',input='bits')
if not self.cfgs['model_weights_file'] == 'None':
self.io.print_info('Porting weights from {0}'.format(self.cfgs['model_weights_file']))
self.init_from_this()
#if
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
#def
def init_from_this(self):
weights_file = self.cfgs['model_weights_file']
if weights_file.endswith('.npz'):
self.load_weights(weights_file)
self.io.print_info('Weights initalized from {0}'.format(weights_file))
#if
if weights_file.endswith('.caffemodel'):
self.io.print_warning('Loading from caffe model not implemented starting with random weights')
#if
#def
def load_weights(self, filepath):
'''
This method does not make use of Graph.set_weights()
for backwards compatibility. Has been modified to fit a graphical model
'''
# Loads weights from npz file
import numpy as np
pfile = open(filepath,'r')
p = np.load(pfile)
params = p['params'].item()
pfile.close()
for key, node in self.model.nodes.items():
if key in self.cfgs['ignore_nodes']:
self.io.print_info('Skipping weights transfer from {0},{1}'.format(key,node.get_config()['name']))
#if
if key in params.keys():
w = params[key]
node.set_weights(w)
self.io.print_info('Transferring weights from {0},{1}'.format(key,node.get_config()['name']))
#if
#for
#def
def compile(self,compile_cfgs):
try:
sgd = SGD(lr=compile_cfgs['lr'], decay=compile_cfgs['decay'], momentum=compile_cfgs['momentum'], nesterov=True)
self.model.compile(loss=loss_functions_dict[compile_cfgs['loss']], optimizer=sgd)
except Exception as e:
self.io.print_error('Error configuring the model, {0}'.format(e))
self.init = False
return
#try
self.init = True
class TheanoTrainerCaptions():
def __init__(self, config_file, verbose):
if not os.path.exists(config_file):
print 'Error : could not find config file'.format(config_file)
self.init = False
return
self.trainer_config_file = config_file
self.read_config()
self.io = EmbeddingIO(self.cfgs['log_dir'], 'training')
self.trainer_name = self.cfgs['model_name']
self.model_config_file = self.cfgs['model_config_file']
self.verbose = verbose
self.init = False
if self.trainer_name not in model_dict.keys():
self.io.print_error(
'Not a valid model type {0} chose among {1}'.format(
self.trainer_name, model_dict.keys()))
self.init = False
return
if not os.path.exists(self.trainer_config_file):
self.io.print_error('Could not find configuration file {0}'.format(
self.trainer_config_file))
self.init = False
return
self.trainer = None
self.train_lmdb = None
self.val_lmdb = None
self.all_labels = None
self.save_configs()
def save_configs(self):
try:
shutil.copy(self.trainer_config_file, self.cfgs['snapshot_dir'])
except Exception as err:
self.io.print_info('Using copy at destination already {}'.format(
self.trainer_config_file))
for var in self.cfgs['save_info']:
try:
value = self.cfgs[var]
shutil.copy(value, self.cfgs['snapshot_dir'])
except Exception as e:
self.io.print_warning(
'Could not find requested attribute {0}'.format(var, e))
def read_config(self):
pfile = open(self.trainer_config_file)
self.cfgs = yaml.load(pfile)
pfile.close()
def setup(self):
self.trainer = model_dict[self.trainer_name]()
self.trainer.configure(self.model_config_file)
self.trainer.define()
self.model_type = self.trainer.model.get_config()['name']
self.im_h, self.im_w = self.trainer.cfgs[
'image_height'], self.trainer.cfgs['image_width']
self.load_mean_file()
self.load_synset_file()
self.setup_training_data()
self.setup_loss_function()
if not self.trainer.init:
self.io.print_error('Error with model definition')
self.init = False
return
self.io.print_info('{0} Model defined from {1}'.format(
self.trainer_name, self.model_config_file))
self.trainer.compile(self.cfgs)
self.compile_inference()
if not self.trainer.init:
self.io.print_error('Error with model complication')
self.init = False
return
self.io.print_info('{0} Model compiled'.format(self.trainer_name))
self.init = True
def compile_inference(self):
"""
This compile version is to be used for testing only. Since its optimized for predictions
"""
if self.model_type == 'Sequential':
self.inference = K.function(
self.trainer.model.get_input(train=False),
[self.trainer.model.layers[-1].get_output(train=False)])
elif self.model_type == 'Graph':
self.inference = K.function(
self.trainer.model.get_input(train=False), [
self.trainer.model.nodes[node].get_output(train=False)
for node in self.cfgs['output_node_name']['test']
])
else:
self.io.print_error(
'Say whaaaat ?! This model type is not supported : {}'.format(
self.model_type))
sel.init = False
self.io.print_info('Inference model compiled')
def setup_training_data(self):
# TODO [HSS]: Reduce the argument lenght of set_params
# This bit of set-up is not clean to read
self.train_lmdb = LMDBParser(self.cfgs['train_file'],
log_dir=self.cfgs['log_dir'])
self.val_lmdb = None
if 'val_file' in self.cfgs.keys():
self.val_lmdb = LMDBParser(self.cfgs['val_file'],
log_dir=self.cfgs['log_dir'])
self.label_image_mapping = None
if 'label_file' in self.cfgs.keys():
self.label_image_mapping = self.io.read_npz_file(
self.cfgs['label_file'], 'info').item()
self.io.print_info('Read label/image lookup file {0}'.format(
self.cfgs['label_file']))
self.train_lmdb.set_params((self.im_h, self.im_w),
self.mean_pixel_value,
self.cfgs['train_lmdb_keep'],
self.channel_swap,
self.pixel_scaling,
self.cfgs['label_sampling'],
self.concepts,
self.label_image_mapping,
image_shuffle=self.image_shuffle)
self.val_lmdb.set_params((self.im_h, self.im_w),
self.mean_pixel_value,
self.cfgs['val_lmdb_keep'],
self.channel_swap,
self.pixel_scaling,
'None',
self.concepts,
None,
image_shuffle=self.image_shuffle)
if 'captions' in self.cfgs['model_name']:
self.train_lmdb.set_captions_params(
self.trainer.LUT, self.trainer.cfgs['max_caption_len'],
self.trainer.vocabulary, self.cfgs['label_sampling'])
self.val_lmdb.set_captions_params(
self.trainer.LUT, self.trainer.cfgs['max_caption_len'],
self.trainer.vocabulary, 'captions')
def setup_loss_function(self):
self.trainer.setup_loss_function(self.train_lmdb.concepts_cdf)
def load_mean_file(self):
if not self.cfgs['mean_file'] == 'None':
pfile = open(self.cfgs['mean_file'], 'r')
m = np.load(pfile)
pfile.close()
m = np.mean(np.mean(m, axis=1), axis=1)
self.mean_pixel_value = np.array([m[2], m[1], m[0]],
dtype=np.float32)
else:
self.mean_pixel_value = np.array([128.0, 128.0, 128.0])
self.pixel_scaling = self.cfgs['pixel_scaling'][
'train'] if 'pixel_scaling' in self.cfgs.keys() else None
self.channel_swap = self.cfgs['channel_swap'][
'train'] if 'channel_swap' in self.cfgs.keys() else None
self.image_shuffle = self.cfgs['image_shuffle'][
'train'] if 'image_shuffle' in self.cfgs.keys() else None
def load_synset_file(self):
pfile = open(self.cfgs['synset_file'], 'r')
concepts = pfile.readlines()
pfile.close()
self.concepts = [p.strip() for p in concepts]
def run(self):
if not self.init:
self.io.print_error('Error initializing trainer')
return
self.show_train_info()
self.save_model_def()
prefix = self.cfgs['snapshot_prefix']
model_name = self.trainer.cfgs['model_weights_file']
#
# HSS : Shorter argument list ?
#
if prefix in model_name:
start_iter = int(model_name.split(prefix)[-1].split('.')[0][1:])
num_epocs = start_iter / self.cfgs['epoc_size']
lr_rates = [
self.trainer.update_optimizer(self.cfgs['decay'])
for i in range(num_epocs)
]
self.io.print_info(
'Learning rate updated for snapshot {},{}'.format(
model_name, lr_rates))
else:
start_iter = -1
for n_iter in range(start_iter + 1, self.cfgs['maximum_iteration']):
self.io.print_info('At iteration {0} of {1}'.format(
n_iter, self.cfgs['maximum_iteration']))
if n_iter % self.cfgs['stepsize'] == 0 and not n_iter == 0:
self.update_lr()
_, images, partial_captions, next_words = self.train_lmdb.get_batch[
self.cfgs['label_sampling']]()
train_logs = LogHistory()
try:
self.fit([images, partial_captions],
next_words,
batch_size=self.cfgs['batch_size'],
nb_epoch=self.cfgs['nb_epocs'],
callbacks=[train_logs],
verbose=1)
except Exception as e:
self.io.print_info('Skipped iteration {0}, {1}'.format(
n_iter, e))
if n_iter % self.cfgs['display'] == 0 and not n_iter == 0:
self.save_train_logs(n_iter, train_logs)
if n_iter % self.cfgs['test_interval'] == 0 and not n_iter == 0:
self.run_validation(n_iter)
if n_iter % self.cfgs['snapshot'] == 0 and not n_iter == 0:
self.save_snapshot(n_iter)
if n_iter % self.cfgs['epoc_size'] == 0 and not n_iter == 0:
last_lr = self.trainer.opt.get_config()['lr']
self.trainer.update_optimizer(self.cfgs['decay'])
current_lr = self.trainer.opt.get_config()['lr']
self.io.print_info(
'[LR] update : last lr {}, current lr {}'.format(
last_lr, current_lr))
self.close_lmdbs()
def run_threaded(self):
if not self.init:
self.io.print_error('Error initializing trainer')
return
self.show_train_info()
self.save_model_def()
prefix = self.cfgs['snapshot_prefix']
model_name = self.trainer.cfgs['model_weights_file']
#
# HSS : Shorter argument list ?
#
if prefix in model_name:
start_iter = int(model_name.split(prefix)[-1].split('.')[0][1:])
num_epocs = start_iter / self.cfgs['epoc_size']
lr_rates = [
self.trainer.update_optimizer(self.cfgs['decay'])
for i in range(num_epocs)
]
self.io.print_info(
'Learning rate updated for snapshot {},{}'.format(
model_name, lr_rates))
else:
start_iter = -1
_, images, partial_captions, next_words = self.train_lmdb.get_batch[
self.cfgs['label_sampling']]()
for n_iter in range(start_iter + 1, self.cfgs['maximum_iteration']):
self.io.print_info('At iteration {0} of {1}'.format(
n_iter, self.cfgs['maximum_iteration']))
if n_iter % self.cfgs['stepsize'] == 0 and not n_iter == 0:
self.update_lr()
async_result = pool.apply_async(
self.train_lmdb.get_batch[self.cfgs['label_sampling']])
train_logs = LogHistory()
try:
t1 = time()
self.fit([images, partial_captions],
next_words,
batch_size=self.cfgs['batch_size'],
nb_epoch=self.cfgs['nb_epocs'],
callbacks=[train_logs],
verbose=1)
t2 = time() - t1
if self.cfgs['logging_for_profiling'] is True:
self.io.print_info('fit took {}s'.format(round(t2, 2)))
t1 = time()
self.eval(n_iter, [images, partial_captions], next_words)
t2 = time() - t1
if self.cfgs['logging_for_profiling']:
self.io.print_info('eval took {}s'.format(round(t2, 2)))
except Exception as e:
self.io.print_info('Skipped iteration {0}, {1}'.format(
n_iter, e))
if n_iter % self.cfgs['display'] == 0 and not n_iter == 0:
self.save_train_logs(n_iter, train_logs)
if n_iter % self.cfgs['test_interval'] == 0 and not n_iter == 0:
self.run_validation(n_iter)
if n_iter % self.cfgs['snapshot'] == 0 and not n_iter == 0:
self.save_snapshot(n_iter)
if n_iter % self.cfgs['epoc_size'] == 0 and not n_iter == 0:
last_lr = self.trainer.opt.get_config()['lr']
self.trainer.update_optimizer(self.cfgs['decay'])
current_lr = self.trainer.opt.get_config()['lr']
self.io.print_info(
'[LR] update : last lr {}, current lr {}'.format(
last_lr, current_lr))
t1 = time()
_, images, partial_captions, next_words = async_result.get()
t2 = time() - t1
if self.cfgs['logging_for_profiling']:
self.io.print_info('waiting for batch took {}s'.format(
round(t2, 2)))
self.close_lmdbs()
def fit(self, input, output, **kwargs):
if self.model_type == 'Sequential':
self.trainer.model.fit(input, output, **kwargs)
elif self.model_type == 'Graph':
self.trainer.model.fit({
'input': input,
'output': output
}, **kwargs)
else:
self.io.print_error(
'Say whaaaat ?! This model type is not supported : {}'.format(
self.model_type))
def eval(self, n_iter, input, output, tag='TRAIN'):
# NOTE : This is loss not accuracy
pred = self.inference(input)[0]
pred /= pred.sum(axis=-1, keepdims=True)
# avoid numerical instability with _EPSILON clipping
pred = np.clip(pred, _EPSILON, 1.0 - _EPSILON)
if 'pn' in self.cfgs['loss']:
val_loss = -np.sum(np.sum(output * np.log(pred) +
(1 - output) * np.log(1 - pred),
axis=-1),
axis=-1)
else:
val_loss = -np.sum(np.sum(output * np.log(pred), axis=-1), axis=-1)
self.io.print_info('{2} At {0} n_iter {1}'.format(
n_iter, np.mean(val_loss), tag))
def update_lr(self):
pass
def build_k_hot(self, svec, getty_dictionary):
if svec == []:
return []
vec = np.zeros(len(getty_dictionary), dtype=float)
idx = np.array(
[getty_dictionary.index(d) for d in svec if d in getty_dictionary])
if idx.size == 0:
return []
vec[idx] = 1.0
return vec
def show_model_info(self):
self.io.print_warning('BEGIN MODEL INFO')
model = self.trainer.model
if self.model_type == 'Sequential':
for idx, layer in enumerate(model.layers):
layer_info = layer.get_config()
self.io.print_info('Layer {0} : {1}'.format(idx, layer_info))
elif self.model_type == 'Graph':
pass
self.io.print_warning('END MODEL INFO')
def show_train_info(self):
self.io.print_warning('BEGIN TRAINING INFO')
for var in self.cfgs['train_info']:
try:
value = self.cfgs[var]
self.io.print_info('{0} : {1}'.format(var, value))
except Exception as e:
self.io.print_warning(
'Could not find requested attribute {0}'.format(var, e))
self.io.print_warning('END TRAINING INFO')
def save_model_def(self):
yaml_string = self.trainer.model.to_yaml()
model_definition_file = os.path.join(self.cfgs['snapshot_dir'],
self.trainer_name + '_def.yaml')
pfile = open(model_definition_file, 'w')
pfile.write(yaml_string)
pfile.close()
def save_snapshot(self, n_iter):
weights_file = os.path.join(
self.cfgs['snapshot_dir'],
'{0}_{2}_{1}.hdf5'.format(self.trainer_name, n_iter,
self.cfgs['snapshot_prefix']))
self.trainer.model.save_weights(weights_file, overwrite=True)
self.io.print_info('Wrote snapshot to {0}'.format(weights_file))
def run_validation(self, n_iter):
self.io.print_info('Running validation')
_, val_images, val_partial_captions, val_next_words = self.val_lmdb.get_batch[
'captions']()
self.eval(n_iter, [val_images, val_partial_captions],
val_next_words,
tag='VALIDATION')
self.io.print_info('End of validation')
def save_train_logs(self, n_iter, train_logs):
self.io.print_info('[{1}] TRAINING : loss:{0}'.format(
np.mean(train_logs.losses), n_iter))
def save_val_logs(self, n_iter, val_logs):
self.io.print_info('[{1}] VALIDATION : loss:{0}'.format(
val_logs, n_iter))
def close_lmdbs(self):
if self.train_lmdb is not None:
self.train_lmdb.close()
if self.val_lmdb is not None:
self.val_lmdb.close()
class TheanoTrainer():
def __init__(self, config_file, verbose):
if not os.path.exists(config_file):
print 'Error : could not find config file'.format(config_file)
self.init = False
return
self.trainer_config_file = config_file
self.read_config()
self.io = EmbeddingIO(self.cfgs['log_dir'], 'training')
self.trainer_name = self.cfgs['model_name']
self.model_config_file = self.cfgs['model_config_file']
self.verbose = verbose
self.init = False
if self.trainer_name not in model_dict.keys():
self.io.print_error('Not a valid model type {0} chose among {1}'.format(self.trainer_name, model_dict.keys()))
self.init = False
return
if not os.path.exists(self.trainer_config_file):
self.io.print_error('Could not find configuration file {0}'.format(self.trainer_config_file))
self.init = False
return
self.trainer = None
self.train_lmdb = None
self.val_lmdb = None
self.all_labels = None
self.save_configs()
def scale_image(self, image):
#
# caffe scales image 0-1 in pixel values we bring it back to 255 as keras likes it that way
#
# No more caffe no more scaling down to 0-1 in caffe
# image *= 255.0
#tstart = time.time()
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info('Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
# tend = time.time()
# tstart = time.time()
try:
if self.cfgs['resize_images']:
image = cv2.resize(image, (self.im_h, self.im_w)).astype(np.float32)
else:
print 'no resize'
image=image.astype(np.float32)
except Exception as err:
self.io.print_error('Could not parse {0}'.format(e))
return None
# tend = time.time()
# tstart = time.time()
if self.mean_pixel_value is not None:
image -= self.mean_pixel_value
if self.pixel_scaling is not None:
image /= self.pixel_scaling
if self.channel_swap is not None:
image = image[:, :, self.channel_swap]
# H,W,C to C,H,W
if self.image_shuffle is not None:
image = image.transpose(self.image_shuffle)
#tend = time.time()
return image
def fetch_images(self, image_file_names):
images = []
basenames = []
#tstart = time.time()
for idx, image_file_name in enumerate(image_file_names):
image_file_name=image_file_name.replace('/nas/','/nas2/')
im_basename = os.path.basename(image_file_name)
im_basename, _ = os.path.splitext(im_basename)
if not urlparse.urlparse(image_file_name).scheme == "":
url_response = urllib.urlopen(image_file_name)
if url_response.code == 404:
print self.io.print_error('[Training] URL error code : {1} for {0}'.format(image_file_name, url_response.code))
continue
try:
string_buffer = StringIO.StringIO(url_response.read())
image = np.asarray(bytearray(string_buffer.read()), dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
image = image[:, :, (2, 1, 0)]
image = image.astype(np.float32)
except Exception as err:
print self.io.print_error('[Training] Error with URL {0} {1}'.format(image_file_name, err))
continue
elif os.path.exists(image_file_name):
try:
fid = open(image_file_name, 'r')
stream = fid.read()
fid.close()
image = Image.open(cStringIO.StringIO(stream))
image = np.array(image)
image = image.astype(np.float32)
# image = cv2.imread(image_file_name)
# image = image[:, :, (2, 1, 0)]
# image = image.astype(np.float32)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error('[Training] Error with image file {0} {1}'.format(image_file_name, err))
continue
else:
try:
image = self.conn.read_image(image_file_name)
if image is None:
raise self.io.print_error("[Training] Image data could not be loaded - %s" % image_file_name)
continue
image = np.array(image)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error('[Training] Error with S3 Bucket hash {0} {1}'.format(image_file_name, err))
continue
# try : do or do not there is not try
#orig_image=image.astype('u1')
image = self.scale_image(image)
if image is None:
continue
if self.verbose:
self.io.print_info('Processing {0} {1}'.format(image_file_name, image.shape))
images.append(image)
basenames.append(im_basename)
#tend = time.time()
#self.io.print_info('Fetching {0} images took {1:.4f} secs'.format(len(image_file_names), tend - tstart))
if self.cfgs['resize_images']:
return np.array(images)
else:
return images
def save_configs(self):
try:
shutil.copy(self.trainer_config_file, self.cfgs['snapshot_dir'])
except Exception as err:
self.io.print_info('Using copy at destination already {}'.format(self.trainer_config_file))
for var in self.cfgs['save_info']:
try:
value = self.cfgs[var]
shutil.copy(value, self.cfgs['snapshot_dir'])
except Exception as e:
self.io.print_warning('Could not find requested attribute {0}'.format(var, e))
def read_config(self):
pfile = open(self.trainer_config_file)
self.cfgs = yaml.load(pfile)
pfile.close()
def see_model(self):
self.trainer = model_dict[self.trainer_name]()
self.trainer.configure(self.model_config_file)
self.trainer.define()
self.load_mean_file()
self.im_h=779
self.im_w=779
self.load_synset_file()
self.trainer.compile(self.cfgs)
def setup(self):
self.trainer = model_dict[self.trainer_name]()
self.trainer.configure(self.model_config_file)
self.trainer.define()
self.model_type = self.trainer.model.get_config()['name']
self.im_h, self.im_w = self.trainer.cfgs['image_height'], self.trainer.cfgs['image_width']
self.load_mean_file()
self.load_synset_file()
self.setup_training_data()
self.setup_loss_function()
if not self.trainer.init:
self.io.print_error('Error with model definition')
self.init = False
return
self.io.print_info('{0} Model defined from {1}'.format(self.trainer_name, self.model_config_file))
self.trainer.compile(self.cfgs)
self.compile_inference()
if not self.trainer.init:
self.io.print_error('Error with model complication')
self.init = False
return
self.io.print_info('{0} Model compiled'.format(self.trainer_name))
self.init = True
def compile_inference(self):
"""
This compile version is to be used for testing only. Since its optimized for predictions
"""
if self.model_type == 'Sequential':
self.inference = K.function([self.trainer.model.get_input(train=False)], [self.trainer.model.layers[-1].get_output(train=False)])
elif self.model_type == 'Graph':
self.inference = K.function([self.trainer.model.get_input(train=False)], [self.trainer.model.nodes[node].get_output(train=False) for node in self.cfgs['output_node_name']['test']])
else:
self.io.print_error('Say whaaaat ?! This model type is not supported : {}'.format(self.model_type))
sel.init = False
self.io.print_info('Inference model compiled')
def setup_training_data(self):
# TODO [HSS]: Reduce the argument lenght of set_params
# This bit of set-up is not clean to read
self.train_lmdb = LMDBParser(self.cfgs['train_file'], log_dir=self.cfgs['log_dir'])
self.val_lmdb = None
if 'val_file' in self.cfgs.keys():
self.val_lmdb = LMDBParser(self.cfgs['val_file'], log_dir=self.cfgs['log_dir'])
self.label_image_mapping = None
self.label_embedding = None
self.label_vectors = None
# if 'label_file' in self.cfgs.keys():
# import ipdb; ipdb.set_trace()
# self.label_image_mapping = self.io.read_npz_file(self.cfgs['label_file'], 'info').item()
# #self.label_image_mapping =None
# self.io.print_info('Read label/image lookup file {0}'.format(self.cfgs['label_file']))
if 'label_file' in self.cfgs.keys():
images, mapping, vocab, vocab_mapping = self.io.read_npz_file(self.cfgs['label_file'], ['images', 'mapping', 'vocabulary', 'vocabulary_mapping'])
self.label_image_mapping = {'images': list(images), 'mapping': list(mapping), 'vocabulary': list(vocab), 'vocabulary_mapping': vocab_mapping.item()}
self.io.print_info('Read label/image lookup file {0}'.format(self.cfgs['label_file']))
if 'label_embedding_file' in self.cfgs.keys():
self.label_embedding = self.io.read_npz_file(self.cfgs['label_embedding_file'], 'matrix')
self.label_vectors = self.io.read_npz_file(self.cfgs['label_embedding_file'], 'vectors')
self.io.print_info('Read label embedding file {0}'.format(self.cfgs['label_embedding_file']))
self.train_lmdb.set_params((self.im_h, self.im_w),
self.mean_pixel_value,
self.cfgs['train_lmdb_keep'],
self.channel_swap,
self.pixel_scaling,
self.cfgs['label_sampling'],
self.concepts,
self.label_image_mapping,
image_shuffle=self.image_shuffle,
label_embedding=self.label_embedding)
self.val_lmdb.set_params((self.im_h, self.im_w),
self.mean_pixel_value,
self.cfgs['val_lmdb_keep'],
self.channel_swap,
self.pixel_scaling,
'None',
self.concepts,
None,
image_shuffle=self.image_shuffle,
label_embedding=self.label_embedding)
if 'captions' in self.cfgs['model_name']:
self.train_lmdb.set_captions_params(self.trainer.LUT,
self.trainer.cfgs['max_caption_len'],
self.trainer.vocabulary,
self.cfgs['label_sampling'])
self.val_lmdb.set_captions_params(self.trainer.LUT,
self.trainer.cfgs['max_caption_len'],
self.trainer.vocabulary,
'captions')
def setup_loss_function(self):
self.trainer.setup_loss_function(self.train_lmdb.concepts_cdf)
def load_mean_file(self):
if not self.cfgs['mean_file'] == 'None':
pfile = open(self.cfgs['mean_file'], 'r')
m = np.load(pfile)
pfile.close()
m = np.mean(np.mean(m, axis=1), axis=1)
self.mean_pixel_value = np.array([m[2], m[1], m[0]], dtype=np.float32)
else:
self.mean_pixel_value = np.array([128.0, 128.0, 128.0])
self.pixel_scaling = self.cfgs['pixel_scaling']['train'] if 'pixel_scaling' in self.cfgs.keys() else None
self.channel_swap = self.cfgs['channel_swap']['train'] if 'channel_swap' in self.cfgs.keys() else None
self.image_shuffle = self.cfgs['image_shuffle']['train'] if 'image_shuffle' in self.cfgs.keys() else None
def load_synset_file(self):
pfile = open(self.cfgs['synset_file'], 'r')
concepts = pfile.readlines()
pfile.close()
self.concepts = [p.strip() for p in concepts]
def run(self):
if not self.init:
self.io.print_error('Error initializing trainer')
return
self.show_train_info()
self.save_model_def()
prefix = self.cfgs['snapshot_prefix']
model_name = self.trainer.cfgs['model_weights_file']
#
# HSS : Shorter argument list ?
#
if prefix in model_name:
start_iter = int(model_name.split(prefix)[-1].split('.')[0][1:])
num_epocs = start_iter / self.cfgs['epoc_size']
lr_rates = [self.trainer.update_optimizer(self.cfgs['decay']) for i in range(num_epocs)]
self.io.print_info('Learning rate updated for snapshot {},{}'.format(model_name, lr_rates))
else:
start_iter = -1
#import ipdb; ipdb.set_trace()
for n_iter in range(start_iter + 1, self.cfgs['maximum_iteration']):
self.io.print_info('At iteration {0} of {1}'.format(n_iter, self.cfgs['maximum_iteration']))
if n_iter % self.cfgs['stepsize'] == 0 and not n_iter == 0:
self.update_lr()
file_names, images, labels_train, labels_test = self.train_lmdb.get_batch[self.cfgs['label_sampling']]()
orig_images=self.fetch_images(file_names)
train_logs = LogHistory()
try:
# if self.cfgs['resize_images'] == False:
# for img,lab_train in zip(orig_images,labels_train):
# self.fit([img], [lab_train], batch_size=self.cfgs['batch_size'], nb_epoch=self.cfgs['nb_epocs'], callbacks=[train_logs], verbose=1)
# else:
self.fit(orig_images, labels_train, batch_size=self.cfgs['batch_size'], nb_epoch=self.cfgs['nb_epocs'], callbacks=[train_logs], verbose=1)
self.eval(n_iter, orig_images, labels_test)
except Exception as e:
self.io.print_info('Skipped iteration {0}, {1}'.format(n_iter, e))
if n_iter % self.cfgs['display'] == 0 and not n_iter == 0:
self.save_train_logs(n_iter, train_logs)
if n_iter % self.cfgs['test_interval'] == 0 and not n_iter == 0:
self.run_validation(n_iter)
if n_iter % self.cfgs['snapshot'] == 0 and not n_iter == 0:
self.save_snapshot(n_iter)
if n_iter % self.cfgs['epoc_size'] == 0 and not n_iter == 0:
last_lr = self.trainer.opt.get_config()['lr']
self.trainer.update_optimizer(self.cfgs['decay'])
current_lr = self.trainer.opt.get_config()['lr']
self.io.print_info('[LR] update : last lr {}, current lr {}'.format(last_lr, current_lr))
self.close_lmdbs()
def run_threaded(self):
if not self.init:
self.io.print_error('Error initializing trainer')
return
self.show_train_info()
self.save_model_def()
prefix = self.cfgs['snapshot_prefix']
model_name = self.trainer.cfgs['model_weights_file']
#
# HSS : Shorter argument list ?
#
if prefix in model_name:
start_iter = int(model_name.split(prefix)[-1].split('.')[0][1:])
num_epocs = start_iter / self.cfgs['epoc_size']
lr_rates = [self.trainer.update_optimizer(self.cfgs['decay']) for i in range(num_epocs)]
self.io.print_info('Learning rate updated for snapshot {},{}'.format(model_name, lr_rates))
else:
start_iter = -1
_, images, labels_train, labels_test = self.train_lmdb.get_batch[self.cfgs['label_sampling']]()
for n_iter in range(start_iter + 1, self.cfgs['maximum_iteration']):
self.io.print_info('At iteration {0} of {1}'.format(n_iter, self.cfgs['maximum_iteration']))
if n_iter % self.cfgs['stepsize'] == 0 and not n_iter == 0:
self.update_lr()
async_result = pool.apply_async(self.train_lmdb.get_batch[self.cfgs['label_sampling']])
train_logs = LogHistory()
try:
t1 = time()
self.fit(images, labels_train, batch_size=self.cfgs['batch_size'], nb_epoch=self.cfgs['nb_epocs'], callbacks=[train_logs], verbose=1)
t2 = time() - t1
if self.cfgs['logging_for_profiling'] is True:
self.io.print_info('fit took {}s'.format(round(t2, 2)))
t1 = time()
self.eval(n_iter, images, labels_test)
t2 = time() - t1
if self.cfgs['logging_for_profiling']:
self.io.print_info('eval took {}s'.format(round(t2, 2)))
except Exception as e:
self.io.print_info('Skipped iteration {0}, {1}'.format(n_iter, e))
if n_iter % self.cfgs['display'] == 0 and not n_iter == 0:
self.save_train_logs(n_iter, train_logs)
if n_iter % self.cfgs['test_interval'] == 0 and not n_iter == 0:
self.run_validation(n_iter)
if n_iter % self.cfgs['snapshot'] == 0 and not n_iter == 0:
self.save_snapshot(n_iter)
if n_iter % self.cfgs['epoc_size'] == 0 and not n_iter == 0:
last_lr = self.trainer.opt.get_config()['lr']
self.trainer.update_optimizer(self.cfgs['decay'])
current_lr = self.trainer.opt.get_config()['lr']
self.io.print_info('[LR] update : last lr {}, current lr {}'.format(last_lr, current_lr))
t1 = time()
_, images, labels_train, labels_test = async_result.get()
t2 = time() - t1
if self.cfgs['logging_for_profiling']:
self.io.print_info('waiting for batch took {}s'.format(round(t2, 2)))
self.close_lmdbs()
def fit(self, images, labels, **kwargs):
if self.model_type == 'Sequential':
self.trainer.model.fit(images, labels, **kwargs)
elif self.model_type == 'Graph':
self.trainer.model.fit({'input': images, 'output': labels}, **kwargs)
else:
self.io.print_error('Say whaaaat ?! This model type is not supported : {}'.format(self.model_type))
def eval(self, n_iter, images, labels, tag='TRAIN'):
img_batches = [images[i:i + self.cfgs['batch_size']] for i in range(0, len(images), self.cfgs['batch_size'])]
img_labels = [labels[i:i + self.cfgs['batch_size']] for i in range(0, len(labels), self.cfgs['batch_size'])]
acc = 0.0
for img, lab in zip(img_batches, img_labels):
(pred, feat) = self.inference([img])
"""
if self.model_type == 'Sequential':
pred = self.trainer.model.predict(img)
elif self.model_type == 'Graph':
pred = self.trainer.model.predict({'input':img})[self.cfgs['output_node_name']['train']]
else:
self.io.print_error('Say whaaaat ?! This model type is not supported : {}'.format(self.model_type))
#if
"""
acc += self.compute_accuracy(pred, lab)
acc /= len(img_batches)
self.io.print_info('{3} At {0} n_iter {2} {1}'.format(n_iter, acc, self.cfgs['metric'], tag))
def compute_accuracy(self, predictions, gt):
if self.label_vectors is not None:
# Need for L2 normalization of predictions and if need self.label_vectors
predictions /= np.linalg.norm(predictions, axis=1)[:, np.newaxis]
predictions = np.dot(predictions, self.label_vectors)
pred_labels = np.zeros(predictions.shape)
if self.cfgs['metric'] in ['precision']:
pred_labels[np.where(predictions > self.cfgs['global_threshold'])] = 1.0
elif self.cfgs['metric'] in ['top-k accuracy']:
good_labels = np.argsort(-predictions, axis=1)[:, :self.cfgs['top_k']['train']]
for idx in range(pred_labels.shape[0]):
pred_labels[idx, good_labels[idx]] = 1
else:
self.io.print_warning('Accuracy not set for {0}'.format(self.cfgs['loss']))
return 0.0
g = pred_labels * gt + 0.0
acc = np.nan_to_num(g.sum(axis=1) / (pred_labels.sum(axis=1))).mean()
return acc
def update_lr(self):
pass
def build_k_hot(self, svec, getty_dictionary):
if svec == []:
return []
vec = np.zeros(len(getty_dictionary), dtype=float)
idx = np.array([getty_dictionary.index(d) for d in svec if d in getty_dictionary])
if idx.size == 0:
return []
vec[idx] = 1.0
return vec
def show_model_info(self):
self.io.print_warning('BEGIN MODEL INFO')
model = self.trainer.model
if self.model_type == 'Sequential':
for idx, layer in enumerate(model.layers):
layer_info = layer.get_config()
self.io.print_info('Layer {0} : {1}'.format(idx, layer_info))
elif self.model_type == 'Graph':
pass
self.io.print_warning('END MODEL INFO')
def show_train_info(self):
self.io.print_warning('BEGIN TRAINING INFO')
for var in self.cfgs['train_info']:
try:
value = self.cfgs[var]
self.io.print_info('{0} : {1}'.format(var, value))
except Exception as e:
self.io.print_warning('Could not find requested attribute {0}'.format(var, e))
self.io.print_warning('END TRAINING INFO')
def save_model_def(self):
yaml_string = self.trainer.model.to_yaml()
model_definition_file = os.path.join(self.cfgs['snapshot_dir'], self.trainer_name + '_def.yaml')
pfile = open(model_definition_file, 'w')
pfile.write(yaml_string)
pfile.close()
def save_snapshot(self, n_iter):
weights_file = os.path.join(self.cfgs['snapshot_dir'], '{0}_{2}_{1}.hdf5'.format(self.trainer_name, n_iter, self.cfgs['snapshot_prefix']))
self.trainer.model.save_weights(weights_file, overwrite=True)
self.io.print_info('Wrote snapshot to {0}'.format(weights_file))
def run_validation(self, n_iter):
self.io.print_info('Running validation')
keys, val_images, _, val_labels = self.val_lmdb.get_batch['None']()
"""
if self.model_type == 'Sequential':
val_logs = self.trainer.model.evaluate(val_images, val_labels, batch_size=1)
elif self.model_type == 'Graph':
val_logs = self.trainer.model.evaluate({'input':val_images,self.cfgs['output_node_name']['train']:val_labels}, batch_size=1)
else:
self.io.print_error('Say whaaaat ?! This model type is not supported : {}'.format(self.model_type))
#if
"""
self.eval(n_iter, val_images, val_labels, tag='VALIDATION')
self.io.print_info('End of validation')
def save_train_logs(self, n_iter, train_logs):
self.io.print_info('[{1}] TRAINING : loss:{0}'.format(np.mean(train_logs.losses), n_iter))
def save_val_logs(self, n_iter, val_logs):
self.io.print_info('[{1}] VALIDATION : loss:{0}'.format(val_logs, n_iter))
def close_lmdbs(self):
if self.train_lmdb is not None:
self.train_lmdb.close()
if self.val_lmdb is not None:
self.val_lmdb.close()
