class VGG19_BN(): def __init__(self): self.config_file = None self.io = EmbeddingIO(None) self.init = False self.cfgs = None #def def configure(self, config_file): self.config_file = config_file self.init = False if not os.path.exists(self.config_file): self.io.print_error( 'Could not find config file for VGG-19-BN {0}'.format( self.config_file)) self.init = False return #if pfile = open(self.config_file, 'r') self.cfgs = yaml.load(pfile) pfile.close() self.init = True #def def define(self): # # TODO : get the filter size, number of filters, reception field size from yaml file # try: self.model = Sequential() # C1 self.model.add( ZeroPadding2D((1, 1), input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width']))) self.model.add(Convolution2D(64, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add(Convolution2D(64, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) # C1 # C2 self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(128, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(128, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) # C2 # C3 self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(256, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(256, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(256, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(256, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) # C3 # C4 self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(512, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(512, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(512, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(512, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) # C4 # C5 self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(512, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(512, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(512, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(ZeroPadding2D((1, 1))) self.model.add( Convolution2D(512, 3, 3, init=self.cfgs['conv_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['conv_non_linearity'])) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) # C5 # FC6 self.model.add(Flatten()) self.model.add(Dense(4096, init=self.cfgs['fc_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['fc_non_linearity'])) # FC6 # FC7 self.model.add(Dropout(self.cfgs['drop_out'])) self.model.add(Dense(4096, init=self.cfgs['fc_init'])) self.model.add(BatchNormalization(mode=0, epsilon=0.0001, axis=1)) self.model.add(Activation(self.cfgs['fc_non_linearity'])) # Fc7 # FC8 self.model.add(Dropout(self.cfgs['drop_out'])) self.model.add( Dense(self.cfgs['nb_classes'], init=self.cfgs['fc_init'])) self.model.add(Activation(self.cfgs['activation'])) # FC8 if not self.cfgs['model_weights_file'] == None: self.init_from_this() #if except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True #def def init_from_this(self): weights_file = self.cfgs['model_weights_file'] if weights_file.endswith('.hdf5'): self.load_weights(weights_file) self.io.print_info('Initalized from {0}'.format(weights_file)) #if if weights_file.endswith('.caffemodel'): self.io.print_warning( 'Loading from caffe model not implemented starting with random weights' ) #if #def def load_weights(self, filepath): ''' This method does not make use of Sequential.set_weights() for backwards compatibility. ''' # Loads weights from HDF5 file import h5py f = h5py.File(filepath) for k in range(f.attrs['nb_layers']): layer_type = self.model.layers[k].get_config()['name'] layer_name_string = '{0}_layer_{1}'.format(layer_type, k) if layer_name_string in self.cfgs['ignore_while_loading']: self.io.print_warning( 'Ignoring weights from {0}'.format(layer_name_string)) continue #if layer_name = 'layer_{}'.format(k) g = f[layer_name] self.io.print_info( 'Transfering parameters from {0}'.format(layer_name_string)) weights = [ g['param_{}'.format(p)] for p in range(g.attrs['nb_params']) ] self.model.layers[k].set_weights(weights) #for f.close() self.io.print_info(self.model.layers[-1].get_config().values()) #def def compile(self, compile_cfgs): try: sgd = SGD(lr=compile_cfgs['lr'], decay=compile_cfgs['decay'], momentum=compile_cfgs['momentum'], nesterov=True) self.model.compile(loss=loss_functions_dict[compile_cfgs['loss']], optimizer=sgd) except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True
class InceptionV3_finetune(): def __init__(self): self.layer_list = [] self.config_file = None self.io = EmbeddingIO(None) self.init = False self.cfgs = None self.loss_functions = ObjectiveFunctions() #def def configure(self, config_file): self.config_file = config_file self.init = False if not os.path.exists(self.config_file): self.io.print_error( 'Could not find config file for InceptionV3 {0}'.format( self.config_file)) self.init = False return #if pfile = open(self.config_file, 'r') self.cfgs = yaml.load(pfile) self.betas = random_float(0, 20, self.cfgs['n_channels_partClassifiers']) pfile.close() #Tracer()() self.init = True #def def add_to_graph(self, *args, **kwargs): self.model.add_node(*args, **kwargs) self.last_added_node = kwargs['name'] self.layer_list.append(kwargs['name']) return kwargs['name'] #def def add_bn_conv_layer(self, *args, **kwargs): layer_name = kwargs['name'] input_layer = kwargs['input'] del kwargs['name'] del kwargs['input'] if 'padding' in kwargs: layer_name = layer_name + '_pad' self.add_to_graph(ZeroPadding2D(padding=kwargs['padding']), name=layer_name, input=input_layer) input_layer = layer_name del kwargs['padding'] #if # CONV with linear activation by default layer_name = layer_name + '_conv' self.add_to_graph(Convolution2D(*args, border_mode='valid', **kwargs), name=layer_name, input=input_layer) # Batch normalization added directly on output of a linear layer input_layer = layer_name layer_name = layer_name + '_bn' _ = self.add_to_graph(BatchNormalization(mode=0, epsilon=0.0001, axis=1), name=layer_name, input=input_layer) # Standard normalization input_layer = layer_name layer_name = layer_name + '_nonlin' _ = self.add_to_graph(Activation('relu'), name=layer_name, input=input_layer) return layer_name #def def add_inceptionA(self, input_layer, list_nb_filter, base_name): l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1) l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2 = self.add_bn_conv_layer(name=base_name + '_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=5, nb_col=5, padding=(2, 2)) l3_1 = self.add_bn_conv_layer(name=base_name + '_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1) l3_2 = self.add_bn_conv_layer(name=base_name + '_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=3, nb_col=3, padding=(1, 1)) l3_3 = self.add_bn_conv_layer(name=base_name + '_l3_3', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=3, nb_col=3, padding=(1, 1)) l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1, 1)), name=base_name + '_14_1', input=input_layer) l4_2 = self.add_to_graph(AveragePooling2D(pool_size=(3, 3), strides=(1, 1)), name=base_name + '_14_2', input=l4_1) l4_3 = self.add_bn_conv_layer(name=base_name + '_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_2, l3_3, l4_3], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-A {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def add_inceptionB(self, input_layer, list_nb_filter, base_name): l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=3, nb_col=3, subsample=(2, 2)) l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2 = self.add_bn_conv_layer(name=base_name + '_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=3, nb_col=3, padding=(1, 1)) l2_3 = self.add_bn_conv_layer(name=base_name + '_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=3, nb_col=3, subsample=(2, 2)) l3_1 = self.add_to_graph(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)), name=base_name + '_13_1', input=input_layer) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_3, l3_1], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-B {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def add_inceptionC(self, input_layer, list_nb_filter, base_name): l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1) l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2 = self.add_bn_conv_layer(name=base_name + '_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=7, padding=(0, 3)) l2_3 = self.add_bn_conv_layer(name=base_name + '_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=7, nb_col=1, padding=(3, 0)) ## padding and nb_row might not match with the lasagne weights l3_1 = self.add_bn_conv_layer(name=base_name + '_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1) l3_2 = self.add_bn_conv_layer(name=base_name + '_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=7, nb_col=1, padding=(3, 0)) l3_3 = self.add_bn_conv_layer(name=base_name + '_l3_3', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=1, nb_col=7, padding=(0, 3)) l3_4 = self.add_bn_conv_layer(name=base_name + '_l3_4', input=l3_3, nb_filter=list_nb_filter[2][3], nb_row=7, nb_col=1, padding=(3, 0)) l3_5 = self.add_bn_conv_layer(name=base_name + '_l3_5', input=l3_4, nb_filter=list_nb_filter[2][4], nb_row=1, nb_col=7, padding=(0, 3)) l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1, 1)), name=base_name + '_14_1', input=input_layer) l4_2 = self.add_to_graph(AveragePooling2D(pool_size=(3, 3), strides=(1, 1)), name=base_name + '_14_2', input=l4_1) l4_3 = self.add_bn_conv_layer(name=base_name + '_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_3, l3_5, l4_3], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-C {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def add_inceptionD(self, input_layer, list_nb_filter, base_name): l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1) l1_2 = self.add_bn_conv_layer(name=base_name + '_l1_2', input=l1_1, nb_filter=list_nb_filter[0][1], nb_row=3, nb_col=3, subsample=(2, 2)) l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2 = self.add_bn_conv_layer(name=base_name + '_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=7, padding=(0, 3)) l2_3 = self.add_bn_conv_layer(name=base_name + '_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=7, nb_col=1, padding=(3, 0)) l2_4 = self.add_bn_conv_layer(name=base_name + '_l2_4', input=l2_3, nb_filter=list_nb_filter[1][2], nb_row=3, nb_col=3, subsample=(2, 2)) l3_1 = self.add_to_graph(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)), name=base_name + '_13_1', input=input_layer) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_2, l2_4, l3_1], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-D {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def add_inceptionE(self, input_layer, list_nb_filter, base_name, pool_mode): l1_1 = self.add_bn_conv_layer(name=base_name + '_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1) l2_1 = self.add_bn_conv_layer(name=base_name + '_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2a = self.add_bn_conv_layer(name=base_name + '_l2_2a', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=3, padding=(0, 1)) l2_2b = self.add_bn_conv_layer(name=base_name + '_l2_2b', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=3, nb_col=1, padding=(1, 0)) l3_1 = self.add_bn_conv_layer(name=base_name + '_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1) l3_2 = self.add_bn_conv_layer(name=base_name + '_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=3, nb_col=3, padding=(1, 1)) l3_3a = self.add_bn_conv_layer(name=base_name + '_l3_3a', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=1, nb_col=3, padding=(0, 1)) l3_3b = self.add_bn_conv_layer(name=base_name + '_l3_3b', input=l3_2, nb_filter=list_nb_filter[2][3], nb_row=3, nb_col=1, padding=(1, 0)) l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1, 1)), name=base_name + '_14_1', input=input_layer) l4_2 = self.add_to_graph(pooling_dict[pool_mode](pool_size=(3, 3), strides=(1, 1)), name=base_name + '_14_2', input=l4_1) l4_3 = self.add_bn_conv_layer(name=base_name + '_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_2a, l2_2b, l3_3a, l3_3b, l4_3], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-E {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def define(self): try: self.model = Graph() #self.model.add_input(name='input', input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width'])) self.model.add_input(name='input', input_shape=(self.cfgs['n_channels'], None, None)) # # Part of the network which is defined in the config file should move here # cgfs_nodes = self.cfgs['nodes'] for node in cgfs_nodes: if not node['type'] == 'Activation': self.add_to_graph( layer_dict[node['type']](**node['parameter']), name=node['name'], input=node['input']) else: self.add_to_graph(layer_dict[node['type']]( node['parameter']['mode']), name=node['name'], input=node['input']) #if self.io.print_info('Added {1}:{0}'.format( node['type'], node['name'])) #for self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64, ), (48, 64), (64, 96, 96), (32, )), base_name='mixed') self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64, ), (48, 64), (64, 96, 96), (64, )), base_name='mixed_1') self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64, ), (48, 64), (64, 96, 96), (64, )), base_name='mixed_2') self.add_inceptionB(input_layer=self.last_added_node, list_nb_filter=((384, ), (64, 96, 96)), base_name='mixed_3') self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192, ), (128, 128, 192), (128, 128, 128, 128, 192), (192, )), base_name='mixed_4') self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192, ), (160, 160, 192), (160, 160, 160, 160, 192), (192, )), base_name='mixed_5') self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192, ), (160, 160, 192), (160, 160, 160, 160, 192), (192, )), base_name='mixed_6') self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192, ), (192, 192, 192), (192, 192, 192, 192, 192), (192, )), base_name='mixed_7') self.add_inceptionD(input_layer=self.last_added_node, list_nb_filter=((192, 320), (192, 192, 192, 192)), base_name='mixed_8') self.add_inceptionE(input_layer=self.last_added_node, list_nb_filter=((320, ), (384, 384, 384), (448, 384, 384, 384), (192, )), pool_mode='average', base_name='mixed_9') self.add_inceptionE(input_layer=self.last_added_node, list_nb_filter=((320, ), (384, 384, 384), (448, 384, 384, 384), (192, )), pool_mode='max', base_name='mixed_10') #self.add_to_graph(AveragePooling2D(pool_size=(5,5)), name='pool3', input=self.last_added_node) #self.io.print_info('Added {1}:{0}'.format('AveragePooling',self.last_added_node)) """ self.add_to_graph(Flatten(), name='flatten', input='pool3') self.io.print_info('Added {1}:{0}'.format('Flatten',self.last_added_node)) self.add_to_graph(Dense(self.cfgs['nb_classes']), name='softmax', input='flatten') self.io.print_info('Added {1}:{0}'.format('Dense',self.last_added_node)) """ #print 'Adding finetuning layers'--------------------------------------------------------------------------------------- self.add_to_graph(BatchNormalization(mode=0, epsilon=0.0001, axis=1), name='batchnorm', input=self.last_added_node) self.io.print_info('Added {1}:{0}'.format( 'BatchNormalization before finetuning', self.last_added_node)) self.add_to_graph(Convolution2D( self.cfgs['n_channels_partClassifiers'], 1, 1), name='partClassifiers', input=self.last_added_node) self.io.print_info('Added {1}:{0}'.format('partClassifiers', self.last_added_node)) self.add_to_graph(Activation('relu'), name='relu_on_partClassifiers', input=self.last_added_node) self.io.print_info('Added {1}:{0}'.format( 'ReLu on partClassifiers', self.last_added_node)) if self.cfgs['pooling_AtfineTuning'] == 'avg': self.add_to_graph(global_Average_Pooling(), name='custom_pool', input=self.last_added_node) self.io.print_info('Added {1}:{0}'.format( 'Custom Pooling Layer:' + self.cfgs['pooling_AtfineTuning'], self.last_added_node)) elif self.cfgs['pooling_AtfineTuning'] == 'softpool': self.add_to_graph(power_mean_Layer(self.betas), name='custom_pool', input=self.last_added_node) # Learnable self.io.print_info('Added {1}:{0}'.format( 'Custom Pooling Layer:' + self.cfgs['pooling_AtfineTuning'], self.last_added_node)) self.add_to_graph(Flatten(), name='flatten', input=self.last_added_node) self.io.print_info('Added {1}:{0}'.format('Flatten', self.last_added_node)) self.add_to_graph(Dense(self.cfgs['nb_classes']), name='softmax', input=self.last_added_node) #Learnable self.io.print_info('Added {1}:{0}'.format('Softmax-dense weights', self.last_added_node)) self.add_to_graph(Activation(self.cfgs['activation']), name='prob', input=self.last_added_node) self.io.print_info('Added {1}:{0}'.format('Activation', self.last_added_node)) #Harsimrat original architecture --------------------------------------------------------------------------------------- # self.add_to_graph(AveragePooling2D(pool_size=(5,5)), name='pool3', input=self.last_added_node) # self.io.print_info('Added {1}:{0}'.format('AveragePooling',self.last_added_node)) # """ # self.add_to_graph(Flatten(), name='flatten', input='pool3') # self.io.print_info('Added {1}:{0}'.format('Flatten',self.last_added_node)) # self.add_to_graph(Dense(self.cfgs['nb_classes']), name='softmax', input='flatten') # self.io.print_info('Added {1}:{0}'.format('Dense',self.last_added_node)) # """ # self.add_to_graph(Convolution2D(self.cfgs['nb_classes'],1,1),name='softmax',input='pool3') # self.add_to_graph(Activation(self.cfgs['activation']), name='prob', input='softmax') #Adding dense finetunning layers #Re # Output to the Graph self.model.add_output(name='output', input='prob') #Tracer()() if not self.cfgs['model_weights_file'] == 'None': #import ipdb; ipdb.set_trace() self.init_from_this() #import ipdb; ipdb.set_trace() #if except Exception as err: self.io.print_error('Error configuring the model, {0}'.format(err)) self.init = False return #try self.init = True #def def init_from_this(self): weights_file = self.cfgs['model_weights_file'] #Tracer()() if not weights_file == 'None': self.load_weights(weights_file) self.io.print_info( 'Weights Initalized from {0}'.format(weights_file)) #if #def def load_weights(self, filepath): #Tracer()() if filepath.endswith('.npz'): pfile = open(filepath, 'r') graph = np.load(pfile)['graph'].item() for node_name, weights in graph.items(): if node_name in self.cfgs['ignore_while_loading']: self.io.print_warning( 'Ignoring weights from {0}'.format(node_name)) continue #if self.io.print_info( 'Transfering parameters from {0}'.format(node_name)) self.model.nodes[node_name].set_weights(weights) #for pfile.close() elif filepath.endswith('.npy'): pfile = open(filepath, 'r') graph = np.load(pfile).item() for node_name, weights in graph.items(): if node_name in self.cfgs['ignore_while_loading']: self.io.print_warning( 'Ignoring weights from {0}'.format(node_name)) continue #if self.io.print_info( 'Transfering parameters from {0}'.format(node_name)) self.model.nodes[node_name].set_weights(weights) #for pfile.close() elif filepath.endswith('.hdf5'): self.model.load_weights(filepath) else: self.io.print_error( 'Unknown model weights file {}'.format(filepath)) #if self.io.print_info(self.model.nodes['softmax'].get_config()) #def def setup_loss_function(self, w): self.loss_functions.set_weights(w) #def def compile(self, compile_cfgs): try: #opt = optimizer_dict[compile_cfgs['optimizer']](lr=compile_cfgs['lr'], epsilon=compile_cfgs['epsilon']) opt = SGD(lr=compile_cfgs['lr']) #model_layer = dict([(ler.name, layer) for layer in self.model.layesrs]) for layer_name, model_layer in self.model.nodes.items(): if layer_name not in self.cfgs['trainable_layers']: model_layer.trainable = False else: self.io.print_info( 'Trainable Layer: {0}'.format(layer_name)) self.model.compile(loss={ 'output': self.loss_functions.dict[compile_cfgs['loss']] }, optimizer=opt) except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True def get_model(self): return self.model
class SqueezeNet(): def __init__(self): self.config_file = None self.io = EmbeddingIO(None) self.init = False self.cfgs = None #def def configure(self,config_file): self.config_file = config_file self.init = False if not os.path.exists(self.config_file): self.io.print_error('Could not find config file for SqueezeNet {0}'.format(self.config_file)) self.init = False return #if pfile = open(self.config_file,'r') self.cfgs = yaml.load(pfile) pfile.close() self.init = True #def def define(self): # # TODO : get the filter size, number of filters, reception field size from yaml file # try: self.model = Graph() self.model.add_input(name='input',input_shape=(3,224,224)) #conv 1 self.model.add_node(Convolution2D(96, 3, 3, activation='relu', init='glorot_uniform',subsample=(2,2),border_mode='valid'),name='conv1', input='input') #maxpool 1 self.model.add_node(MaxPooling2D((2,2)),name='pool1', input='conv1') #fire 2 self.model.add_node(Convolution2D(16, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire2/conv1x1_1', input='pool1') self.model.add_node(Convolution2D(64, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire2/conv1x1_2', input='fire2/conv1x1_1') self.model.add_node(Convolution2D(64, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire2/conv3x3_2', input='fire2/conv1x1_1') self.model.add_node(Activation("linear"),name='fire2/concat', inputs=["fire2/conv1x1_2","fire2/conv3x3_2"], merge_mode="concat", concat_axis=1) #fire 3 self.model.add_node(Convolution2D(16, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='"fire3/conv1x1_1', input='fire2/concat') self.model.add_node(Convolution2D(64, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire3/conv1x1_2', input='"fire3/conv1x1_1') self.model.add_node(Convolution2D(64, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire3/conv3x3_2', input='"fire3/conv1x1_1') self.model.add_node(Activation("linear"),name='fire3/concat', inputs=["fire3/conv1x1_2","fire3/conv3x3_2"], merge_mode="concat", concat_axis=1) #fire 4 self.model.add_node(Convolution2D(32, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='"fire4/conv1x1_1', input='fire3/concat') self.model.add_node(Convolution2D(128, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire4/conv1x1_2', input='"fire4/conv1x1_1') self.model.add_node(Convolution2D(128, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire4/conv3x3_2', input='"fire4/conv1x1_1') self.model.add_node(Activation("linear"),name='fire4/concat', inputs=["fire4/conv1x1_2","fire4/conv3x3_2"], merge_mode="concat", concat_axis=1) #maxpool 4 self.model.add_node(MaxPooling2D((2,2)),name='pool4', input='fire4/concat') #fire 5 self.model.add_node(Convolution2D(32, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire5/conv1x1_1', input='pool4') self.model.add_node(Convolution2D(128, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire5/conv1x1_2', input='fire5/conv1x1_1') self.model.add_node(Convolution2D(128, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire5/conv3x3_2', input='fire5/conv1x1_1') self.model.add_node(Activation("linear"),name='fire5/concat', inputs=["fire5/conv1x1_2","fire5/conv3x3_2"], merge_mode="concat", concat_axis=1) #fire 6 self.model.add_node(Convolution2D(48, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire6/conv1x1_1', input='fire5/concat') self.model.add_node(Convolution2D(192, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire6/conv1x1_2', input='fire6/conv1x1_1') self.model.add_node(Convolution2D(192, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire6/conv3x3_2', input='fire6/conv1x1_1') self.model.add_node(Activation("linear"),name='fire6/concat', inputs=["fire6/conv1x1_2","fire6/conv3x3_2"], merge_mode="concat", concat_axis=1) #fire 7 self.model.add_node(Convolution2D(48, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire7/conv1x1_1', input='fire6/concat') self.model.add_node(Convolution2D(192, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire7/conv1x1_2', input='fire7/conv1x1_1') self.model.add_node(Convolution2D(192, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire7/conv3x3_2', input='fire7/conv1x1_1') self.model.add_node(Activation("linear"),name='fire7/concat', inputs=["fire7/conv1x1_2","fire7/conv3x3_2"], merge_mode="concat", concat_axis=1) #fire 8 self.model.add_node(Convolution2D(64, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire8/conv1x1_1', input='fire7/concat') self.model.add_node(Convolution2D(256, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire8/conv1x1_2', input='fire8/conv1x1_1') self.model.add_node(Convolution2D(256, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire8/conv3x3_2', input='fire8/conv1x1_1') self.model.add_node(Activation("linear"),name='fire8/concat', inputs=["fire8/conv1x1_2","fire8/conv3x3_2"], merge_mode="concat", concat_axis=1) #maxpool 8 self.model.add_node(MaxPooling2D((2,2)),name='pool8', input='fire8/concat') #fire 9 self.model.add_node(Convolution2D(64, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire9/conv1x1_1', input='pool8') self.model.add_node(Convolution2D(256, 1, 1, activation='relu', init='glorot_uniform',border_mode='same'),name='fire9/conv1x1_2', input='fire9/conv1x1_1') self.model.add_node(Convolution2D(256, 3, 3, activation='relu', init='glorot_uniform',border_mode='same'),name='fire9/conv3x3_2', input='fire9/conv1x1_1') self.model.add_node(Activation("linear"),name='fire9/concat', inputs=["fire9/conv1x1_2","fire9/conv3x3_2"], merge_mode="concat", concat_axis=1) self.model.add_node(Dropout(0.5),input='fire9/concat',name='drop9') #conv 10 self.model.add_node(Convolution2D(self.cfgs['nb_classes'], 1, 1, activation='relu', init='glorot_uniform',border_mode='valid'),name='conv_final', input='drop9') #avgpool 1 self.model.add_node(AveragePooling2D((13,13)),name='pool_final', input='conv_final') self.model.add_node(Flatten(),name='flatten',input='pool_final') self.model.add_node(Activation(self.cfgs['activation']),input='flatten',name='prob') self.model.add_output(name='output',input='prob') if not self.cfgs['model_weights_file'] == None: self.init_from_this() #if except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True #def def init_from_this(self): weights_file = self.cfgs['model_weights_file'] if weights_file.endswith('.hdf5'): self.load_weights(weights_file) self.io.print_info('Initalized from {0}'.format(weights_file)) #if if weights_file.endswith('.caffemodel'): self.io.print_warning('Loading from caffe model not implemented starting with random weights') #if #def def load_weights(self, filepath): ''' This method does not make use of Sequential.set_weights() for backwards compatibility. ''' # Loads weights from HDF5 file import h5py f = h5py.File(filepath) for k in range(f.attrs['nb_layers']): layer_type = self.model.layers[k].get_config()['name'] layer_name_string = '{0}_layer_{1}'.format(layer_type,k) if layer_name_string in self.cfgs['ignore_while_loading']: self.io.print_warning('Ignoring weights from {0}'.format(layer_name_string)) continue #if layer_name = 'layer_{}'.format(k) g = f[layer_name] self.io.print_info('Transfering parameters from {0}'.format(layer_name_string)) weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])] self.model.layers[k].set_weights(weights) #for f.close() self.io.print_info(self.model.layers[-1].get_config().values()) #def def compile(self,compile_cfgs): try: sgd = SGD(lr=compile_cfgs['lr'], decay=compile_cfgs['decay'], momentum=compile_cfgs['momentum'], nesterov=True) self.model.compile(loss=loss_functions_dict[compile_cfgs['loss']], optimizer=sgd) except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True #def
class GoogleNet(): def __init__(self): self.layer_list = [] self.config_file = None self.io = EmbeddingIO(None) self.init = False self.cfgs = None self.loss_functions = ObjectiveFunctions() #def def configure(self,config_file): self.config_file = config_file self.init = False if not os.path.exists(self.config_file): self.io.print_error('Could not find config file for GoogLeNet {0}'.format(self.config_file)) self.init = False return #if pfile = open(self.config_file,'r') self.cfgs = yaml.load(pfile) pfile.close() self.init = True #def def add_to_graph(self, *args, **kwargs): self.model.add_node(*args, **kwargs) self.last_added_node = kwargs['name'] self.layer_list.append(kwargs['name']) return kwargs['name'] #def def add_bn_conv_layer(self, *args, **kwargs): layer_name = kwargs['name'] input_layer = kwargs['input'] del kwargs['name'] del kwargs['input'] kwargs['border_mode'] = 'same' if 'padding' in kwargs: layer_name = layer_name + '_pad' self.add_to_graph(ZeroPadding2D(padding=kwargs['padding']), name=layer_name, input=input_layer) input_layer = layer_name del kwargs['padding'] #if # CONV with linear activation by default layer_name = layer_name + '_conv' self.add_to_graph(Convolution2D(*args, **kwargs), name=layer_name, input=input_layer) # Batch normalization added directly on output of a linear layer input_layer = layer_name layer_name = layer_name + '_bn' _ = self.add_to_graph(BatchNormalization(mode=0, epsilon=0.0001, axis=1), name=layer_name, input=input_layer) # Standard normalization input_layer = layer_name layer_name = layer_name + '_nonlin' _ = self.add_to_graph(Activation('relu'), name=layer_name, input=input_layer) return layer_name #def def add_inception(self, input_layer, list_nb_filter, base_name): tower_1_1 = self.add_bn_conv_layer(name=base_name+'tower_1_1', input=input_layer, nb_filter=list_nb_filter[0], nb_row=1, nb_col=1) tower_2_1 = self.add_bn_conv_layer(name=base_name+'tower_2_1', input=input_layer, nb_filter=list_nb_filter[1], nb_row=1, nb_col=1) tower_2_2 = self.add_bn_conv_layer(name=base_name+'tower_2_2', input=tower_2_1, nb_filter=list_nb_filter[2], nb_row=3, nb_col=3) tower_3_1 = self.add_bn_conv_layer(name=base_name+'tower_3_1', input=input_layer, nb_filter=list_nb_filter[3], nb_row=1, nb_col=1) tower_3_2 = self.add_bn_conv_layer(name=base_name+'tower_3_2', input=tower_3_1, nb_filter=list_nb_filter[4], nb_row=5, nb_col=5) tower_4_1 = self.add_to_graph(MaxPooling2D((3, 3), strides=(1, 1), border_mode='same'), name=base_name+'tower_4_1', input=input_layer) tower_4_2 = self.add_bn_conv_layer(name=base_name+'tower_4_2', input=tower_4_1, nb_filter=list_nb_filter[5], nb_row=1, nb_col=1) self.add_to_graph(Activation("linear"), name=base_name, inputs=[tower_1_1,tower_2_2,tower_3_2,tower_4_2], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception {0}'.format(base_name)) def define(self): try: self.model = Graph() self.model.add_input(name='input', input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width'])) pad_1 = self.add_to_graph(ZeroPadding2D((3,3)), name='pad_1',input='input') conv_1 = self.add_bn_conv_layer(name='conv_1', input=self.last_added_node, nb_filter=64, nb_row=7, nb_col=7,subsample=(2,2)) pad_2 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_2',input=self.last_added_node) max_1 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name='max_1', input=self.last_added_node) pad_3 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_3', input=self.last_added_node) conv_2 = self.add_bn_conv_layer( name='conv_2', input=self.last_added_node, nb_filter=192, nb_row=3, nb_col=3 ) pad_4 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_4', input=self.last_added_node) max_2 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name='max_2', input=self.last_added_node) #Inception_layer 1 nb_kernel_inception_1=[64,96,128,16,32,32] self.add_inception(self.last_added_node,nb_kernel_inception_1,'inception_1') #Inception_layer 2 nb_kernel_inception_2=[128,128,192,32,96,64] self.add_inception(self.last_added_node,nb_kernel_inception_2,'inception_2') pad_5 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_5', input=self.last_added_node) max_3 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name='max_3', input=self.last_added_node) #Inception_layer 3 nb_kernel_inception_3=[192,96,208,16,48,64] self.add_inception(self.last_added_node,nb_kernel_inception_3,'inception_3') #Inception_layer 4 nb_kernel_inception_4=[160,112,224,24,64,64] self.add_inception(self.last_added_node,nb_kernel_inception_4,'inception_4') #Inception_layer 5 nb_kernel_inception_5=[128,128,256,24,64,64] self.add_inception(self.last_added_node,nb_kernel_inception_5,'inception_5') #Inception_layer 6 nb_kernel_inception_6=[112,144,288,32,64,64] self.add_inception(self.last_added_node,nb_kernel_inception_6,'inception_6') #Inception_layer 7 nb_kernel_inception_7=[256,160,320,32,128,128] self.add_inception(self.last_added_node,nb_kernel_inception_7,'inception_7') pad_6 = self.add_to_graph(ZeroPadding2D((1,1)), name='pad_6', input=self.last_added_node) max_4 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name='max_4', input=self.last_added_node) #Inception_layer 8 nb_kernel_inception_8=[256,160,320,32,128,128] self.add_inception(self.last_added_node,nb_kernel_inception_8,'inception_8') #Inception_layer 9 nb_kernel_inception_9=[384,192,384,48,128,128] self.add_inception(self.last_added_node,nb_kernel_inception_9,'inception_9') self.add_to_graph(AveragePooling2D((7,7)), name='pool_1', input=self.last_added_node) self.add_to_graph(Flatten(), name='flatten', input='pool_1') self.add_to_graph(Dense(self.cfgs['nb_classes']), name='logits', input='flatten') self.add_to_graph(Activation(self.cfgs['activation']), name='prob', input='logits') # Output to the Graph self.model.add_output(name='output', input='prob') self.init_from_this() except Exception as err: self.io.print_error('Error configuring the model, {0}'.format(err)) self.init = False return #try self.init = True #def def init_from_this(self): weights_file = self.cfgs['model_weights_file'] if not weights_file == 'None': self.load_weights(weights_file) self.io.print_info('Weights Initalized from {0}'.format(weights_file)) #if #def def load_weights(self, filepath): if filepath.endswith('.npz'): pfile = open(filepath,'r') graph = np.load(pfile)['graph'].item() for node_name,weights in graph.items(): if node_name in self.cfgs['ignore_while_loading']: self.io.print_warning('Ignoring weights from {0}'.format(node_name)) continue #if self.io.print_info('Transfering parameters from {0}'.format(node_name)) self.model.nodes[node_name].set_weights(weights) #for pfile.close() elif filepath.endswith('.hdf5'): self.model.load_weights(filepath) else: self.io.print_error('Unknown model weights file {}'.format(filepath)) #if self.io.print_info(self.model.nodes['prob'].get_config()) def setup_loss_function(self,w): self.loss_functions.set_weights(w) #def def compile(self,compile_cfgs): try: opt = optimizer_dict[compile_cfgs['optimizer']](lr=compile_cfgs['lr'], epsilon=compile_cfgs['epsilon']) self.model.compile(loss={'output':self.loss_functions.dict[compile_cfgs['loss']]}, optimizer=opt) except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True
class InceptionV3RNN(): def __init__(self): self.layer_list = [] self.config_file = None self.io = EmbeddingIO(None) self.init = False self.cfgs = None #def def configure(self,config_file): self.config_file = config_file self.init = False if not os.path.exists(self.config_file): self.io.print_error('Could not find config file for InceptionV3 {0}'.format(self.config_file)) self.init = False return #if pfile = open(self.config_file,'r') self.cfgs = yaml.load(pfile) pfile.close() self.init = True #def def add_to_graph(self, *args, **kwargs): self.model.add_node(*args, **kwargs) self.last_added_node = kwargs['name'] self.layer_list.append(kwargs['name']) return kwargs['name'] #def def add_bn_conv_layer(self, *args, **kwargs): layer_name = kwargs['name'] input_layer = kwargs['input'] del kwargs['name'] del kwargs['input'] if 'padding' in kwargs: layer_name = layer_name + '_pad' self.add_to_graph(ZeroPadding2D(padding=kwargs['padding']), name=layer_name, input=input_layer) input_layer = layer_name del kwargs['padding'] #if # CONV with linear activation by default layer_name = layer_name + '_conv' self.add_to_graph(Convolution2D(*args, **kwargs), name=layer_name, input=input_layer) # Batch normalization added directly on output of a linear layer input_layer = layer_name layer_name = layer_name + '_bn' _ = self.add_to_graph(BatchNormalization(mode=0, epsilon=0.0001, axis=1), name=layer_name, input=input_layer) # Standard normalization input_layer = layer_name layer_name = layer_name + '_nonlin' _ = self.add_to_graph(Activation('relu'), name=layer_name, input=input_layer) return layer_name #def def add_inceptionA(self, input_layer, list_nb_filter, base_name): l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1) l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2 = self.add_bn_conv_layer(name=base_name+'_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=5, nb_col=5, padding=(2,2)) l3_1 = self.add_bn_conv_layer(name=base_name+'_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1) l3_2 = self.add_bn_conv_layer(name=base_name+'_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=3, nb_col=3, padding=(1,1)) l3_3 = self.add_bn_conv_layer(name=base_name+'_l3_3', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=3, nb_col=3, padding=(1,1)) l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1,1)), name=base_name+'_14_1', input=input_layer) l4_2 = self.add_to_graph(AveragePooling2D(pool_size=(3,3), strides=(1,1)), name=base_name+'_14_2', input=l4_1) l4_3 = self.add_bn_conv_layer(name=base_name+'_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_2, l3_3, l4_3], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-A {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def add_inceptionB(self, input_layer, list_nb_filter, base_name): l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=3, nb_col=3, subsample=(2, 2)) l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2 = self.add_bn_conv_layer(name=base_name+'_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=3, nb_col=3, padding=(1,1)) l2_3 = self.add_bn_conv_layer(name=base_name+'_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=3, nb_col=3, subsample=(2,2)) l3_1 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name=base_name+'_13_1', input=input_layer) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_3, l3_1], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-B {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def add_inceptionC(self, input_layer, list_nb_filter, base_name): l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1) l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2 = self.add_bn_conv_layer(name=base_name+'_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=7, padding=(0,3)) l2_3 = self.add_bn_conv_layer(name=base_name+'_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=7, nb_col=1, padding=(3,0)) ## padding and nb_row might not match with the lasagne weights l3_1 = self.add_bn_conv_layer(name=base_name+'_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1) l3_2 = self.add_bn_conv_layer(name=base_name+'_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=7, nb_col=1, padding=(3,0)) l3_3 = self.add_bn_conv_layer(name=base_name+'_l3_3', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=1, nb_col=7, padding=(0,3)) l3_4 = self.add_bn_conv_layer(name=base_name+'_l3_4', input=l3_3, nb_filter=list_nb_filter[2][3], nb_row=7, nb_col=1, padding=(3,0)) l3_5 = self.add_bn_conv_layer(name=base_name+'_l3_5', input=l3_4, nb_filter=list_nb_filter[2][4], nb_row=1, nb_col=7, padding=(0,3)) l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1,1)), name=base_name+'_14_1', input=input_layer) l4_2 = self.add_to_graph(AveragePooling2D(pool_size=(3,3), strides=(1,1)), name=base_name+'_14_2', input=l4_1) l4_3 = self.add_bn_conv_layer(name=base_name+'_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_3, l3_5, l4_3], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-C {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def add_inceptionD(self, input_layer, list_nb_filter, base_name): l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1) l1_2 = self.add_bn_conv_layer(name=base_name+'_l1_2', input=l1_1, nb_filter=list_nb_filter[0][1], nb_row=3, nb_col=3, subsample=(2,2)) l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2 = self.add_bn_conv_layer(name=base_name+'_l2_2', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=7, padding=(0,3)) l2_3 = self.add_bn_conv_layer(name=base_name+'_l2_3', input=l2_2, nb_filter=list_nb_filter[1][2], nb_row=7, nb_col=1, padding=(3,0)) l2_4 = self.add_bn_conv_layer(name=base_name+'_l2_4', input=l2_3, nb_filter=list_nb_filter[1][2], nb_row=3, nb_col=3, subsample=(2,2)) l3_1 = self.add_to_graph(MaxPooling2D(pool_size=(3,3), strides=(2,2)), name=base_name+'_13_1', input=input_layer) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_2, l2_4, l3_1], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-D {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def add_inceptionE(self, input_layer, list_nb_filter, base_name, pool_mode): l1_1 = self.add_bn_conv_layer(name=base_name+'_l1_1', input=input_layer, nb_filter=list_nb_filter[0][0], nb_row=1, nb_col=1) l2_1 = self.add_bn_conv_layer(name=base_name+'_l2_1', input=input_layer, nb_filter=list_nb_filter[1][0], nb_row=1, nb_col=1) l2_2a = self.add_bn_conv_layer(name=base_name+'_l2_2a', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=1, nb_col=3, padding=(0,1)) l2_2b = self.add_bn_conv_layer(name=base_name+'_l2_2b', input=l2_1, nb_filter=list_nb_filter[1][1], nb_row=3, nb_col=1, padding=(1,0)) l3_1 = self.add_bn_conv_layer(name=base_name+'_l3_1', input=input_layer, nb_filter=list_nb_filter[2][0], nb_row=1, nb_col=1) l3_2 = self.add_bn_conv_layer(name=base_name+'_l3_2', input=l3_1, nb_filter=list_nb_filter[2][1], nb_row=3, nb_col=3, padding=(1,1)) l3_3a = self.add_bn_conv_layer(name=base_name+'_l3_3a', input=l3_2, nb_filter=list_nb_filter[2][2], nb_row=1, nb_col=3, padding=(0,1)) l3_3b = self.add_bn_conv_layer(name=base_name+'_l3_3b', input=l3_2, nb_filter=list_nb_filter[2][3], nb_row=3, nb_col=1, padding=(1,0)) l4_1 = self.add_to_graph(ZeroPadding2D(padding=(1,1)), name=base_name+'_14_1', input=input_layer) l4_2 = self.add_to_graph(pooling_dict[pool_mode](pool_size=(3,3), strides=(1,1)), name=base_name+'_14_2', input=l4_1) l4_3 = self.add_bn_conv_layer(name=base_name+'_l4_3', input=l4_2, nb_filter=list_nb_filter[3][0], nb_row=1, nb_col=1) self.add_to_graph(Activation("linear"), name=base_name, inputs=[l1_1, l2_2a, l2_2b, l3_3a, l3_3b, l4_3], merge_mode="concat", concat_axis=1) self.io.print_info('Added Inception-E {0}'.format(base_name)) # https://github.com/fchollet/keras/issues/391 def define(self): try: self.model = Graph() self.model.add_input(name='input_image', input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width'])) self.model.add_input(name='input_label', input_shape=(1, self.cfgs['nb_classes'])) # CONVNET MODEL STARTS HERE # # Part of the network which is defined in the config file should move here # cgfs_nodes = self.cfgs['nodes'] for node in cgfs_nodes: if not node['type'] == 'Activation': self.add_to_graph(layer_dict[node['type']](**node['parameter']), name=node['name'] , input=node['input_image']) else: self.add_to_graph(layer_dict[node['type']](node['parameter']['mode']), name=node['name'] , input=node['input_image']) #if self.io.print_info('Added {1}:{0}'.format(node['type'],node['name'])) #for self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64,), (48, 64), (64, 96, 96), (32,)), base_name='mixed') self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64,), (48, 64), (64, 96, 96), (64,)), base_name='mixed_1') self.add_inceptionA(input_layer=self.last_added_node, list_nb_filter=((64,), (48, 64), (64, 96, 96), (64,)), base_name='mixed_2') self.add_inceptionB(input_layer=self.last_added_node, list_nb_filter=((384,), (64, 96, 96)), base_name='mixed_3') self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192,), (128, 128, 192), (128, 128, 128, 128, 192), (192,)), base_name='mixed_4') self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192,), (160, 160, 192), (160, 160, 160, 160, 192), (192,)), base_name='mixed_5') self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192,), (160, 160, 192), (160, 160, 160, 160, 192), (192,)), base_name='mixed_6') self.add_inceptionC(input_layer=self.last_added_node, list_nb_filter=((192,), (192, 192, 192), (192, 192, 192, 192, 192), (192,)), base_name='mixed_7') self.add_inceptionD(input_layer=self.last_added_node, list_nb_filter=((192, 320), (192, 192, 192, 192)), base_name='mixed_8') self.add_inceptionE(input_layer=self.last_added_node, list_nb_filter=((320,), (384, 384, 384), (448, 384, 384, 384), (192,)), pool_mode='average', base_name='mixed_9') self.add_inceptionE(input_layer=self.last_added_node, list_nb_filter=((320,), (384, 384, 384), (448, 384, 384, 384), (192,)), pool_mode='max', base_name='mixed_10') self.add_to_graph(AveragePooling2D(pool_size=(5,5)), name='pool3', input=self.last_added_node) self.io.print_info('Added {1}:{0}'.format('AveragePooling',self.last_added_node)) self.add_to_graph(Flatten(), name='flatten', input='pool3') self.io.print_info('Added {1}:{0}'.format('Flatten',self.last_added_node)) self.add_to_graph(Dense(self.cfgs['nb_classes']), name='softmax', input='flatten') self.io.print_info('Added {1}:{0}'.format('Dense',self.last_added_node)) # CONVNET MODEL ENDS HERE # RNN MODEL STARTS HERE self.add_to_graph(Embedding(self.cfgs['nb_classes'],self.cfgs['label_embedding_matrix'],self.cfgs['time_stamps']),name='label_embedding',input='input_label')) # RNN MODEL ENDS HERE if not self.cfgs['model_weights_file'] == None: self.init_from_this() #if except Exception as err: self.io.print_error('Error configuring the model, {0}'.format(err)) self.init = False return #try self.init = True #def def init_from_this(self): weights_file = self.cfgs['model_weights_file'] if not weights_file == 'None': self.load_weights(weights_file) self.io.print_info('Weights Initalized from {0}'.format(weights_file)) #if #def def load_weights(self, filepath): if filepath.endswith('.npz'): pfile = open(filepath,'r') graph = np.load(pfile)['graph'].item() for node_name,weights in graph.items(): if node_name in self.cfgs['ignore_while_loading']: self.io.print_warning('Ignoring weights from {0}'.format(node_name)) continue #if self.io.print_info('Transfering parameters from {0}'.format(node_name)) self.model.nodes[node_name].set_weights(weights) #for pfile.close() elif filepath.endswith('.hdf5'): self.model.load_weights(filepath) else: self.io.print_error('Unknown model weights file {}'.format(filepath)) #if self.io.print_info(self.model.nodes['softmax'].get_config()) def compile(self,compile_cfgs): try: opt = optimizer_dict[compile_cfgs['optimizer']](lr=compile_cfgs['lr'], epsilon=compile_cfgs['epsilon']) self.model.compile(loss={'output':loss_functions_dict[compile_cfgs['loss']]}, optimizer=opt) except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True #def
class LMDBParser(): def __init__(self, lmdb_dir_path, log_dir='/tmp', keep=32): # 1. Path to directory containing data.mdb # 2. keep which will take 1000 records from lmdb file into CPU memory self.lmdb_dir_path = lmdb_dir_path self.keep = keep self.passes = 0 self.datum_seen = 0 self.mean_weight = 1.4 self.label_distribution = None self.label_statistics = None self.label_statistics_train = None self.io = EmbeddingIO(log_dir, log_type='lmdb-parsing-{0}'.format( lmdb_dir_path.split('/')[-1])) if not os.path.exists(self.lmdb_dir_path): self.io.print_error('Could not find LMDB path {0}'.format( self.lmdb_dir_path)) self.init = False return try: self.lmdb_env = lmdb.open(self.lmdb_dir_path, readonly=True, lock=False) self.lmdb_txn = self.lmdb_env.begin() self.lmdb_cursor = self.lmdb_txn.cursor() self.read_label = self.lmdb_txn.get self.lmdb_cursor.first() self.datum = Datum() self.lmdb_entries = int(self.lmdb_txn.stat()['entries']) except Exception as e: self.io.print_error('Error reading LDMB file {0},{1}'.format( self.lmdb_dir_path, e)) self.init = False return self.io.print_info('Train LMDB file : {0} '.format(self.lmdb_dir_path)) self.init = True self.get_batch = { 'None': self.get_next_slice, 'drop_out': self.get_next_slice, 'mikolov': self.get_next_slice_sampled, 'mikolov_original': self.get_next_slice_sampled, 'mikolov_drop_out': self.get_next_slice_sampled, 'mikolov_em_drop_out': self.get_next_slice_sampled, 'mikolov_tetris_drop_out': self.get_next_slice_sampled, 'smote': self.get_next_slice_smote, 'captions': self.get_next_slice_captions, 'captions_mikolov': self.get_next_slice_captions_mikolov } def reset_data_stats(self): self.io.print_info('Mean weight for sampling {}'.format( self.mean_weight)) self.label_distribution = np.ones( (len(self.vocabulary), )) / (len(self.vocabulary) + 0.0) self.label_statistics = np.zeros((len(self.vocabulary), )) self.label_statistics_train = np.zeros((len(self.vocabulary), )) def set_params_test(self, vocabulary): self.vocabulary = vocabulary def set_params(self, crop_size, mean_pixel_value, keep, channel_swap, pixel_scaling, label_sampling, vocabulary, label_mapping, max_overlap=4, image_shuffle=[2, 0, 1], label_reduce=None, label_embedding=None): self.keep = keep self.crop_size = crop_size self.mean_pixel_value = mean_pixel_value self.vocabulary = vocabulary self.label_mapping = label_mapping self.channel_swap = channel_swap self.pixel_scaling = pixel_scaling self.label_sampling = label_sampling self.image_shuffle = image_shuffle self.label_reduce = label_reduce self.max_overlap = max_overlap self.label_embedding = label_embedding self.reset_data_stats() self.build_dataset_cdf() def set_captions_params(self, LUT=None, max_caption_len=None, caption_vocabulary=None, label_sampling=None): self.LUT = LUT self.max_caption_len = max_caption_len self.caption_vocabulary = caption_vocabulary self.label_sampling = label_sampling self.START = self.LUT['BOS'] self.END = self.LUT['EOS'] self.PAD = self.LUT['PAD'] def build_dataset_cdf(self): if self.label_mapping is None: return dist = [(k, len(v)) for k, v in self.label_mapping.items() if k in self.vocabulary] concepts = self.label_mapping['vocabulary'] concepts_count = [len(d) for d in self.label_mapping['mapping']] sort_idx = np.argsort(-np.array(concepts_count)) sorted_concepts = [concepts[idx] for idx in sort_idx] concepts_pdf = np.array(concepts_count) / (np.sum(concepts_count) + 0.0) concepts_cdf = np.cumsum(concepts_pdf) self.concepts_pdf = np.array( [concepts_pdf[sorted_concepts.index(c)] for c in self.vocabulary]) self.concepts_cdf = np.array( [concepts_cdf[sorted_concepts.index(c)] for c in self.vocabulary]) self.concepts_cdf_norm = self.concepts_cdf / np.sum(self.concepts_cdf) self.io.print_info('Training set statistics set up') # def build_dataset_cdf(self): # if self.label_mapping is None: # return # dist = [(k, len(v)) for k, v in self.label_mapping.items() if k in self.vocabulary] # concepts = [d[0] for d in dist] # concepts_count = [d[1] for d in dist] # sort_idx = np.argsort(-np.array(concepts_count)) # sorted_concepts = [concepts[idx] for idx in sort_idx] # concepts_pdf = np.array(concepts_count) / (np.sum(concepts_count) + 0.0) # concepts_cdf = np.cumsum(concepts_pdf) # self.concepts_pdf = np.array([concepts_pdf[sorted_concepts.index(c)] for c in self.vocabulary]) # self.concepts_cdf = np.array([concepts_cdf[sorted_concepts.index(c)] for c in self.vocabulary]) # self.concepts_cdf_norm = self.concepts_cdf / np.sum(self.concepts_cdf) # self.io.print_info('Training set statistics set up') def get_label_Frimgfile(self, filename): dirname = '/nas/datasets/getty-collection/getty-all/' key = dirname + filename value = self.lmdb_txn.get(key) #import ipdb; ipdb.set_trace() if not (value == None): self.datum.ParseFromString(value) lab = self.datum.label else: lab = [] return lab def get_labels(self, f, ID): #import ipdb; ipdb.set_trace() stream = f(ID) return pickle.loads(stream) def create_lmdb_keys_labels(self): import ipdb ipdb.set_trace() l = 0 env = lmdb.open("./keywording_test_images_labels", map_size=1e12) try: with env.begin(write=True) as txn: while 1: keys, labels = self.get_next_slice_img_lab() pickled_labels = [ pickle.dumps(label, protocol=1) for label in labels ] import ipdb ipdb.set_trace() if self.passes < 1: for i, key in enumerate(keys): ID, _ = os.path.splitext(os.path.basename(key)) if pickled_labels[i] != None: txn.put(ID.encode('ascii'), pickled_labels[i]) l = l + 1 print "Processes %d" % l #break else: break except Exception as e: print 'problem processing image %s' % ID import ipdb ipdb.set_trace() def create_keys_labels_dict(self): import ipdb ipdb.set_trace() keys_labels = dict() l = 0 import ipdb ipdb.set_trace() while 1: keys, labels = self.get_next_slice_img_lab() #pickled_labels = [pickle.dumps(label, protocol=1) for label in labels] if self.passes < 1: for i, key in enumerate(keys): ID, _ = os.path.splitext(os.path.basename(key)) if len(labels[i]) == len(self.vocabulary): keys_labels[ID] = labels[i] print keys_labels[ID].sum() l = l + 1 #print "Processes %d" % l #break else: print "Not Processes image %s" % ID break else: break import ipdb ipdb.set_trace() pickle.dump( keys_labels, open('/nas2/praveen/home/rnd-libs/rnd_libs/lib/keras/key_labels', 'wb')) import ipdb ipdb.set_trace() def get_next_slice_img_lab(self): # Warning : This is hard coded for inception_v3 model should be set from config file labels = np.zeros((self.keep, len(self.vocabulary)), dtype='u1') keys = [] # TODO : HSS - setting fields from YAML # [HSS] get rid of while loops with cleaner sampling file i = 0 while i < self.keep: # if End of LMDB file start from the begining if not self.lmdb_cursor.next(): self.lmdb_cursor.first() self.passes += 1 self.io.print_info( 'Start of next pass over the test set {0}'.format( self.passes)) # Read a key-value record from the queue try: key, value = self.lmdb_cursor.item() self.datum.ParseFromString(value) lab = self.datum.label if lab is None: self.io.print_warning( '{1}/{2} Skipping {0}, no useful label found'.format( key, i, self.keep)) continue labels[i, :] = lab keys.append(key) i += 1 except Exception as e: self.io.print_info('Skipping corrupted images, {0}'.format(e)) # update label statistics and sampling distribution #self.update_label_stats(labels) test_labels_full = labels #test_labels_semi = self.label_drop(keys, test_labels_full) if 'drop_out' in self.label_sampling else test_labels_full # if self.label_embedding is not None: # train_labels_full = np.dot(test_labels_full, self.label_embedding) # train_labels_semi = np.dot(test_labels_semi, self.label_embedding) # else: # train_labels_full = test_labels_full # train_labels_semi = test_labels_semi return keys, test_labels_full def get_next_slice(self): # Warning : This is hard coded for inception_v3 model should be set from config file images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1])) labels = np.zeros((self.keep, len(self.vocabulary))) keys = [] # TODO : HSS - setting fields from YAML # [HSS] get rid of while loops with cleaner sampling file i = 0 while i < self.keep: # if End of LMDB file start from the begining if not self.lmdb_cursor.next(): self.lmdb_cursor.first() self.passes += 1 self.io.print_info( 'Start of next pass over the training set {0}'.format( self.passes)) # Read a key-value record from the queue try: key, value = self.lmdb_cursor.item() arr, lab = self.get_image_label(value) if lab is None: self.io.print_warning( '{1}/{2} Skipping {0}, no useful label found'.format( key, i, self.keep)) continue images[i, :, :, :] = arr labels[i, :] = lab keys.append(key) i += 1 except Exception as e: self.io.print_info('Skipping corrupted images, {0}'.format(e)) # update label statistics and sampling distribution self.update_label_stats(labels) test_labels_full = labels test_labels_semi = self.label_drop( keys, test_labels_full ) if 'drop_out' in self.label_sampling else test_labels_full if self.label_embedding is not None: train_labels_full = np.dot(test_labels_full, self.label_embedding) train_labels_semi = np.dot(test_labels_semi, self.label_embedding) else: train_labels_full = test_labels_full train_labels_semi = test_labels_semi return keys, images, train_labels_semi, test_labels_full def get_next_slice_sampled(self): # Warning : This is hard coded for inception_v3 model should be set from config file images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1])) labels = np.zeros((self.keep, len(self.vocabulary))) pseudo_labels = np.zeros((self.keep, len(self.vocabulary))) keys = [] # # while since some smaples might not be in this lmdb # i = 0 # # [HSS] get rid of while loops with cleaner sampling file # while i < self.keep: # sample random class if 'tetris' in self.label_sampling: # sample classes which havent been seen if self.datum_seen == 0: random_class = np.random.choice(self.vocabulary, 1)[0] else: prob = 1.0 - self.label_statistics_train / ( np.max(self.label_statistics_train) + 0.0) prob /= np.sum(prob) random_class = np.random.choice(self.vocabulary, 1, p=prob)[0] else: random_class = np.random.choice( self.label_mapping['vocabulary'], 1)[0] # get the class index to build the vector later random_class_index = self.label_mapping['vocabulary_mapping'][ random_class] # select random image sample for the sampled class image_index = np.random.choice( self.label_mapping['mapping'][random_class_index], 1)[0] # fetching actual image string from integer to image mapping key = str(self.label_mapping['images'][image_index]) # casting to str is important, because sometimes the key is unicode # look up in LMDB for image-data value = self.lmdb_txn.get(key) # skip if not in current lmdb if value is None: continue self.datum_seen += 1 if self.datum_seen >= self.lmdb_entries: self.datum_seen = 0 self.passes += 1 self.io.print_info( 'Start of next pass over the training set {0}'.format( self.passes)) self.reset_data_stats() try: arr, lab = self.get_image_label(value) if lab is None: self.io.print_warning( '{1}/{2} Skipping {0}, no useful label found'.format( key, i, self.keep)) continue nlab = np.zeros_like(lab) # # set random class index to 1 rest to 0 # nlab[random_class_index] = 1.0 if len(lab) == 0: continue images[i, :, :, :] = arr labels[i, :] = lab pseudo_labels[i, :] = nlab keys.append(key) i += 1 except Exception as e: self.io.print_info('Skipping corrupted images, {0}'.format(e)) # update label statistics and sampling distribution self.update_label_stats(labels) base_labels = pseudo_labels if self.label_sampling == 'mikolov' else labels base_labels = self.label_drop( keys, base_labels) if 'drop_out' in self.label_sampling else base_labels test_labels_full = labels self.label_statistics_train += np.sum(base_labels, axis=0) if self.label_embedding is not None: train_labels_semi = np.dot(base_labels, self.label_embedding) else: train_labels_semi = base_labels return keys, images, train_labels_semi, test_labels_full def update_label_stats(self, labels): self.label_statistics += np.sum(labels, axis=0) self.label_distribution[np.where( self.label_statistics)] /= self.label_statistics[np.where( self.label_statistics)] self.label_distribution /= np.sum(self.label_distribution) def get_next_slice_smote(self): # Warning : This is hard coded for inception_v3 model should be set from config file images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1])) labels = np.zeros((self.keep, len(self.vocabulary))) all_labels = np.zeros((self.keep, len(self.vocabulary))) keys = [] # # while since some smaples might not be in this lmdb # i = 0 # # [HSS] get rid of while loops with cleaner sampling file # current_labels = None while i < self.keep: # sample random class random_class = np.random.choice(self.vocabulary, 1)[0] # self.io.print_info('Sampled {0}'.format(random_class)) # get the class index to build the vector later random_class_index = self.vocabulary.index(random_class) # select random image sample for the sampled class key = str(np.random.choice(self.label_mapping[random_class], 1)[0]) # casting to str is important, because sometimes the key is unicode # look up in LMDB for image-data value = self.lmdb_txn.get(key) # skip if not in current lmdb if value is None: continue self.datum_seen += 1 if self.datum_seen >= self.lmdb_entries: self.datum_seen = 0 self.passes += 1 self.io.print_info( 'Start of next pass over the training set {0}'.format( self.passes)) try: arr, lab = self.get_image_label(value) if lab is None: self.io.print_warning( '{1}/{2} Skipping {0}, no useful label found'.format( key, i, self.keep)) continue # current_labels, flag = self.is_good_to_add(current_labels,lab) current_labels, lab, true_labels, flag = self.get_good_labels( current_labels, lab) if not flag: continue images[i, :, :, :] = arr labels[i, :] = lab all_labels[i, :] = true_labels keys.append(key) i += 1 # self.io.print_info('Current batch size {}'.format(len(images))) except Exception as e: self.io.print_info('Skipping corrupted images, {0}'.format(e)) return keys, images, labels, all_labels def is_good_to_add(self, current, lab): if current is None: return np.copy(lab), True dot = np.dot(current, lab) # self.io.print_info('Dot product {0}'.format(dot)) if dot >= self.max_overlap: return np.copy(current), False else: current = np.bitwise_or(current, lab) # self.io.print_info('Used labels {0}'.format(np.sum(current))) return np.copy(current), True def get_good_labels(self, current, lab): all_labels = np.copy(lab) if current is None: return np.copy(lab), np.copy(lab), all_labels, True new_lab = np.bitwise_and(lab, np.invert(np.bitwise_and(current, lab))) if np.sum(new_lab) == 0: return np.copy(current), np.copy(lab), all_labels, False else: current = np.bitwise_or(current, lab) return np.copy(current), np.copy(new_lab), all_labels, True def get_image_label(self, value): self.datum.ParseFromString(value) arr = np.fromstring(self.datum.data, dtype=np.uint8).reshape(self.datum.channels, self.datum.height, self.datum.width) arr = arr.transpose(1, 2, 0) # This casting is required since uint8 and float32 cause an under-flow while mean subtraction below arr = np.asarray(arr, dtype=np.float32) if self.crop_size is not None: arr = cv2.resize(arr, self.crop_size).astype(np.float32) if self.mean_pixel_value is not None: arr -= self.mean_pixel_value if self.channel_swap is not None: arr = arr[:, :, self.channel_swap] if self.pixel_scaling is not None: arr /= self.pixel_scaling # As theano likes it N_channels x image_H x image_w if self.image_shuffle is not None: arr = arr.transpose(self.image_shuffle) # Label is a single field but could be an array lab = self.datum.label if self.label_reduce is not None: dlab = np.dot(self.label_reduce, lab) lab = np.zeros_like(dlab) lab[np.where(dlab)[0]] = 1 if np.sum(lab) == 0: lab = None return arr, lab def label_drop(self, keys, labels): if 'em' in self.label_sampling: # sample inversly to what network has seen in the entire history freq = self.label_statistics_train + 0.0 non_empty_inices = np.where(self.label_statistics_train) if not non_empty_inices == []: freq[np.where( self.label_statistics_train > self.mean_weight * np.mean(self.label_statistics_train[non_empty_inices]) )] = self.datum_seen freq /= self.datum_seen elif 'tetris' in self.label_sampling: residue = np.max( self.label_statistics_train) - self.label_statistics_train # reset if everything has been seen if np.sum(residue) == 0: return labels drop_labels = np.zeros_like(labels) # keep at most if available residue amount of samples for the label for idx, f in enumerate(residue): for idy in np.random.choice(np.where(labels[:, idx]), f): drop_labels[idy][idx] = 1.0 return np.multiply(labels, drop_labels) else: # sample inversly to batch frequency freq = np.sum(labels, axis=0) / (len(keys) + 0.0) # freq = 1 - self.concepts_cdf freq = [max(0, min(1, f)) for f in freq] drop_labels = np.array([ np.random.binomial(1, 1 - f, len(keys)) for f in freq ]).transpose() return np.multiply(labels, drop_labels) def get_next_slice_captions(self): images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1])) partial_captions = np.zeros((self.keep, self.max_caption_len)) next_words = np.zeros((self.keep, self.max_caption_len + 1, len(self.caption_vocabulary))) keys = [] caption_range = range(self.max_caption_len + 1) for i in range(self.keep): # if End of LMDB file start from the begining if not self.lmdb_cursor.next(): self.lmdb_cursor.first() self.passes += 1 self.io.print_info( 'Start of next pass over the training set {0}'.format( self.passes)) # Read a key-value record from the queue try: key, value = self.lmdb_cursor.item() arr, cap = self.get_image_label(value) caption = [self.START] + list(cap) + [self.END] caption = pad_sequences([caption], maxlen=self.max_caption_len, padding='post', value=self.PAD)[0] images[i, :, :, :] = arr partial_captions[i, :] = caption next_words[i, caption_range, np.hstack((caption, [self.PAD]))] = 1 keys.append(key) except Exception as e: self.io.print_info('Skipping corrupted images, {0}'.format(e)) return keys, images, partial_captions, next_words def get_next_slice_captions_mikolov(self): images = np.zeros((self.keep, 3, self.crop_size[0], self.crop_size[1])) partial_captions = np.zeros((self.keep, self.max_caption_len)) next_words = np.zeros((self.keep, self.max_caption_len + 1, len(self.caption_vocabulary))) keys = [] i = 0 caption_range = range(self.max_caption_len + 1) while i < self.keep: # if End of LMDB file start from the begining if not self.lmdb_cursor.next(): self.lmdb_cursor.first() self.passes += 1 self.io.print_info( 'Start of next pass over the training set {0}'.format( self.passes)) # Read a key-value record from the queue try: random_class = np.random.choice(self.vocabulary, 1)[0] # get the class index to build the vector later random_class_index = self.vocabulary.index(random_class) # select random image sample for the sampled class key = str( np.random.choice(self.label_mapping[random_class], 1)[0]) # casting to str is important, because sometimes the key is unicode # look up in LMDB for image-data value = self.lmdb_txn.get(key) arr, cap = self.get_image_label(value) if value is None: continue if len(cap) + 2 > self.max_caption_len: continue caption = [self.START] + list(cap) + [self.END] caption = pad_sequences([caption], maxlen=self.max_caption_len, padding='post', value=self.PAD) images[i, :, :, :] = arr partial_captions[i, :] = caption next_words[i, caption_range, np.hstack((caption, [self.PAD]))] = 1 keys.append(key) i += 1 except Exception as e: self.io.print_info( 'Skipping corrupted invalid images, {0}'.format(e)) return keys, images, partial_captions, next_words def get_triplets(self, queries, crop_size=None, mean_value=None, keep=None): pass def check_passes(self): self.io.print_info('Done {0} passes of lmdb'.format(self.passes)) return self.passes def close(self): self.lmdb_env.close()
class TheanoTester(): def __init__(self, config_file, verbose=False, raw_predictions=False): if not os.path.exists(config_file): print 'Error : could not find config file'.format(config_file) self.init = False return self.tester_config_file = config_file self.verbose = verbose self.raw_predictions = raw_predictions self.read_config() self.io = EmbeddingIO(self.cfgs['log_dir'], 'testing') self.tester_name = self.cfgs['model_name'] self.model_config_file = self.cfgs['model_config_file'] self.init = False self.thresholds = None mapping_keys = [d[0] for d in self.cfgs['mapping_list']] mapping_values = [d[1] for d in self.cfgs['mapping_list']] self.mapping = dict(zip(mapping_keys, mapping_values)) self.softpooling_func = f if self.tester_name not in model_dict.keys(): self.io.print_error( 'Not a valid model type {0} chose among {1}'.format( self.tester_name, model_dict.keys())) self.init = False return if not os.path.exists(self.tester_config_file): self.io.print_error('Could not find configuration file {0}'.format( self.tester_config_file)) self.init = False return self.tester = None self.action_items = {'test': self.run, 'eval': self.eval} self.test_lmdb = LMDBParser_test(self.cfgs['test_file'], log_dir=self.cfgs['log_dir']) self.resize_test = self.cfgs['resize_test'] # self.load_synset_file() # self.test_lmdb = LMDBParser(self.cfgs['test_file'], log_dir=self.cfgs['log_dir']) # self.test_lmdb.set_params_test(self.concepts) def _read_threshold_file(self): if 'threshold_file' in self.cfgs.keys(): try: pfile = open(self.cfgs['threshold_file'], 'r') thresholds = json.load(pfile) pfile.close() self.thresholds = { c: thresholds[c] if c in thresholds.keys() else self.cfgs['global_threshold'] for c in self.concepts } except Exception as err: self.io.print_error( 'Error parsing threshold file {0},{1}'.format( self.cfgs['threshold_file'], err)) def read_config(self): pfile = open(self.tester_config_file) self.cfgs = yaml.load(pfile) pfile.close() def setup(self): self.tester = model_dict[self.tester_name]() self.tester.configure(self.model_config_file) self.tester.define() #import ipdb; ipdb.set_trace() self.model_type = self.tester.model.get_config()['name'] self.im_h, self.im_w = self.tester.cfgs[ 'image_height'], self.tester.cfgs['image_width'] if not self.tester.init: self.io.print_error('Error with model definition') self.init = False return self.io.print_info('{0} Model defined from {1}'.format( self.tester_name, self.model_config_file)) self.compile() if not self.tester.init: self.io.print_error('Error with model compilation') self.init = False return self.io.print_info('{0} Model compiled'.format(self.tester_name)) self.init = True self.load_synset_file() self.load_mean_file() self._read_threshold_file() self.load_confidence_model() self.read_label_embedding() def compile(self): """ This compile version is to be used for testing only. Since its optimized for predictions """ self.output_nodes = self.cfgs['output_node_name']['test'] if self.model_type == 'Sequential': self.inference = K.function( [self.tester.model.get_input(train=False)], [self.tester.model.layers[-1].get_output(train=False)]) elif self.model_type == 'Graph': #import ipdb; ipdb.set_trace() self.inference = K.function( [self.tester.model.get_input(train=False)], [ self.tester.model.nodes[node].get_output(train=False) for node in self.output_nodes ]) else: self.io.print_error( 'Say whaaaat ?! This model type is not supported : {}'.format( self.model_type)) sel.init = False def read_label_embedding(self): self.label_embedding = None self.label_vectors = None if 'label_embedding_file' in self.cfgs.keys(): self.label_embedding = self.io.read_npz_file( self.cfgs['label_embedding_file'], 'matrix') self.label_vectors = self.io.read_npz_file( self.cfgs['label_embedding_file'], 'vectors') self.io.print_info('Read label embedding file {0}'.format( self.cfgs['label_embedding_file'])) def load_confidence_model(self): self.confidence_model = None if 'confidence_model' in self.cfgs.keys(): self.confidence_model = joblib.load(self.cfgs['confidence_model']) #to be used in def get_images_frVisualization(self): self.load_synset_file() GndTr = dict() if not self.init: return indexes = dict() self.test_images = self.io.read_images(self.cfgs['test_file_list']) #filterout images from getty self.io.print_info( 'Removing getty scrapped images from validation data') filter_getty = [ contains(img, 'getty-concept-sourcing') for img in self.test_images ] ind = np.where(np.array(filter_getty) == True)[0] test_images = np.array(self.test_images)[ind] #--------------- for image in test_images: im_basename = os.path.basename(image) im_basename, img_ext = os.path.splitext(im_basename) g = self.test_lmdb.get_labels(im_basename) concepts = np.array(self.concepts) index = np.where(g == 1)[0] GndTr[im_basename] = concepts[index] indexes[im_basename] = index return (test_images, GndTr, indexes) def visualize_image(self, beta, pixels): self.beta = beta self.load_synset_file() flag, predictions, confidence, t = self.classify_images(pixels) #file_paths = [os.path.join(self.cfgs['results_dir'], 'visualization_',str(self.beta)+'_'+im + save_ext) for im in basenames] def run(self): self.load_synset_file() if not self.init: return self.test_images = self.io.read_images(self.cfgs['test_file_list']) image_batches = [ self.test_images[i:i + self.cfgs['batch_size']] for i in range(0, len(self.test_images), self.cfgs['batch_size']) ] self.io.print_info('{0} images split into {1} Batches of {2}'.format( len(self.test_images), len(image_batches), self.cfgs['batch_size'])) for beta in self.cfgs['betas']: self.beta = beta for idx, images in enumerate(image_batches): if not self.cfgs['force']: flag = self.check_jsons(images) if flag: self.io.print_warning( 'Skipping batch {0} of {1}'.format( idx, len(image_batches))) continue pixels, basenames, dummy = self.fetch_images(images) dummy = [] flag, predictions, confidence, t = self.classify_images(pixels) if self.raw_predictions is False: getty_like_filter = [ self.suppress_stop_list(g) for g in predictions ] getty_like_safe = [ self.run_thresholds(g) for g in getty_like_filter ] getty_like_pc = [ self.map_concepts(g) for g in getty_like_safe ] getty_like_public = [ self.resolve_antonyms(g) for g in getty_like_pc ] getty_like_unique = [ self.resolve_duplicates(g) for g in getty_like_public ] predictions_pc = getty_like_unique save_ext = '.json' else: predictions_pc = predictions save_ext = '.npz' if not flag: continue file_paths = [ os.path.join(self.cfgs['results_dir'], str(self.beta) + '_' + im + save_ext) for im in basenames ] if not self.raw_predictions: to_save = [dict({'external': p}) for p in predictions_pc] else: to_save = predictions_pc #import ipdb; ipdb.set_trace() #to_save = [dict({'external': p}) for p in predictions_pc] #to_save=predictions_pc #to_save_id = [dict({'external': id_}) for id_ in ids] assert (len(file_paths) == len(to_save)) self.io.print_info('Done batch {0} of {1} in {2} secs'.format( idx, len(image_batches), t)) #import ipdb; ipdb.set_trace() #import ipdb; ipdb.set_trace() f = map(self.io.save_good_labels, file_paths, to_save) #f = map(self.io.save_good_labels, file_paths, to_save_id) def check_jsons(self, images): basenames = [] for image in images: im_basename = os.path.basename(image) im_basename, _ = os.path.splitext(im_basename) basenames.append(im_basename) flags = [ os.path.exists(os.path.join(self.cfgs['results_dir'], b + '.json')) for b in basenames ] return np.all(flags) def compare_various_betas(self): betas = self.cfgs['betas'] Area_PR = dict() dir_path = os.path.dirname(os.path.realpath(__file__)) #print dir_path self.limit_NoofClasses = None self.load_synset_file() self.load_synset_file_training() self.imgHist_Concepts_inTraindata = [ (np.where(np.array(self.concepts) == i)[0][0]) for i in self.trained_concepts ] #import lmdb #lmdb_env = lmdb.open("./keywording_test_images_labels", readonly=True, lock=False) # lmdb_txn = lmdb_env.begin() # read_label = lmdb_txn.get # label1test=self.test_lmdb.get_labels(read_label,'10085536') # label2test=self.test_lmdb.get_labels(read_label,'10085536') #self.imgname_labels=self.test_lmdb.create_keys_labels_dict() var = dict() for beta in betas: var.update(self.eval_usinglmdb(str(beta))) #import ipdb; ipdb.set_trace() if self.limit_NoofClasses == None: df2 = pd.DataFrame(var, index=self.trained_concepts) else: df2 = pd.DataFrame( var, index=self.trained_concepts[0:self.limit_NoofClasses]) df2.to_csv(dir_path + '/results/' + self.cfgs['results_dir'].split('/')[-2] + '.csv') #import ipdb; ipdb.set_trace() # mAP=[] # mRec90=[] # mRec80=[] # max_f1=[] # for beta in betas: # vAP,v90,v80,vf1=Area_PR[beta] # mAP.append(vAP) # mRec90.append(v90) # mRec80.append(v80) # max_f1.append(vf1) # #print ("mAP:%.4f,mean Recall:%.4f,beta:%s" % (mAP,mRec,str(beta))) # #import ipdb; ipdb.set_trace() # #print(*betas, sep='\t') # print('\t'.join([str(x) for x in betas])) # #print(*mRec80, sep='\t') # print('\t'.join([str(round(x*100,3)) for x in mRec80])) # #print(*mRec90, sep='\t') # print('\t'.join([str(round(x*100,3)) for x in mRec90])) # print('\t'.join([str(round(x*100,3)) for x in mAP])) # print('\t'.join([str(round(x*100,3)) for x in max_f1])) #print(*mAP, sep='\t') #pickle.dump(Area_PR,open('/nas2/praveen/home/rnd-libs/rnd_libs/lib/keras/result_Area_PR','wb')) # def generate_csv_file(self): # betas=self.cfgs['betas'] # Area_PR=dict() # #import lmdb # #lmdb_env = lmdb.open("./keywording_test_images_labels", readonly=True, lock=False) # #import ipdb; ipdb.set_trace() # # lmdb_txn = lmdb_env.begin() # # read_label = lmdb_txn.get # # label1test=self.test_lmdb.get_labels(read_label,'10085536') # # label2test=self.test_lmdb.get_labels(read_label,'10085536') # #self.imgname_labels=self.test_lmdb.create_keys_labels_dict() # for beta in betas: # Area_PR[beta]=self.eval_usinglmdb(str(beta)) # mAP=[] # mRec90=[] # mRec80=[] # max_f1=[] # detections=[] # for beta in betas: # vAP,v90,v80,max_f1,detections=Area_PR[beta] # mAP.append(vAP) # mRec90.append(v90) # mRec80.append(v80) #print ("mAP:%.4f,mean Recall:%.4f,beta:%s" % (mAP,mRec,str(beta))) #import ipdb; ipdb.set_trace() #print(*betas, sep='\t') #import ipdb; ipdb.set_trace() # def compare_various_betas(self): # import ipdb; ipdb.set_trace() # betas=self.cfgs['betas'] # for beta in betas: # self.eval_usinglmdb(str(beta) def eval_usinglmdb(self, beta): #import ipdb; ipdb.set_trace() #import ipdb; ipdb.set_trace() self.test_images = self.io.read_images(self.cfgs['test_file_list']) self.predictions = {} self.gt = {} valid_keys = [] notgoodGTimgs = [] ImgsSmallNumofTags = [] #import ipdb; ipdb.set_trace() for idx, image in enumerate(self.test_images): if idx % 1000 == 0: self.io.print_info('Reading {0}/{1}'.format( idx, len(self.test_images))) im_basename = os.path.basename(image) im_basename, img_ext = os.path.splitext(im_basename) #result_json = os.path.join(self.cfgs['results_dir'], im_basename + '.json') result_json = os.path.join(self.cfgs['results_dir'], beta + '_' + im_basename + '.npz') #gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json') if os.path.exists(result_json): #p, s,ids = self.io.read_prediction_json_withIDs(result_json) p = np.load(result_json)['features'] else: p = [] #g = self.io.read_vector(gt_json) #import ipdb; ipdb.set_trace() g = self.test_lmdb.get_labels(im_basename) #print np.sum(g) #print p.shape if len(p) == 0: print("images with no prediction generated: %s") % im_basename continue if len(g) == 0: #import ipdb; ipdb.set_trace() print("images with no GT data: %s") % im_basename notgoodGTimgs.append(im_basename) continue if len(g) < 7735 and len(g) != 0: print("images with small number of tags: %s") % im_basename ImgsSmallNumofTags.append(im_basename) continue #g_filtered=self.parse_gndTruth_withFilter(ids,g) # if p is [] or g is []: # continue self.predictions[im_basename] = p self.gt[im_basename] = g valid_keys.append(im_basename) # if idx>1000: # break print 'Number of images: with no GT %d and with small no of tags %d' % ( len(notgoodGTimgs), len(ImgsSmallNumofTags)) #import ipdb; ipdb.set_trace() y_test = np.zeros((len(valid_keys), len(self.concepts)), dtype='u1') y_score = np.zeros((len(valid_keys), len(self.concepts))) for i, key in enumerate(valid_keys): y_test[i] = self.gt[key] y_score[i] = self.predictions[key] gt = dict() pred = defaultdict(dict) for i, concept in enumerate(self.concepts): imgname_ind = np.where(y_test[i] == 1) np_array_photo_ids = np.array(valid_keys)[imgname_ind] gt[concept] = list(np_array_photo_ids) #pred[concept]={key: value for key, value in self.predictions.items() if key in list(np_array_photo_ids)} # for key in list(np_array_photo_ids): # pred[concept].update({key: self.predictions[key]}) # pairof_photoid_predictions=self.predictions.items() # for photo_id, predictions in pairof_photoid_predictions: # for concept, score in zip(self.concepts,predictions): # pred[concept][photo_id] = score # #import ipdb; ipdb.set_trace() # thresholds=get_thresholds(pred, self.concepts, gt) # detections=get_detections_per_concept(pred, self.concepts, gt, thresholds) #class_histogram=y_test.sum(axis=0) #import ipdb; ipdb.set_trace() if self.cfgs['percentate_of_classesFrEval'] != None or self.cfgs[ 'percentate_of_classesFrEval'] < 1.0: use_subsetofClsFrEval = True n_classes = len(self.concepts) limit_NoofClasses = int(self.cfgs['percentate_of_classesFrEval'] * n_classes) subset_indices_FrEval = self.imgHist_Concepts_inTraindata[: limit_NoofClasses] self.limit_NoofClasses = limit_NoofClasses #self.concepts=np.array(self.concepts)[indices] #y_test_subset=np.zeros((len(valid_keys),len(self.concepts)),dtype='u1') #y_score_subset=np.zeros((len(valid_keys),len(self.concepts))) #for i,key in enumerate(valid_keys): # y_test_subset[i]=self.gt[key][indices] # y_score_subset[i]=self.predictions[key][indices] #print "Using %d out of %d rare number of classes for evaluations" % (limit_NoofClasses,n_classes) else: use_subsetofClsFrEval = False subset_indices_FrEval = self.imgHist_Concepts_inTraindata #subset_indices_FrEval=range(len(self.concepts)) #if use_subsetofClsFrEval: # y_test=y_test_subset # y_score=y_score_subset #plt.plot(np.sort(temp)[::-1]) #plt.savefig('/nas2/praveen/home/rnd-libs/rnd_libs/lib/keras/intermediate/class_histogram') #fpr = dict() #tpr = dict() recAt90 = [] recAt70 = [] ap = [] names = ['ap', 'recAt70', 'recAt90'] #max_f1=[] for ind in subset_indices_FrEval: #import ipdb; ipdb.set_trace() # if y_test[:, i].sum()==0.0: # continue #fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) ap.append( roundoff( average_precision_score(y_test[:, ind], y_score[:, ind]))) recAt90.append( roundoff( recall_at_specified_precision(y_test[:, ind], y_score[:, ind], specified_precision=0.9))) recAt70.append( roundoff( recall_at_specified_precision(y_test[:, ind], y_score[:, ind], specified_precision=0.7))) #max_f1[self.concepts[ind]] = max_f1_score(y_test[:, ind], y_score[:, ind]) #import ipdb; ipdb.set_trace() #return (np.array(ap.values()).mean(), np.array(recAt90.values()).mean(), np.array(recAt80.values()).mean()) #max_f1=[0.0] #import ipdb; ipdb.set_trace() returndict = dict() for name in names: returndict[beta + '_' + name] = locals()[name] return (returndict) #import ipdb; ipdb.set_trace() # import ipdb; ipdb.set_trace() # list_thres=[self.cfgs['global_threshold']] # for t in list_thres: # self.compute_precision_recall(t) def eval(self): self.load_synset_file() self.test_images = self.io.read_images(self.cfgs['test_file_list']) self.predictions = {} self.gt = {} for idx, image in enumerate(self.test_images): if idx % 1000 == 0: self.io.print_info('Reading {0}/{1}'.format( idx, len(self.test_images))) im_basename = os.path.basename(image) im_basename, _ = os.path.splitext(im_basename) result_json = os.path.join(self.cfgs['results_dir'], im_basename + '.json') gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json') p, s = self.io.read_prediction_json(result_json) g = self.io.read_vector(gt_json) if p is [] or g is []: continue self.predictions[im_basename] = zip(p, s) self.gt[im_basename] = list(set(g).intersection(self.concepts)) for t in [self.cfgs['thresholds']]: self.compute_precision_recall(t) def compute_precision_recall(self, thresh): fps = {} tps = {} fns = {} gt_dict = {} for c in self.concepts: fps[c], tps[c], fns[c], gt_dict[c] = [], [], [], [] for key, value in self.predictions.items(): detections = [v[0] for v in value if v[1] >= thresh] gt = self.gt[key] for g in gt: gt_dict[g].append(key) tp = list(set(gt).intersection(detections)) fp = list(set(detections) - set(gt)) fn = list(set(gt) - set(detections)) for t in tp: tps[t].append(key) for f in fp: fps[f].append(key) for n in fn: fns[n].append(key) for c in self.concepts: if gt_dict[c] == []: self.io.print_info( 'Skipping {0}, no samples in ground-truth'.format(c)) continue p = len(tps[c]) / (len(tps[c]) + len(fps[c]) + 0.00000001) r = len(tps[c]) / (len(gt_dict[c]) + 0.0) f1 = self.compute_f1(p, r) self.io.print_info( 'At {0} : {1} precision {2}, recall {3}, f1-score {4}'.format( c, thresh, p, r, f1)) def compute_f1(self, p, r): return 2 * (p * r) / (p + r + 0.000001) def load_synset_file(self): pfile = open(self.cfgs['synset_file'], 'r') concepts = pfile.readlines() pfile.close() self.concepts = [p.strip() for p in concepts] def load_synset_file_training(self): pfile = open(self.cfgs['synset_file_training'], 'r') concepts = pfile.readlines() pfile.close() self.trained_concepts = [p.strip() for p in concepts] def load_mean_file(self): if not self.cfgs['mean_file'] == 'None': pfile = open(self.cfgs['mean_file'], 'r') m = np.load(pfile) pfile.close() else: m = [[[128.0]], [[128.0]], [[128.0]]] m = np.mean(np.mean(m, axis=1), axis=1) self.mean_pixel_value = np.array([m[0], m[1], m[2]]) self.pixel_scaling = self.cfgs['pixel_scaling'][ 'test'] if 'pixel_scaling' in self.cfgs.keys() else None self.channel_swap = self.cfgs['channel_swap'][ 'test'] if 'channel_swap' in self.cfgs.keys() else None self.image_shuffle = self.cfgs['image_shuffle'][ 'test'] if 'image_shuffle' in self.cfgs.keys() else None def jsonify(self, results): return [[r[0], str(r[1])] for r in results] def map_to_synsets(self, results): results = {r[0]: r[1] for r in results} return [results[c] for c in self.concepts] def parse_predictions(self, p): sort_idx = np.argsort(-p) concepts = [(self.concepts[idx], float(p[idx]), idx) for idx in sort_idx[:self.cfgs['top_k']['test']]] #import ipdb; ipdb.set_trace() return concepts def parse_gndTruth_withFilter(self, ids, p): p = np.array(p) concepts = np.array(self.concepts) filtered_gt = p[ids] filtered_concepts = concepts[ids] gt_ids = np.where(filtered_gt == 1)[0] #sort_idx = np.argsort(-p) #import ipdb; ipdb.set_trace() return filtered_concepts[gt_ids] def parse_gndTruth(self, p): #import ipdb; ipdb.set_trace() gt = np.array(p) concepts = np.array(self.concepts) #filtered_gt=p[ids] #filtered_concepts=concepts[ids] gt_ids = np.where(gt == 1)[0] #sort_idx = np.argsort(-p) #import ipdb; ipdb.set_trace() return concepts[gt_ids] def classify_images(self, images): confidence = [None] if images == []: return False, [(), ()], 0.0 try: tstart = time.time() (predictions, features) = self.inference(images) #import ipdb; ipdb.set_trace() predictions = f(predictions, self.beta)[0] #import ipdb; ipdb.set_trace() #pickle.dump(predictions,open('predictions','wb')) #test=f(predictions,self.beta)[0] #predictions=numpy_tensor_mean(predictions,self.beta,1)[0] #import ipdb; ipdb.set_trace() # if len(predictions.shape) == 4: # predictions=predictions[:,:,0,0] #import ipdb; ipdb.set_trace() if self.label_vectors is not None: predictions /= np.linalg.norm(predictions, axis=1)[:, np.newaxis] predictions = np.dot(predictions, self.label_vectors) if self.confidence_model is not None: confidence = [('Confidence', p) for p in self.confidence_model.predict(features)] tend = time.time() except Exception as err: self.io.print_error('Error processing image, {0}'.format(err)) return False, [(), ()], None, 0.0 # if not self.raw_predictions: # results = [self.parse_predictions(p)[1] for p in predictions] , True, [self.parse_predictions(p)[0] for p in predictions], confidence, '{0:.4f}'.format((tend - tstart)) # else: # results = True, predictions, confidence, '{0:.4f}'.format((tend - tstart)) #results = True, [self.parse_predictions(p) for p in predictions], confidence, '{0:.4f}'.format((tend - tstart)) results = True, predictions, confidence, '{0:.4f}'.format( (tend - tstart)) return results def predict(self, image): image = self.normalize_images([image]) return self.classify_images(image) def normalize_images(self, images): normalize_images = [] for image in images: image *= 255.0 if self.cfgs['square_crop']: if self.verbose: self.io.print_info( 'Yo ! As you like it - Squaring up the image') image = self.square_it_up(image) try: image = cv2.resize(image, (self.im_h, self.im_w)) except Exception as err: self.io.print_error('Could not parse test image {0}'.format(e)) return False, [], 0.0 image -= self.mean_pixel_value image = image.transpose(2, 0, 1) # Channel swap since caffe.io.load_image returns RGB image and training was done with BGR image = image[(2, 1, 0), :, :] normalize_images.append(image) return [np.array(normalize_images)] def scale_image(self, image): # # caffe scales image 0-1 in pixel values we bring it back to 255 as keras likes it that way # # No more caffe no more scaling down to 0-1 in caffe # image *= 255.0 tstart = time.time() if self.cfgs['square_crop']: if self.verbose: self.io.print_info( 'Yo ! As you like it - Squaring up the image') image = self.square_it_up(image) tend = time.time() tstart = time.time() try: if self.resize_test: #Tracer()() #print image.size image = cv2.resize(image, (self.im_h, self.im_w)).astype(np.float32) else: print 'no resize' image = image.astype(np.float32) except Exception as e: self.io.print_error('Could not parse {0}'.format(e)) return None tend = time.time() tstart = time.time() if self.mean_pixel_value is not None: image -= self.mean_pixel_value if self.pixel_scaling is not None: image /= self.pixel_scaling if self.channel_swap is not None: image = image[:, :, self.channel_swap] # H,W,C to C,H,W if self.image_shuffle is not None: image = image.transpose(self.image_shuffle) tend = time.time() return image def fetch_images(self, image_file_names): images = [] basenames = [] tstart = time.time() for idx, image_file_name in enumerate(image_file_names): image_file_name = image_file_name.replace('/nas/', '/nas2/') im_basename = os.path.basename(image_file_name) im_basename, _ = os.path.splitext(im_basename) if not urlparse.urlparse(image_file_name).scheme == "": url_response = urllib.urlopen(image_file_name) if url_response.code == 404: print self.io.print_error( '[Testing] URL error code : {1} for {0}'.format( image_file_name, url_response.code)) continue try: string_buffer = StringIO.StringIO(url_response.read()) image = np.asarray(bytearray(string_buffer.read()), dtype="uint8") image = cv2.imdecode(image, cv2.IMREAD_COLOR) image = image[:, :, (2, 1, 0)] image = image.astype(np.float32) except Exception as err: print self.io.print_error( '[Testing] Error with URL {0} {1}'.format( image_file_name, err)) continue elif os.path.exists(image_file_name): try: fid = open(image_file_name, 'r') stream = fid.read() fid.close() image = Image.open(cStringIO.StringIO(stream)) image = np.array(image) image = image.astype(np.float32) # image = cv2.imread(image_file_name) # image = image[:, :, (2, 1, 0)] # image = image.astype(np.float32) if image.ndim == 2: image = image[:, :, np.newaxis] image = np.tile(image, (1, 1, 3)) elif image.shape[2] == 4: image = image[:, :, :3] except Exception as err: print self.io.print_error( '[Testing] Error with image file {0} {1}'.format( image_file_name, err)) continue else: try: image = self.conn.read_image(image_file_name) if image is None: raise self.io.print_error( "[Testing] Image data could not be loaded - %s" % image_file_name) continue image = np.array(image) if image.ndim == 2: image = image[:, :, np.newaxis] image = np.tile(image, (1, 1, 3)) elif image.shape[2] == 4: image = image[:, :, :3] except Exception as err: print self.io.print_error( '[Testing] Error with S3 Bucket hash {0} {1}'.format( image_file_name, err)) continue # try : do or do not there is not try #orig_image=image.astype('u1') image = self.scale_image(image) if image is None: continue if self.verbose: self.io.print_info('Processing {0} {1}'.format( image_file_name, image.shape)) images.append(image) basenames.append(im_basename) tend = time.time() self.io.print_info('Fetching {0} images took {1:.4f} secs'.format( len(image_file_names), tend - tstart)) #return ([np.array(images)], basenames,orig_images) return ([np.array(images)], basenames) def fetch_gif_frames(self, inGif): # Only works for URLs for now # write frames to disk and pass filenames to fetch_images outFolder = '/tmp' frame = Image.open(BytesIO(urllib.urlopen(inGif).read())) nframes = 0 file_names = [] while frame: frame_name = '{0}/{1}-{2}.png'.format(outFolder, os.path.basename(inGif), nframes) frame.save(frame_name, 'png') file_names.append(frame_name) nframes += 1 try: frame.seek(nframes) except EOFError: break file_names = [file_names[i] for i in range(0, len(file_names), 10)] return file_names def non_max(self, predictions): concepts = {} for pred in predictions: for p in pred: if p[0] in concepts.keys(): concepts[p[0]].append(float(p[1])) else: concepts[p[0]] = [float(p[1])] for key, value in concepts.items(): concepts[key] = np.max(np.array(value)) labels = concepts.keys() scores = [concepts[l] for l in labels] sort_idx = np.argsort(-np.array(scores)) results = [(labels[idx], str(scores[idx])) for idx in sort_idx] return [results] def square_it_up(self, image): edge_size = np.min(image.shape[:2]) cy, cx = np.array(image.shape[:2]) // 2 r_img = image[np.max([0, cy - edge_size // 2]):np.min([cy + edge_size // 2, image.shape[0]]), np.max([0, cx - edge_size // 2]):np.min([cx + edge_size // 2, image.shape[1]]), :] return r_img def resolve_duplicates(self, concepts): seen = set() unique = [] for c in concepts: if c[0] in seen: logger.debug("Suppressing duplicate {}:{}".format(c[0], c[1])) continue seen.add(c[0]) unique.append(c) return unique def filter_concepts(self, predictions): return [p[:self.cfgs['store_top_k']] for p in predictions] def suppress_stop_list(self, predictions): return [c for c in predictions if c[0] not in self.cfgs['stop_list_']] def run_thresholds(self, predictions): return self.threshold_concepts(predictions) def resolve_antonyms(self, human_predictions): conflicting_predictions = [] human_dict = {d[0]: d[1] for d in human_predictions} human_labels = [d[0] for d in human_predictions] human_scores = [d[1] for d in human_predictions] for c1, c2 in self.cfgs['antonym_list']: if not (c1 in human_labels and c2 in human_labels): continue # if s1, s2 = human_dict[c1], human_dict[c2] idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1) logger.debug('Suppressing {0}:{1}'.format(human_labels[idx], human_scores[idx])) del human_labels[idx] del human_scores[idx] # for remove_flag = -1 for idx, group in enumerate(self.cfgs['count_order_list']): _this = np.intersect1d(human_dict.keys(), group) if len(_this) > 0: remove_flag = idx + 1 break # if # for if not remove_flag == len(self.cfgs['count_order_list']): remove_tags = [] for g in self.cfgs['count_order_list'][remove_flag:]: remove_tags.extend(g) # for for t in remove_tags: if t not in human_labels: continue # if ridx = human_labels.index(t) del human_labels[ridx] del human_scores[ridx] # for # if return [(g, s) for g, s in zip(human_labels, human_scores)] # def def map_concepts(self, concepts): mapped_predictions = [(self.mapping[c[0]], c[1]) if c[0] in self.mapping.keys() else c for c in concepts] mapped_predictions = self.conditional_mapping(mapped_predictions) return mapped_predictions # def def conditional_mapping(self, predictions): if 'conditional_list' in self.cfgs.keys(): for condition, mapping in self.cfgs['conditional_list']: concepts = [p[0] for p in predictions] holds = list(set(concepts).intersection(set(condition))) if not len(holds) == 0: predictions = [ p for p in predictions if not p[0] in mapping ] # if # for # if return predictions # def def threshold_concepts(self, predictions): if self.thresholds is not None: predictions = [ p for p in predictions if p[1] >= self.thresholds[p[0]] ] # if return predictions
class LabelEmbedding(object): """ The main/useful class to implement some of the ideas in : https://hal.inria.fr/hal-00815747v1/document Given reasonable amout of data with dense predictions and getty tags we learn a mapping that can align them. arg_max( dense_predictions*W*getty_labels^T ) """ def __init__(self, config_file, force=False, reduced=False): self.config_file = config_file self.init = True self.force = force self._read_configuration() self.init = self._verify_files() self.is_mat = True if not reduced: self.io = EmbeddingIO(self.cfgs['path_logging_dir']) else: self.io = EmbeddingIO(None, True, True) print 'Reduced functionality of Label Embedding' #if self.working_order = [(0, 'run_preprocess', self._run_preprocess), (1, 'run_train', self._run_training), (2, 'build_embedding', self._build_embedding), (3, 'run_thresholds', self._run_thresholds), (4, 'run_test', self._run_testing)] self.train_method = { 'CFM': self._solve_closed_form, 'CFM_PC': self._solve_closed_form_pc, 'SGD': self._solve_sgd, 'WSABIE': self._solve_wsabie, 'CCA': self._solve_cca } lemb_mapping_keys = [d[0] for d in self.cfgs['lemb_mapping_list']] lemb_mapping_values = [d[1] for d in self.cfgs['lemb_mapping_list']] self.lemb_mapping = dict(zip(lemb_mapping_keys, lemb_mapping_values)) #def """ Check if all files and path exists. From yaml config file there are variables beginning with path_ """ def prepare_for_testing(self): self._read_all_labels() if not '_PC' in self.cfgs['train_method']: self.W = self.io.read_weight_matrix( self.cfgs['npz_files_loc']['weights']) self.is_mat = True self.threshold_map, self.black_list = self.io.read_threshold_file( self.cfgs['npz_files_loc']['thresholds']) self.threshold_map = self.threshold_map.tolist() self.io.print_info( '{0}/{1} concepts are black-listed (not enough validation data)' .format(len(self.black_list), len(self.getty_labels))) else: self.W = self.io.read_all_emb( self.getty_labels, self.cfgs['train_method'], self.cfgs['npz_files_loc']['weights']) self.is_mat = False if self.cfgs['use_global_threshold']: self.threshold_map, self.black_list = { c: 0.5 for c in self.W.keys() }, [] else: self.threshold_map, self.black_list = self.io.read_threshold_file( self.cfgs['npz_files_loc']['thresholds']) #if self.not_black_list = self.io.read_getty_labels( self.cfgs['path_getty_label_file'], self.cfgs['minimum_samples_for_validation'], self.cfgs['getty_key_words_types']) #if if self.is_mat: assert self.W.shape[1] == len(self.getty_labels) self.io.print_info('Found weight {0}x{1} matrix'.format( self.W.shape[0], self.W.shape[1])) #if def init_embedding(self, hedger_file, synset_names, getty_file, emb_weights_file, emb_threshold_file): self.all_wn_labels = self.io.read_hedged_labels(hedger_file) self.leaf_wn_labels = synset_names self.getty_labels = self.io.read_getty_labels( getty_file, self.cfgs['getty_stop_list'], self.cfgs['getty_key_words_types']) print('Reading label embedding from {0}'.format(emb_weights_file)) if not '_PC' in self.cfgs['train_method']: self.W = self.io.read_weight_matrix(emb_weights_file) self.is_mat = True self.threshold_map, self.black_list = self.io.read_threshold_file( emb_threshold_file) self.threshold_map = self.threshold_map.tolist() self.io.print_info( '{0}/{1} concepts are black-listed (not enough validation data)' .format(len(self.black_list), len(self.getty_labels))) else: self.W = self.io.read_all_emb(self.getty_labels, self.cfgs['train_method'], emb_weights_file) self.is_mat = False if self.cfgs['use_global_threshold']: self.threshold_map, self.black_list = { c: 0.5 for c in self.W.keys() }, [] else: self.threshold_map, self.black_list = self.io.read_threshold_file( self.cfgs['npz_files_loc']['thresholds']) #if self.not_black_list = self.io.read_getty_labels( getty_file, self.cfgs['minimum_samples_for_validation'], self.cfgs['getty_key_words_types']) #if if self.is_mat: assert self.W.shape[1] == len(self.getty_labels) self.io.print_info('Found weight {0}x{1} matrix'.format( self.W.shape[0], self.W.shape[1])) #if self.io.print_info('Testing set-up for label embedding') #if def _verify_files(self): flag = True paths_to_check = [(k, v) for k, v in self.cfgs.items() if 'path_' in k] for var, file_path in paths_to_check: if type(file_path) == dict: check_files = [(var + '+' + r, j) for r, j in file_path.items()] else: check_files = [(var, file_path)] #if file_flag = [os.path.exists(f[1]) for f in check_files] flag = flag and np.all(file_flag) for idx, f in enumerate(file_flag): if not f: print colored( '[WARN] Could not find {0} for {1}, no such file or directory' .format(check_files[idx][1], check_files[idx][0]), 'blue') #if #if #def return flag #def """ Simple yaml config file reader """ def _read_configuration(self): pfile = open(self.config_file) self.cfgs = yaml.load(pfile) pfile.close() #def """ Read wordnet synset file and getty labels. """ def _read_all_labels(self): self.leaf_wn_labels, self.all_wn_labels = self.io.read_wn_labels( self.cfgs['path_bet_file'], self.cfgs['path_wn_labels_file']) self.getty_labels = self.io.read_getty_labels( self.cfgs['path_getty_label_file'], self.cfgs['getty_stop_list'], self.cfgs['getty_key_words_types']) #def def get_to_work(self, tasks): for idx, action, action_func in self.working_order: if tasks[action]: action_func() #if #for #def """ Read all possible labels wordnet and getty Run converting dense prediction vectors from DL into histogram features. Run converting getty labels into binary vectors (presense or absence) Save processed vectors. """ def _run_preprocess(self): self._read_all_labels() self._read_training_samples() self._process_vectors() #def """ Second step after pre-processing vectors, depending on training method selected used closed form solution or use SGD """ def _run_training(self): if self.cfgs['train_method'] in self.train_method.keys(): self.train_method[self.cfgs['train_method']]() else: self.io.print_error( 'Could not find the specified training method {0}'.format( self.cfgs['train_method'])) #if #def def _build_embedding(self): self._read_all_labels() W = self.io.read_all_emb(self.getty_labels, self.cfgs['train_method'], self.cfgs['path_weight_matrix']) self.io.write_all_emb(self.cfgs['npz_files_loc']['weights'], W) self.io.print_info('Done writing {1} embeddings to {0}'.format( self.cfgs['npz_files_loc']['weights'], len(W.keys()))) #def """ Run testing woudl included and instance of FullConvNet and dense predictions. Mapping predictions using learned weight matrix. Save Mapped predictions. TODO : Check code to reflect changes for individual embeddings """ def _run_thresholds(self): self.prepare_for_testing() self.validation_samples = self.io.read_images( self.cfgs['path_file_list']['val']) D_val, G_val, _ = self._read_all_data(self.validation_samples, 'val') G_val_pred = [ v.predict_proba(D_val)[:, np.where(v.named_steps['train'].classes_ == 1)[0][0]] if type(v) == sklearn.pipeline.Pipeline else v.predict_proba(D_val)[:, np.where(v.classes_ == 1)[0][0]] for k, v in self.W.items() ] G_val_pred = np.array(G_val_pred).transpose() sample_range = np.array(range(1, D_val.shape[0] + 1), dtype=float) concepts_threshold = {} non_trained_concepts = [] for l in range(D_val.shape[1]): gt_l = np.where(G_val[:, l] > 0)[0].tolist() if len(gt_l) < self.cfgs['minimum_samples_for_validation']: self.io.print_warning( 'Skipping {0}, not enough samples {1}'.format( self.getty_labels[l], len(gt_l))) non_trained_concepts.append(self.getty_labels[l]) continue #if sorted_idx = np.argsort(-G_val_pred[:, l]) good_idx = [1 if idx in gt_l else 0 for idx in sorted_idx.tolist()] c_sum = np.cumsum(np.array(good_idx)) / sample_range here = np.where(c_sum > self.cfgs['concept_precision'])[0].tolist() fhere = here[-1] if not here == [] else np.argmax(c_sum) concepts_threshold[self.getty_labels[l]] = G_val_pred[ sorted_idx[fhere], l] self.io.print_info('Concept {0} has threshold {1}'.format( self.getty_labels[l], concepts_threshold[self.getty_labels[l]])) #for self.io.save_threshold_file(self.cfgs['npz_files_loc']['thresholds'], concepts_threshold, non_trained_concepts) #def def predict(self, dense_pred): vec_dense = [] for pred in dense_pred: vec_dense.extend(pred) #for predictions = self.dense_to_labels(vec_dense) predictions = [(p) for p in predictions] return predictions #def def resolve_label_embedding(self, emb_pred): filtered_pred = [ d[0] for d in emb_pred if d[0] not in self.cfgs['stop_list_lemb'] ] mapped_pred = self.map_lemb_concepts(filtered_pred) return mapped_pred #def def map_lemb_concepts(self, concepts): return [ self.lemb_mapping[c] if c in self.lemb_mapping.keys() else c for c in concepts ] #def def resolve_antonyms(self, human_predictions): conflicting_predictions = [] human_dict = {d[0]: d[1] for d in human_predictions} human_labels = [d[0] for d in human_predictions] human_scores = [d[1] for d in human_predictions] for c1, c2 in self.cfgs['antonym_list']: if not (c1 in human_labels and c2 in human_labels): continue #if s1, s2 = human_dict[c1], human_dict[c2] idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1) print 'Supressing {0}:{1}'.format(human_labels[idx], human_scores[idx]) del human_labels[idx] del human_scores[idx] #for remove_flag = -1 for idx, group in enumerate(self.cfgs['count_order_list']): _this = np.intersect1d(human_dict.keys(), group) if len(_this) > 0: remove_flag = idx + 1 break #if #for if not remove_flag == len(self.cfgs['count_order_list']): remove_tags = [] for g in self.cfgs['count_order_list'][remove_flag:]: remove_tags.extend(g) #for for t in remove_tags: if not t in human_labels: continue #if ridx = human_labels.index(t) del human_labels[ridx] del human_scores[ridx] #for #if return [(g, s) for g, s in zip(human_labels, human_scores)] #def def softmax(self, w, t=1.0): e = np.exp(w / t) dist = e / np.sum(e) return dist #def def dense_to_labels(self, vec_dense): if vec_dense == []: return [] #def f_vec_dense = self._build_wn_label_vectors(vec_dense) if self.is_mat: getty_like_vec = np.dot(f_vec_dense, self.W) good = np.argsort(-getty_like_vec) good_labels = [ (self.getty_labels[g], getty_like_vec[g]) for g in good if (not self.getty_labels[g] in self.black_list and self.getty_labels[g] in self.threshold_map.keys()) ] good_labels = [(g, s) for g, s in good_labels if s >= 0][:self.cfgs['top_k_tags']] else: getty_like_vec = [(k, v.predict_proba(f_vec_dense)[0][np.where( v.named_steps['train'].classes_ == 1)[0][0]]) if type(v) == sklearn.pipeline.Pipeline else (k, v.predict_proba(f_vec_dense)[0][np.where( v.classes_ == 1)[0][0]]) for k, v in self.W.items()] good = np.argsort(-np.array([d[1] for d in getty_like_vec])) good_labels = [ getty_like_vec[g] for g in good if getty_like_vec[g][0] in self.not_black_list ] good_labels = [ (g, s) for g, s in good_labels if s > self.threshold_map[g] ][:self.cfgs['top_k_tags']] #if return good_labels #for def _run_testing(self): if not os.path.exists(self.cfgs['npz_files_loc']['weights']): self.io.print_error( 'Could not find label emdbedding matrix file {}'.format( self.cfgs['npz_files_loc']['weights'])) return #if self.prepare_for_testing() self.testing_samples = self.io.read_images( self.cfgs['path_file_list']['test']) self.io.print_info('Will process {0} test images'.format( len(self.testing_samples))) shuffle(self.testing_samples) #self.testing_samples = self.testing_samples[110:120] for idx, sample in enumerate(self.testing_samples): if idx % 1000 == 0: self.io.print_info('Processing {0}/{1} image'.format( idx, len(self.testing_samples))) #if basename = os.path.basename(sample) basename, ext = os.path.splitext(basename) json_file = os.path.join(self.cfgs['path_results_dir']['test'], basename + '.json') if os.path.exists(json_file): if not self.force: self.io.print_warning( 'Skipping {0}, already exists'.format(json_file)) continue #if #if vec_dense = self.io.read_vector( os.path.join(self.cfgs['path_dense_pred_loc']['test'], basename) + self.cfgs['wn_suffix']) vec_getty = self.io.read_vector( os.path.join(self.cfgs['path_getty_labels_loc']['test'], basename) + self.cfgs['getty_suffix']) good_labels = self.dense_to_labels(vec_dense) self.io.save_good_labels(json_file, dict(good_labels)) if self.cfgs['show_predicted_labels']: self.io.print_info('Photo-id {0}'.format(basename)) self.io.print_info('Predicted {0}'.format(good_labels)) self.io.print_warning('GT {0}'.format(vec_getty)) #if #for #def """ Read images pointed to by training and validation file lists """ def _read_training_samples(self): self.training_data = {} self.validation_data = {} self.training_data['samples'] = self.io.read_images( self.cfgs['path_file_list']['train']) self.validation_data['samples'] = self.io.read_images( self.cfgs['path_file_list']['val']) self.io.print_info( 'Found {0} training samples {1} validation samples'.format( len(self.training_data['samples']), len(self.validation_data['samples']))) #def """ Convert dense predictions into vectors Convert getty labels into binary vectors """ def _process_vectors(self): self._label_to_vectors(self.training_data['samples'], 'train') self._label_to_vectors(self.validation_data['samples'], 'val') #def def _label_to_vectors(self, training_images, tag='train'): for idx, sample in enumerate(training_images): if idx % 1000 == 0: self.io.print_info( 'Processing {0}/{1} vectors for {2}, {3}'.format( idx, len(training_images), tag, sample)) #if basename = os.path.basename(sample) basename, ext = os.path.splitext(basename) npz_file = os.path.join(self.cfgs['path_processed_vectors'][tag], basename + '.npz') if (os.path.exists(npz_file) and not self.force): self.io.print_info( 'Skipping {0}, already exists'.format(npz_file)) continue #if vec_dense = self.io.read_vector( os.path.join(self.cfgs['path_dense_pred_loc']['train'], basename) + self.cfgs['wn_suffix']) vec_getty = self.io.read_vector( os.path.join(self.cfgs['path_getty_labels_loc']['train'], basename) + self.cfgs['getty_suffix']) #f_vec_dense = vec_dense f_vec_dense = self._build_wn_label_vectors(vec_dense) f_vec_getty = self._build_getty_label_vectors(vec_getty) self.io.save_npz_file(f_vec_dense, f_vec_getty, npz_file) #for self.io.print_info('Done building {0} vectors'.format(tag)) #def """ Convert wn dense predictions to sparse vectors and L2 normalization. """ def _build_wn_label_vectors(self, svec): if svec == []: return [] #if vec = np.zeros(len(self.leaf_wn_labels), dtype=float) idx = np.array([d[0] for d in svec]) values = np.array([d[1] for d in svec]) for d in svec: vec[d[0]] += d[1] #if if not self.cfgs['normalization']['wn_norm'] == 'None': vec /= (np.linalg.norm(vec) + 0.000001) #if return vec #def """ Convert getty label vectors to binary vectors of presense of absence and L2 normalization. """ def _build_getty_label_vectors(self, svec): if svec == []: return [] #if vec = np.zeros(len(self.getty_labels), dtype=float) idx = np.array([ self.getty_labels.index(d) for d in svec if d in self.getty_labels ]) if idx.size == 0: return [] #if vec[idx] = 1.0 if not self.cfgs['normalization']['getty_norm'] == 'None': vec /= (np.linalg.norm(vec) + 0.000001) #if return vec #def def _init_embedding_matrix(self): self.W = np.zeros((len(self.leaf_wn_labels), len(self.getty_labels))) self.io.print_info('Weight matrix set-up {0}x{1}'.format( self.W.shape[0], self.W.shape[1])) #def def _init_identity_matrix(self): self.I = np.eye(len(self.leaf_wn_labels), len(self.leaf_wn_labels)) #def def _read_all_data(self, image_list, tag): D_all_vec = [] G_all_vec = [] all_files = [] for idx, sample in enumerate(image_list): if idx % 1000 == 0: self.io.print_info( 'Reading {0}/{1} vectors for {3}, {2}'.format( idx, len(image_list), sample, tag)) #if basename = os.path.basename(sample) basename, ext = os.path.splitext(basename) npz_file = os.path.join(self.cfgs['path_processed_vectors'][tag], basename + '.npz') if not os.path.exists(npz_file): self.io.print_warning( 'Skipping {0} no such file'.format(npz_file)) continue #if D, G = self.io.read_npz_file(npz_file) D = D.flatten() if (D.size == 0 or G.size == 0 or not len(D) == self.cfgs['expected_dim']): self.io.print_warning( 'Skipping {0} mis-match of dimensionality'.format( npz_file)) continue #if D_all_vec.append(D) G_all_vec.append(G) all_files.append(basename) #for return np.array(D_all_vec), np.array(G_all_vec), all_files #def """ Different solutions to label embedding Closed form solution (CFM) / SGD / WSABIE / CCA """ def _solve_closed_form(self): self._read_all_labels() self._read_training_samples() #self._init_embedding_matrix() self._init_identity_matrix() D_train, G_train, _ = self._read_all_data( self.training_data['samples'], 'train') if self.cfgs['balance_data']: D_train, G_train = self._balance_training_data(D_train, G_train) #if D_val, G_val, _ = self._read_all_data(self.validation_data['samples'], 'val') train_size = (D_train.nbytes + G_train.nbytes) / (1000.0 * 1024**2) val_size = (D_val.nbytes + G_val.nbytes) / (1000.0 * 1024**2) # TODO : compute expected size of data in memory before reading it LOL # TODO : Future harsimrat will fix this # Yo future Harsimrat here, this is obsolte now. self.io.print_info('Train : {0:2f} GB, val : {1:2f} GB'.format( train_size, val_size)) for l in self.cfgs['lambda']: self.io.print_info( 'Computing closed form solution (CFM) for lambda = {0}'.format( l)) self.W = np.dot( np.linalg.inv( np.dot(D_train.transpose(), D_train) + l * self.I), np.dot(D_train.transpose(), G_train)) error = 1 - self._compute_validation_error_on_tags( self.W, D_val, G_val) self.io.print_info('(CFM) error for lambda = {0} : {1}'.format( l, error)) self.io.save_weight_matrix( self.W, os.path.join(self.cfgs['path_weight_matrix'], 'CFM-{0}.npz'.format(l))) #for #def """ Closed form solution per concepts """ def _solve_closed_form_pc(self): self._read_all_labels() self._read_training_samples() D_train, G_train, train_files = self._read_all_data( self.training_data['samples'], 'train') D_val, G_val, val_files = self._read_all_data( self.validation_data['samples'], 'val') train_size = (D_train.nbytes + G_train.nbytes) / (1000.0 * 1024**2) val_size = (D_val.nbytes + G_val.nbytes) / (1000.0 * 1024**2) self.io.print_info('Train : {0:2f} GB, val : {1:2f} GB'.format( train_size, val_size)) self.reset_pipe_line() for label, label_name in len(self.getty_labels): svm_file_path = os.path.join(self.cfgs['path_weight_matrix'], 'CFM_PC-{0}.pkl'.format(label_name)) if os.path.exists(svm_file_path): if not self.force: self.io.print_warning( 'Skipping {}, already exists'.format(svm_file_path)) continue #if #if tidx = np.sum(G_train[:, label]) vidx = np.sum(G_val[:, label]) if tidx == 0 or vidx == 0: self.io.print_warning( 'Not enough samples for {0}, train:{1} or val :{2}'.format( label_name, int(tidx), int(vidx))) continue #if self.io.print_info( '{1}/{2} Learning embedding for {0} with {3} +samples {4} -ive samples' .format(label_name, label, len(self.getty_labels), np.sum(G_train[:, label]), G_train.shape[0] - np.sum(G_train[:, label]))) self.pipe_line.fit(D_train, G_train[:, label]) error = 1 - self.pipe_line.score(D_val, G_val[:, label]) self.io.print_info('(CFM_PC) error for {1} : {0}'.format( error, label_name)) self.io.save_emb_file(self.pipe_line, svm_file_path) #for #def def reset_pipe_line(self): action_list = [ ('norm', normalizer_dict[self.cfgs['normalization_type']]), ('scale', scaling_dict[self.cfgs['scaling_type']]), ('train', classifier_dict[self.cfgs['classifier_type']]) ] action_list = [(a, b) for a, b in action_list if not b == None] self.pipe_line = Pipeline(action_list) #def def _solve_sgd(self): self.io.print_error('SGD not implemented') #def def _solve_wsabie(self): self.io.print_error('WSABIE not implemented') #fef def _solve_cca(self): self.io.print_error('CCA not implemented') #def """ Validation needs to change since we optimize for tag coverage we should compute intersecton over union of predicted tags + getty tags on validation set """ def _compute_validation_error(self, W, D, G): n_samples = G.shape[0] pG = np.dot(D, W) for i in range(pG.shape[0]): pG[i, :] /= (np.linalg.norm(pG[i, :]) + 0.000001) #for cG = np.dot(pG, G.transpose()) predictions = np.argmax(cG, axis=0) gt_match = np.array(range(n_samples)) good = np.bitwise_xor(predictions, gt_match) correct_match = good == 0 return np.sum(correct_match) / (n_samples + 0.0) #def """ This function needs to change towards what we should optimize currently global predictions of tag is optimized. """ def _compute_validation_error_on_tags(self, W, D, G): pG = np.dot(D, W) predictions = np.zeros(pG.shape) for i in range(pG.shape[0]): pidx = np.argsort(-pG[i, :])[:self.cfgs['top_k_tags']] predictions[i, pidx] = 1 #for gt = np.zeros(G.shape) idx = np.where(G > 0) gt[idx] = 1 good = predictions * gt n_samples = np.sum(predictions) #self.io.print_info('{0} correct out of {1}'.format(np.sum(good),n_samples)) return np.sum(good) / (n_samples + 0.0) #def def _compute_validation_error_on_tags_pc(self, W, D, G): pG = np.dot(D, W) predictions = np.zeros(pG.shape) pidx = np.where(pG > 0)[0] predictions[pidx] = 1 gt = G good = predictions * gt n_samples = np.sum(predictions) # error : (fp + tn)/n_samples return (np.sum(predictions) + np.sum(gt) - 2 * np.sum(good)) / (n_samples + 0.0) #def def _restore_snapshot(self, label_idx): if label_idx == 0: self._init_embedding_matrix() return #if label = labe_idx - 1 weight_matrix_path = os.path.join( self.cfgs['path_weight_matrix'], 'CFM-{1}.npz'.format(l, self.getty_labels[label])) self.W = self.io.read_weight_matrix(weight_matrix_path) #def def _balance_training_data(self, DL_train, G_train): return DL_train_balance, G_train_balance #def #class
class TheanoTesterCaptions(): def __init__(self, config_file, verbose=False, raw_predictions=False): if not os.path.exists(config_file): print 'Error : could not find config file'.format(config_file) self.init = False return self.tester_config_file = config_file self.verbose = verbose self.raw_predictions = raw_predictions self.read_config() self.io = EmbeddingIO(self.cfgs['log_dir'], 'testing') self.tester_name = self.cfgs['model_name'] self.model_config_file = self.cfgs['model_config_file'] self.init = False self.thresholds = None mapping_keys = [d[0] for d in self.cfgs['mapping_list']] mapping_values = [d[1] for d in self.cfgs['mapping_list']] self.mapping = dict(zip(mapping_keys, mapping_values)) if self.tester_name not in model_dict.keys(): self.io.print_error( 'Not a valid model type {0} chose among {1}'.format( self.tester_name, model_dict.keys())) self.init = False return if not os.path.exists(self.tester_config_file): self.io.print_error('Could not find configuration file {0}'.format( self.tester_config_file)) self.init = False return self.tester = None self.action_items = {'test': self.run, 'eval': self.eval} def _read_threshold_file(self): if 'threshold_file' in self.cfgs.keys(): try: pfile = open(self.cfgs['threshold_file'], 'r') thresholds = json.load(pfile) pfile.close() self.thresholds = { c: thresholds[c] if c in thresholds.keys() else self.cfgs['global_threshold'] for c in self.concepts } except Exception as err: self.io.print_error( 'Error parsing threshold file {0},{1}'.format( self.cfgs['threshold_file'], err)) def read_config(self): pfile = open(self.tester_config_file) self.cfgs = yaml.load(pfile) pfile.close() def setup(self): self.tester = model_dict[self.tester_name]() self.tester.configure(self.model_config_file) self.tester.define() self.model_type = self.tester.model.get_config()['name'] self.im_h, self.im_w = self.tester.cfgs[ 'image_height'], self.tester.cfgs['image_width'] if not self.tester.init: self.io.print_error('Error with model definition') self.init = False return self.io.print_info('{0} Model defined from {1}'.format( self.tester_name, self.model_config_file)) self.compile() if not self.tester.init: self.io.print_error('Error with model compilation') self.init = False return self.io.print_info('{0} Model compiled'.format(self.tester_name)) self.init = True self.load_synset_file() self.load_mean_file() self._read_threshold_file() self.load_confidence_model() self.read_label_embedding() def compile(self): """ This compile version is to be used for testing only. Since its optimized for predictions """ self.output_nodes = self.cfgs['output_node_name']['test'] if self.model_type == 'Sequential': self.inference = K.function( [self.tester.model.get_input(train=False)], [self.tester.model.layers[-1].get_output(train=False)]) elif self.model_type == 'Graph': self.inference = K.function( [self.tester.model.get_input(train=False)], [ self.tester.model.nodes[node].get_output(train=False) for node in self.output_nodes ]) else: self.io.print_error( 'Say whaaaat ?! This model type is not supported : {}'.format( self.model_type)) sel.init = False def read_label_embedding(self): self.label_embedding = None self.label_vectors = None if 'label_embedding_file' in self.cfgs.keys(): self.label_embedding = self.io.read_npz_file( self.cfgs['label_embedding_file'], 'matrix') self.label_vectors = self.io.read_npz_file( self.cfgs['label_embedding_file'], 'vectors') self.io.print_info('Read label embedding file {0}'.format( self.cfgs['label_embedding_file'])) def load_confidence_model(self): self.confidence_model = None if 'confidence_model' in self.cfgs.keys(): self.confidence_model = joblib.load(self.cfgs['confidence_model']) def run(self): self.load_synset_file() if not self.init: return self.test_images = self.io.read_images(self.cfgs['test_file_list']) image_batches = [ self.test_images[i:i + self.cfgs['batch_size']] for i in range(0, len(self.test_images), self.cfgs['batch_size']) ] self.io.print_info('{0} images split into {1} Batches of {2}'.format( len(self.test_images), len(image_batches), self.cfgs['batch_size'])) for idx, images in enumerate(image_batches): if not self.cfgs['force']: flag = self.check_jsons(images) if flag: self.io.print_warning('Skipping batch {0} of {1}'.format( idx, len(image_batches))) continue pixels, basenames = self.fetch_images(images) flag, predictions, confidence, t = self.classify_images(pixels) if self.raw_predictions is False: getty_like_filter = [ self.suppress_stop_list(g) for g in predictions ] getty_like_safe = [ self.run_thresholds(g) for g in getty_like_filter ] getty_like_pc = [self.map_concepts(g) for g in getty_like_safe] getty_like_public = [ self.resolve_antonyms(g) for g in getty_like_pc ] getty_like_unique = [ self.resolve_duplicates(g) for g in getty_like_public ] predictions_pc = getty_like_unique save_ext = '.json' else: predictions_pc = predictions save_ext = '.npz' if not flag: continue file_paths = [ os.path.join(self.cfgs['results_dir'], im + save_ext) for im in basenames ] if not self.raw_predictions: to_save = [dict({'external': p}) for p in predictions_pc] else: to_save = predictions_pc assert (len(file_paths) == len(to_save)) self.io.print_info('Done batch {0} of {1} in {2} secs'.format( idx, len(image_batches), t)) f = map(self.io.save_good_labels, file_paths, to_save) def check_jsons(self, images): basenames = [] for image in images: im_basename = os.path.basename(image) im_basename, _ = os.path.splitext(im_basename) basenames.append(im_basename) flags = [ os.path.exists(os.path.join(self.cfgs['results_dir'], b + '.json')) for b in basenames ] return np.all(flags) # # Which measure to keep # Precision/Recall/F1 scores # def eval(self): self.load_synset_file() self.test_images = self.io.read_images(self.cfgs['val_file_list']) self.predictions = {} self.gt = {} for idx, image in enumerate(self.test_images): if idx % 1000 == 0: self.io.print_info('Reading {0}/{1}'.format( idx, len(self.test_images))) im_basename = os.path.basename(image) im_basename, _ = os.path.splitext(im_basename) result_json = os.path.join(self.cfgs['results_dir'], im_basename + '.json') gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json') p, s = self.io.read_prediction_json(result_json) g = self.io.read_vector(gt_json) if p is [] or g is []: continue self.predictions[im_basename] = zip(p, s) self.gt[im_basename] = list(set(g).intersection(self.concepts)) for t in self.cfgs['thresholds']: self.compute_precision_recall(t) def compute_precision_recall(self, thresh): fps = {} tps = {} fns = {} gt_dict = {} for c in self.concepts: fps[c], tps[c], fns[c], gt_dict[c] = [], [], [], [] for key, value in self.predictions.items(): detections = [v[0] for v in value if v[1] >= thresh] gt = self.gt[key] for g in gt: gt_dict[g].append(key) tp = list(set(gt).intersection(detections)) fp = list(set(detections) - set(gt)) fn = list(set(gt) - set(detections)) for t in tp: tps[t].append(key) for f in fp: fps[f].append(key) for n in fn: fns[n].append(key) for c in self.concepts: if gt_dict[c] == []: self.io.print_info( 'Skipping {0}, no samples in ground-truth'.format(c)) continue p = len(tps[c]) / (len(tps[c]) + len(fps[c]) + 0.00000001) r = len(tps[c]) / (len(gt_dict[c]) + 0.0) f1 = self.compute_f1(p, r) self.io.print_info( 'At {0} : {1} precision {2}, recall {3}, f1-score {4}'.format( c, thresh, p, r, f1)) def compute_f1(self, p, r): return 2 * (p * r) / (p + r + 0.000001) def load_synset_file(self): pfile = open(self.cfgs['synset_file'], 'r') concepts = pfile.readlines() pfile.close() self.concepts = [p.strip() for p in concepts] def load_mean_file(self): if not self.cfgs['mean_file'] == 'None': pfile = open(self.cfgs['mean_file'], 'r') m = np.load(pfile) pfile.close() else: m = [[[128.0]], [[128.0]], [[128.0]]] m = np.mean(np.mean(m, axis=1), axis=1) self.mean_pixel_value = np.array([m[0], m[1], m[2]]) self.pixel_scaling = self.cfgs['pixel_scaling'][ 'test'] if 'pixel_scaling' in self.cfgs.keys() else None self.channel_swap = self.cfgs['channel_swap'][ 'test'] if 'channel_swap' in self.cfgs.keys() else None self.image_shuffle = self.cfgs['image_shuffle'][ 'test'] if 'image_shuffle' in self.cfgs.keys() else None def jsonify(self, results): return [[r[0], str(r[1])] for r in results] def map_to_synsets(self, results): results = {r[0]: r[1] for r in results} return [results[c] for c in self.concepts] def parse_predictions(self, p): sort_idx = np.argsort(-p) concepts = [(self.concepts[idx], float(p[idx])) for idx in sort_idx[:self.cfgs['top_k']['test']]] return concepts def classify_images(self, images): confidence = [None] if images == []: return False, [(), ()], 0.0 try: tstart = time.time() (predictions, features) = self.inference(images) if self.label_vectors is not None: predictions /= np.linalg.norm(predictions, axis=1)[:, np.newaxis] predictions = np.dot(predictions, self.label_vectors) if self.confidence_model is not None: confidence = [('Confidence', p) for p in self.confidence_model.predict(features)] tend = time.time() except Exception as err: self.io.print_error('Error processing image, {0}'.format(err)) return False, [(), ()], None, 0.0 if not self.raw_predictions: results = True, [self.parse_predictions(p) for p in predictions ], confidence, '{0:.4f}'.format((tend - tstart)) else: results = True, predictions, confidence, '{0:.4f}'.format( (tend - tstart)) return results def predict(self, image): image = self.normalize_images([image]) return self.classify_images(image) def normalize_images(self, images): normalize_images = [] for image in images: image *= 255.0 if self.cfgs['square_crop']: if self.verbose: self.io.print_info( 'Yo ! As you like it - Squaring up the image') image = self.square_it_up(image) try: image = cv2.resize(image, (self.im_h, self.im_w)) except Exception as err: self.io.print_error('Could not parse test image {0}'.format(e)) return False, [], 0.0 image -= self.mean_pixel_value image = image.transpose(2, 0, 1) # Channel swap since caffe.io.load_image returns RGB image and training was done with BGR image = image[(2, 1, 0), :, :] normalize_images.append(image) return [np.array(normalize_images)] def scale_image(self, image): # # caffe scales image 0-1 in pixel values we bring it back to 255 as keras likes it that way # # No more caffe no more scaling down to 0-1 in caffe # image *= 255.0 tstart = time.time() if self.cfgs['square_crop']: if self.verbose: self.io.print_info( 'Yo ! As you like it - Squaring up the image') image = self.square_it_up(image) tend = time.time() tstart = time.time() try: image = cv2.resize(image, (self.im_h, self.im_w)).astype(np.float32) except Exception as err: self.io.print_error('Could not parse {0}'.format(e)) return None tend = time.time() tstart = time.time() if self.mean_pixel_value is not None: image -= self.mean_pixel_value if self.pixel_scaling is not None: image /= self.pixel_scaling if self.channel_swap is not None: image = image[:, :, self.channel_swap] # H,W,C to C,H,W if self.image_shuffle is not None: image = image.transpose(self.image_shuffle) tend = time.time() return image def fetch_images(self, image_file_names): images = [] basenames = [] tstart = time.time() for idx, image_file_name in enumerate(image_file_names): im_basename = os.path.basename(image_file_name) im_basename, _ = os.path.splitext(im_basename) if not urlparse.urlparse(image_file_name).scheme == "": url_response = urllib.urlopen(image_file_name) if url_response.code == 404: print self.io.print_error( '[Testing] URL error code : {1} for {0}'.format( image_file_name, url_response.code)) continue try: string_buffer = StringIO.StringIO(url_response.read()) image = np.asarray(bytearray(string_buffer.read()), dtype="uint8") image = cv2.imdecode(image, cv2.IMREAD_COLOR) image = image[:, :, (2, 1, 0)] image = image.astype(np.float32) except Exception as err: print self.io.print_error( '[Testing] Error with URL {0} {1}'.format( image_file_name, err)) continue elif os.path.exists(image_file_name): try: fid = open(image_file_name, 'r') stream = fid.read() fid.close() image = Image.open(cStringIO.StringIO(stream)) image = np.array(image) image = image.astype(np.float32) # image = cv2.imread(image_file_name) # image = image[:, :, (2, 1, 0)] # image = image.astype(np.float32) if image.ndim == 2: image = image[:, :, np.newaxis] image = np.tile(image, (1, 1, 3)) elif image.shape[2] == 4: image = image[:, :, :3] except Exception as err: print self.io.print_error( '[Testing] Error with image file {0} {1}'.format( image_file_name, err)) continue else: try: image = self.conn.read_image(image_file_name) if image is None: raise self.io.print_error( "[Testing] Image data could not be loaded - %s" % image_file_name) continue image = np.array(image) if image.ndim == 2: image = image[:, :, np.newaxis] image = np.tile(image, (1, 1, 3)) elif image.shape[2] == 4: image = image[:, :, :3] except Exception as err: print self.io.print_error( '[Testing] Error with S3 Bucket hash {0} {1}'.format( image_file_name, err)) continue # try : do or do not there is not try image = self.scale_image(image) if image is None: continue if self.verbose: self.io.print_info('Processing {0} {1}'.format( image_file_name, image.shape)) images.append(image) basenames.append(im_basename) tend = time.time() self.io.print_info('Fetching {0} images took {1:.4f} secs'.format( len(image_file_names), tend - tstart)) return [np.array(images)], basenames def fetch_gif_frames(self, inGif): # Only works for URLs for now # write frames to disk and pass filenames to fetch_images outFolder = '/tmp' frame = Image.open(BytesIO(urllib.urlopen(inGif).read())) nframes = 0 file_names = [] while frame: frame_name = '{0}/{1}-{2}.png'.format(outFolder, os.path.basename(inGif), nframes) frame.save(frame_name, 'png') file_names.append(frame_name) nframes += 1 try: frame.seek(nframes) except EOFError: break file_names = [file_names[i] for i in range(0, len(file_names), 10)] return file_names def non_max(self, predictions): concepts = {} for pred in predictions: for p in pred: if p[0] in concepts.keys(): concepts[p[0]].append(float(p[1])) else: concepts[p[0]] = [float(p[1])] for key, value in concepts.items(): concepts[key] = np.max(np.array(value)) labels = concepts.keys() scores = [concepts[l] for l in labels] sort_idx = np.argsort(-np.array(scores)) results = [(labels[idx], str(scores[idx])) for idx in sort_idx] return [results] def square_it_up(self, image): edge_size = np.min(image.shape[:2]) cy, cx = np.array(image.shape[:2]) // 2 r_img = image[np.max([0, cy - edge_size // 2]):np.min([cy + edge_size // 2, image.shape[0]]), np.max([0, cx - edge_size // 2]):np.min([cx + edge_size // 2, image.shape[1]]), :] return r_img def resolve_duplicates(self, concepts): seen = set() unique = [] for c in concepts: if c[0] in seen: logger.debug("Suppressing duplicate {}:{}".format(c[0], c[1])) continue seen.add(c[0]) unique.append(c) return unique def filter_concepts(self, predictions): return [p[:self.cfgs['store_top_k']] for p in predictions] def suppress_stop_list(self, predictions): return [c for c in predictions if c[0] not in self.cfgs['stop_list_']] def run_thresholds(self, predictions): return self.threshold_concepts(predictions) def resolve_antonyms(self, human_predictions): conflicting_predictions = [] human_dict = {d[0]: d[1] for d in human_predictions} human_labels = [d[0] for d in human_predictions] human_scores = [d[1] for d in human_predictions] for c1, c2 in self.cfgs['antonym_list']: if not (c1 in human_labels and c2 in human_labels): continue # if s1, s2 = human_dict[c1], human_dict[c2] idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1) logger.debug('Suppressing {0}:{1}'.format(human_labels[idx], human_scores[idx])) del human_labels[idx] del human_scores[idx] # for remove_flag = -1 for idx, group in enumerate(self.cfgs['count_order_list']): _this = np.intersect1d(human_dict.keys(), group) if len(_this) > 0: remove_flag = idx + 1 break # if # for if not remove_flag == len(self.cfgs['count_order_list']): remove_tags = [] for g in self.cfgs['count_order_list'][remove_flag:]: remove_tags.extend(g) # for for t in remove_tags: if t not in human_labels: continue # if ridx = human_labels.index(t) del human_labels[ridx] del human_scores[ridx] # for # if return [(g, s) for g, s in zip(human_labels, human_scores)] # def def map_concepts(self, concepts): mapped_predictions = [(self.mapping[c[0]], c[1]) if c[0] in self.mapping.keys() else c for c in concepts] mapped_predictions = self.conditional_mapping(mapped_predictions) return mapped_predictions # def def conditional_mapping(self, predictions): if 'conditional_list' in self.cfgs.keys(): for condition, mapping in self.cfgs['conditional_list']: concepts = [p[0] for p in predictions] holds = list(set(concepts).intersection(set(condition))) if not len(holds) == 0: predictions = [ p for p in predictions if not p[0] in mapping ] # if # for # if return predictions # def def threshold_concepts(self, predictions): if self.thresholds is not None: predictions = [ p for p in predictions if p[1] >= self.thresholds[p[0]] ] # if return predictions
class AlexHash(): def __init__(self): self.config_file = None self.io = EmbeddingIO(None) self.init = False self.cfgs = None #def def configure(self,config_file): self.config_file = config_file self.init = False if not os.path.exists(self.config_file): self.io.print_error('Could not find config file for VGG-19 {0}'.format(self.config_file)) self.init = False return #if pfile = open(self.config_file,'r') self.cfgs = yaml.load(pfile) pfile.close() self.init = True #def def define(self): # # TODO : get the filter size, number of filters, reception field size from yaml file # try: self.model = Graph() self.model.add_input(name='img', input_shape=(self.cfgs['n_channels'], self.cfgs['image_height'], self.cfgs['image_width'])) # C1 self.model.add_node(Convolution2D(96, 11, 11,subsample=(4,4)),name='conv1',input='img') self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu1',input='conv1') #self.model.add_node(LRN2D(alpha=0.0001, k=1, beta=0.75, n=5),name='norm1', input='relu1') self.model.add_node(MaxPooling2D(pool_size=(3, 3),stride=(2,2)),name='pool1',input='norm1') # C1 # C2 self.model.add_node(ZeroPadding2D((2,2)),name='zpad5',input='pool1') self.model.add_node(Convolution2D(256, 5, 5),name='conv2',input='zpad5') self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu2',input='conv2') #self.model.add_node(LRN2D(alpha=0.0001, k=1, beta=0.75, n=5),name='norm2', input='relu2') self.model.add_node(MaxPooling2D(pool_size=(3, 3),stride=(2,2)),name='pool2',input='norm2') # C2 # C3 self.model.add_node(ZeroPadding2D((1,1)),name='zpad9',input='pool2') self.model.add_node(Convolution2D(384, 3, 3),name='conv3',input='zpad9') self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu3',input='conv3') # C3 # C4 self.model.add_node(ZeroPadding2D((1,1)),name='zpad11',input='relu3') self.model.add_node(Convolution2D(384, 3, 3),name='conv4',input='zpad11') self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu4',input='conv4') # C4 # C5 self.model.add_node(ZeroPadding2D((1,1)),name='zpad13',input='relu4') self.model.add_node(Convolution2D(256, 3, 3),name='conv5',input='zpad13') self.model.add_node(Activation(self.cfgs['conv_non_linearity']),name='relu5',input='conv5') self.model.add_node(MaxPooling2D(pool_size=(3, 3),stride=(2,2)),name='pool5',input='relu5') # C5 # Flatten - flat16 since its the 16th layer errr. self.model.add_node(Flatten(),name='flat16',input='pool5') # Flatten # FC6 self.model.add_node(Dense(4096,init=self.cfgs['fc_init']),name='fc6',input='flat16') self.model.add_node(Activation(self.cfgs['fc_non_linearity']),name='relu6',input='fc6') self.model.add_node(Dropout(self.cfgs['drop_out']),name='drop6',input='relu6') # FC6 # FC7 self.model.add_node(Dense(4096,init=self.cfgs['fc_init']),name='fc7',input='drop6') self.model.add_node(Activation(self.cfgs['fc_non_linearity']),name='relu7',input='fc7') self.model.add_node(Dropout(self.cfgs['drop_out']),name='drop7',input='relu7') # FC7 # Contact fc6, fc7 self.model.add_node(Activation('linear'),name='concat',merge_mode='concat',inputs=['drop6','drop7']) # # Hash Layer self.model.add_node(Dense(self.cfgs['n_bits']),name='hash',input='concat') # Hash layer # Last Aktivation self.model.add_node(Activation(self.cfgs['activation']),name='bits',input='hash') # Totally last Aktivation self.model.add_output(name='output',input='bits') if not self.cfgs['model_weights_file'] == 'None': self.io.print_info('Porting weights from {0}'.format(self.cfgs['model_weights_file'])) self.init_from_this() #if except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True #def def init_from_this(self): weights_file = self.cfgs['model_weights_file'] if weights_file.endswith('.npz'): self.load_weights(weights_file) self.io.print_info('Weights initalized from {0}'.format(weights_file)) #if if weights_file.endswith('.caffemodel'): self.io.print_warning('Loading from caffe model not implemented starting with random weights') #if #def def load_weights(self, filepath): ''' This method does not make use of Graph.set_weights() for backwards compatibility. Has been modified to fit a graphical model ''' # Loads weights from npz file import numpy as np pfile = open(filepath,'r') p = np.load(pfile) params = p['params'].item() pfile.close() for key, node in self.model.nodes.items(): if key in self.cfgs['ignore_nodes']: self.io.print_info('Skipping weights transfer from {0},{1}'.format(key,node.get_config()['name'])) #if if key in params.keys(): w = params[key] node.set_weights(w) self.io.print_info('Transferring weights from {0},{1}'.format(key,node.get_config()['name'])) #if #for #def def compile(self,compile_cfgs): try: sgd = SGD(lr=compile_cfgs['lr'], decay=compile_cfgs['decay'], momentum=compile_cfgs['momentum'], nesterov=True) self.model.compile(loss=loss_functions_dict[compile_cfgs['loss']], optimizer=sgd) except Exception as e: self.io.print_error('Error configuring the model, {0}'.format(e)) self.init = False return #try self.init = True
class TheanoTrainerCaptions(): def __init__(self, config_file, verbose): if not os.path.exists(config_file): print 'Error : could not find config file'.format(config_file) self.init = False return self.trainer_config_file = config_file self.read_config() self.io = EmbeddingIO(self.cfgs['log_dir'], 'training') self.trainer_name = self.cfgs['model_name'] self.model_config_file = self.cfgs['model_config_file'] self.verbose = verbose self.init = False if self.trainer_name not in model_dict.keys(): self.io.print_error( 'Not a valid model type {0} chose among {1}'.format( self.trainer_name, model_dict.keys())) self.init = False return if not os.path.exists(self.trainer_config_file): self.io.print_error('Could not find configuration file {0}'.format( self.trainer_config_file)) self.init = False return self.trainer = None self.train_lmdb = None self.val_lmdb = None self.all_labels = None self.save_configs() def save_configs(self): try: shutil.copy(self.trainer_config_file, self.cfgs['snapshot_dir']) except Exception as err: self.io.print_info('Using copy at destination already {}'.format( self.trainer_config_file)) for var in self.cfgs['save_info']: try: value = self.cfgs[var] shutil.copy(value, self.cfgs['snapshot_dir']) except Exception as e: self.io.print_warning( 'Could not find requested attribute {0}'.format(var, e)) def read_config(self): pfile = open(self.trainer_config_file) self.cfgs = yaml.load(pfile) pfile.close() def setup(self): self.trainer = model_dict[self.trainer_name]() self.trainer.configure(self.model_config_file) self.trainer.define() self.model_type = self.trainer.model.get_config()['name'] self.im_h, self.im_w = self.trainer.cfgs[ 'image_height'], self.trainer.cfgs['image_width'] self.load_mean_file() self.load_synset_file() self.setup_training_data() self.setup_loss_function() if not self.trainer.init: self.io.print_error('Error with model definition') self.init = False return self.io.print_info('{0} Model defined from {1}'.format( self.trainer_name, self.model_config_file)) self.trainer.compile(self.cfgs) self.compile_inference() if not self.trainer.init: self.io.print_error('Error with model complication') self.init = False return self.io.print_info('{0} Model compiled'.format(self.trainer_name)) self.init = True def compile_inference(self): """ This compile version is to be used for testing only. Since its optimized for predictions """ if self.model_type == 'Sequential': self.inference = K.function( self.trainer.model.get_input(train=False), [self.trainer.model.layers[-1].get_output(train=False)]) elif self.model_type == 'Graph': self.inference = K.function( self.trainer.model.get_input(train=False), [ self.trainer.model.nodes[node].get_output(train=False) for node in self.cfgs['output_node_name']['test'] ]) else: self.io.print_error( 'Say whaaaat ?! This model type is not supported : {}'.format( self.model_type)) sel.init = False self.io.print_info('Inference model compiled') def setup_training_data(self): # TODO [HSS]: Reduce the argument lenght of set_params # This bit of set-up is not clean to read self.train_lmdb = LMDBParser(self.cfgs['train_file'], log_dir=self.cfgs['log_dir']) self.val_lmdb = None if 'val_file' in self.cfgs.keys(): self.val_lmdb = LMDBParser(self.cfgs['val_file'], log_dir=self.cfgs['log_dir']) self.label_image_mapping = None if 'label_file' in self.cfgs.keys(): self.label_image_mapping = self.io.read_npz_file( self.cfgs['label_file'], 'info').item() self.io.print_info('Read label/image lookup file {0}'.format( self.cfgs['label_file'])) self.train_lmdb.set_params((self.im_h, self.im_w), self.mean_pixel_value, self.cfgs['train_lmdb_keep'], self.channel_swap, self.pixel_scaling, self.cfgs['label_sampling'], self.concepts, self.label_image_mapping, image_shuffle=self.image_shuffle) self.val_lmdb.set_params((self.im_h, self.im_w), self.mean_pixel_value, self.cfgs['val_lmdb_keep'], self.channel_swap, self.pixel_scaling, 'None', self.concepts, None, image_shuffle=self.image_shuffle) if 'captions' in self.cfgs['model_name']: self.train_lmdb.set_captions_params( self.trainer.LUT, self.trainer.cfgs['max_caption_len'], self.trainer.vocabulary, self.cfgs['label_sampling']) self.val_lmdb.set_captions_params( self.trainer.LUT, self.trainer.cfgs['max_caption_len'], self.trainer.vocabulary, 'captions') def setup_loss_function(self): self.trainer.setup_loss_function(self.train_lmdb.concepts_cdf) def load_mean_file(self): if not self.cfgs['mean_file'] == 'None': pfile = open(self.cfgs['mean_file'], 'r') m = np.load(pfile) pfile.close() m = np.mean(np.mean(m, axis=1), axis=1) self.mean_pixel_value = np.array([m[2], m[1], m[0]], dtype=np.float32) else: self.mean_pixel_value = np.array([128.0, 128.0, 128.0]) self.pixel_scaling = self.cfgs['pixel_scaling'][ 'train'] if 'pixel_scaling' in self.cfgs.keys() else None self.channel_swap = self.cfgs['channel_swap'][ 'train'] if 'channel_swap' in self.cfgs.keys() else None self.image_shuffle = self.cfgs['image_shuffle'][ 'train'] if 'image_shuffle' in self.cfgs.keys() else None def load_synset_file(self): pfile = open(self.cfgs['synset_file'], 'r') concepts = pfile.readlines() pfile.close() self.concepts = [p.strip() for p in concepts] def run(self): if not self.init: self.io.print_error('Error initializing trainer') return self.show_train_info() self.save_model_def() prefix = self.cfgs['snapshot_prefix'] model_name = self.trainer.cfgs['model_weights_file'] # # HSS : Shorter argument list ? # if prefix in model_name: start_iter = int(model_name.split(prefix)[-1].split('.')[0][1:]) num_epocs = start_iter / self.cfgs['epoc_size'] lr_rates = [ self.trainer.update_optimizer(self.cfgs['decay']) for i in range(num_epocs) ] self.io.print_info( 'Learning rate updated for snapshot {},{}'.format( model_name, lr_rates)) else: start_iter = -1 for n_iter in range(start_iter + 1, self.cfgs['maximum_iteration']): self.io.print_info('At iteration {0} of {1}'.format( n_iter, self.cfgs['maximum_iteration'])) if n_iter % self.cfgs['stepsize'] == 0 and not n_iter == 0: self.update_lr() _, images, partial_captions, next_words = self.train_lmdb.get_batch[ self.cfgs['label_sampling']]() train_logs = LogHistory() try: self.fit([images, partial_captions], next_words, batch_size=self.cfgs['batch_size'], nb_epoch=self.cfgs['nb_epocs'], callbacks=[train_logs], verbose=1) except Exception as e: self.io.print_info('Skipped iteration {0}, {1}'.format( n_iter, e)) if n_iter % self.cfgs['display'] == 0 and not n_iter == 0: self.save_train_logs(n_iter, train_logs) if n_iter % self.cfgs['test_interval'] == 0 and not n_iter == 0: self.run_validation(n_iter) if n_iter % self.cfgs['snapshot'] == 0 and not n_iter == 0: self.save_snapshot(n_iter) if n_iter % self.cfgs['epoc_size'] == 0 and not n_iter == 0: last_lr = self.trainer.opt.get_config()['lr'] self.trainer.update_optimizer(self.cfgs['decay']) current_lr = self.trainer.opt.get_config()['lr'] self.io.print_info( '[LR] update : last lr {}, current lr {}'.format( last_lr, current_lr)) self.close_lmdbs() def run_threaded(self): if not self.init: self.io.print_error('Error initializing trainer') return self.show_train_info() self.save_model_def() prefix = self.cfgs['snapshot_prefix'] model_name = self.trainer.cfgs['model_weights_file'] # # HSS : Shorter argument list ? # if prefix in model_name: start_iter = int(model_name.split(prefix)[-1].split('.')[0][1:]) num_epocs = start_iter / self.cfgs['epoc_size'] lr_rates = [ self.trainer.update_optimizer(self.cfgs['decay']) for i in range(num_epocs) ] self.io.print_info( 'Learning rate updated for snapshot {},{}'.format( model_name, lr_rates)) else: start_iter = -1 _, images, partial_captions, next_words = self.train_lmdb.get_batch[ self.cfgs['label_sampling']]() for n_iter in range(start_iter + 1, self.cfgs['maximum_iteration']): self.io.print_info('At iteration {0} of {1}'.format( n_iter, self.cfgs['maximum_iteration'])) if n_iter % self.cfgs['stepsize'] == 0 and not n_iter == 0: self.update_lr() async_result = pool.apply_async( self.train_lmdb.get_batch[self.cfgs['label_sampling']]) train_logs = LogHistory() try: t1 = time() self.fit([images, partial_captions], next_words, batch_size=self.cfgs['batch_size'], nb_epoch=self.cfgs['nb_epocs'], callbacks=[train_logs], verbose=1) t2 = time() - t1 if self.cfgs['logging_for_profiling'] is True: self.io.print_info('fit took {}s'.format(round(t2, 2))) t1 = time() self.eval(n_iter, [images, partial_captions], next_words) t2 = time() - t1 if self.cfgs['logging_for_profiling']: self.io.print_info('eval took {}s'.format(round(t2, 2))) except Exception as e: self.io.print_info('Skipped iteration {0}, {1}'.format( n_iter, e)) if n_iter % self.cfgs['display'] == 0 and not n_iter == 0: self.save_train_logs(n_iter, train_logs) if n_iter % self.cfgs['test_interval'] == 0 and not n_iter == 0: self.run_validation(n_iter) if n_iter % self.cfgs['snapshot'] == 0 and not n_iter == 0: self.save_snapshot(n_iter) if n_iter % self.cfgs['epoc_size'] == 0 and not n_iter == 0: last_lr = self.trainer.opt.get_config()['lr'] self.trainer.update_optimizer(self.cfgs['decay']) current_lr = self.trainer.opt.get_config()['lr'] self.io.print_info( '[LR] update : last lr {}, current lr {}'.format( last_lr, current_lr)) t1 = time() _, images, partial_captions, next_words = async_result.get() t2 = time() - t1 if self.cfgs['logging_for_profiling']: self.io.print_info('waiting for batch took {}s'.format( round(t2, 2))) self.close_lmdbs() def fit(self, input, output, **kwargs): if self.model_type == 'Sequential': self.trainer.model.fit(input, output, **kwargs) elif self.model_type == 'Graph': self.trainer.model.fit({ 'input': input, 'output': output }, **kwargs) else: self.io.print_error( 'Say whaaaat ?! This model type is not supported : {}'.format( self.model_type)) def eval(self, n_iter, input, output, tag='TRAIN'): # NOTE : This is loss not accuracy pred = self.inference(input)[0] pred /= pred.sum(axis=-1, keepdims=True) # avoid numerical instability with _EPSILON clipping pred = np.clip(pred, _EPSILON, 1.0 - _EPSILON) if 'pn' in self.cfgs['loss']: val_loss = -np.sum(np.sum(output * np.log(pred) + (1 - output) * np.log(1 - pred), axis=-1), axis=-1) else: val_loss = -np.sum(np.sum(output * np.log(pred), axis=-1), axis=-1) self.io.print_info('{2} At {0} n_iter {1}'.format( n_iter, np.mean(val_loss), tag)) def update_lr(self): pass def build_k_hot(self, svec, getty_dictionary): if svec == []: return [] vec = np.zeros(len(getty_dictionary), dtype=float) idx = np.array( [getty_dictionary.index(d) for d in svec if d in getty_dictionary]) if idx.size == 0: return [] vec[idx] = 1.0 return vec def show_model_info(self): self.io.print_warning('BEGIN MODEL INFO') model = self.trainer.model if self.model_type == 'Sequential': for idx, layer in enumerate(model.layers): layer_info = layer.get_config() self.io.print_info('Layer {0} : {1}'.format(idx, layer_info)) elif self.model_type == 'Graph': pass self.io.print_warning('END MODEL INFO') def show_train_info(self): self.io.print_warning('BEGIN TRAINING INFO') for var in self.cfgs['train_info']: try: value = self.cfgs[var] self.io.print_info('{0} : {1}'.format(var, value)) except Exception as e: self.io.print_warning( 'Could not find requested attribute {0}'.format(var, e)) self.io.print_warning('END TRAINING INFO') def save_model_def(self): yaml_string = self.trainer.model.to_yaml() model_definition_file = os.path.join(self.cfgs['snapshot_dir'], self.trainer_name + '_def.yaml') pfile = open(model_definition_file, 'w') pfile.write(yaml_string) pfile.close() def save_snapshot(self, n_iter): weights_file = os.path.join( self.cfgs['snapshot_dir'], '{0}_{2}_{1}.hdf5'.format(self.trainer_name, n_iter, self.cfgs['snapshot_prefix'])) self.trainer.model.save_weights(weights_file, overwrite=True) self.io.print_info('Wrote snapshot to {0}'.format(weights_file)) def run_validation(self, n_iter): self.io.print_info('Running validation') _, val_images, val_partial_captions, val_next_words = self.val_lmdb.get_batch[ 'captions']() self.eval(n_iter, [val_images, val_partial_captions], val_next_words, tag='VALIDATION') self.io.print_info('End of validation') def save_train_logs(self, n_iter, train_logs): self.io.print_info('[{1}] TRAINING : loss:{0}'.format( np.mean(train_logs.losses), n_iter)) def save_val_logs(self, n_iter, val_logs): self.io.print_info('[{1}] VALIDATION : loss:{0}'.format( val_logs, n_iter)) def close_lmdbs(self): if self.train_lmdb is not None: self.train_lmdb.close() if self.val_lmdb is not None: self.val_lmdb.close()
class TheanoTrainer(): def __init__(self, config_file, verbose): if not os.path.exists(config_file): print 'Error : could not find config file'.format(config_file) self.init = False return self.trainer_config_file = config_file self.read_config() self.io = EmbeddingIO(self.cfgs['log_dir'], 'training') self.trainer_name = self.cfgs['model_name'] self.model_config_file = self.cfgs['model_config_file'] self.verbose = verbose self.init = False if self.trainer_name not in model_dict.keys(): self.io.print_error('Not a valid model type {0} chose among {1}'.format(self.trainer_name, model_dict.keys())) self.init = False return if not os.path.exists(self.trainer_config_file): self.io.print_error('Could not find configuration file {0}'.format(self.trainer_config_file)) self.init = False return self.trainer = None self.train_lmdb = None self.val_lmdb = None self.all_labels = None self.save_configs() def scale_image(self, image): # # caffe scales image 0-1 in pixel values we bring it back to 255 as keras likes it that way # # No more caffe no more scaling down to 0-1 in caffe # image *= 255.0 #tstart = time.time() if self.cfgs['square_crop']: if self.verbose: self.io.print_info('Yo ! As you like it - Squaring up the image') image = self.square_it_up(image) # tend = time.time() # tstart = time.time() try: if self.cfgs['resize_images']: image = cv2.resize(image, (self.im_h, self.im_w)).astype(np.float32) else: print 'no resize' image=image.astype(np.float32) except Exception as err: self.io.print_error('Could not parse {0}'.format(e)) return None # tend = time.time() # tstart = time.time() if self.mean_pixel_value is not None: image -= self.mean_pixel_value if self.pixel_scaling is not None: image /= self.pixel_scaling if self.channel_swap is not None: image = image[:, :, self.channel_swap] # H,W,C to C,H,W if self.image_shuffle is not None: image = image.transpose(self.image_shuffle) #tend = time.time() return image def fetch_images(self, image_file_names): images = [] basenames = [] #tstart = time.time() for idx, image_file_name in enumerate(image_file_names): image_file_name=image_file_name.replace('/nas/','/nas2/') im_basename = os.path.basename(image_file_name) im_basename, _ = os.path.splitext(im_basename) if not urlparse.urlparse(image_file_name).scheme == "": url_response = urllib.urlopen(image_file_name) if url_response.code == 404: print self.io.print_error('[Training] URL error code : {1} for {0}'.format(image_file_name, url_response.code)) continue try: string_buffer = StringIO.StringIO(url_response.read()) image = np.asarray(bytearray(string_buffer.read()), dtype="uint8") image = cv2.imdecode(image, cv2.IMREAD_COLOR) image = image[:, :, (2, 1, 0)] image = image.astype(np.float32) except Exception as err: print self.io.print_error('[Training] Error with URL {0} {1}'.format(image_file_name, err)) continue elif os.path.exists(image_file_name): try: fid = open(image_file_name, 'r') stream = fid.read() fid.close() image = Image.open(cStringIO.StringIO(stream)) image = np.array(image) image = image.astype(np.float32) # image = cv2.imread(image_file_name) # image = image[:, :, (2, 1, 0)] # image = image.astype(np.float32) if image.ndim == 2: image = image[:, :, np.newaxis] image = np.tile(image, (1, 1, 3)) elif image.shape[2] == 4: image = image[:, :, :3] except Exception as err: print self.io.print_error('[Training] Error with image file {0} {1}'.format(image_file_name, err)) continue else: try: image = self.conn.read_image(image_file_name) if image is None: raise self.io.print_error("[Training] Image data could not be loaded - %s" % image_file_name) continue image = np.array(image) if image.ndim == 2: image = image[:, :, np.newaxis] image = np.tile(image, (1, 1, 3)) elif image.shape[2] == 4: image = image[:, :, :3] except Exception as err: print self.io.print_error('[Training] Error with S3 Bucket hash {0} {1}'.format(image_file_name, err)) continue # try : do or do not there is not try #orig_image=image.astype('u1') image = self.scale_image(image) if image is None: continue if self.verbose: self.io.print_info('Processing {0} {1}'.format(image_file_name, image.shape)) images.append(image) basenames.append(im_basename) #tend = time.time() #self.io.print_info('Fetching {0} images took {1:.4f} secs'.format(len(image_file_names), tend - tstart)) if self.cfgs['resize_images']: return np.array(images) else: return images def save_configs(self): try: shutil.copy(self.trainer_config_file, self.cfgs['snapshot_dir']) except Exception as err: self.io.print_info('Using copy at destination already {}'.format(self.trainer_config_file)) for var in self.cfgs['save_info']: try: value = self.cfgs[var] shutil.copy(value, self.cfgs['snapshot_dir']) except Exception as e: self.io.print_warning('Could not find requested attribute {0}'.format(var, e)) def read_config(self): pfile = open(self.trainer_config_file) self.cfgs = yaml.load(pfile) pfile.close() def see_model(self): self.trainer = model_dict[self.trainer_name]() self.trainer.configure(self.model_config_file) self.trainer.define() self.load_mean_file() self.im_h=779 self.im_w=779 self.load_synset_file() self.trainer.compile(self.cfgs) def setup(self): self.trainer = model_dict[self.trainer_name]() self.trainer.configure(self.model_config_file) self.trainer.define() self.model_type = self.trainer.model.get_config()['name'] self.im_h, self.im_w = self.trainer.cfgs['image_height'], self.trainer.cfgs['image_width'] self.load_mean_file() self.load_synset_file() self.setup_training_data() self.setup_loss_function() if not self.trainer.init: self.io.print_error('Error with model definition') self.init = False return self.io.print_info('{0} Model defined from {1}'.format(self.trainer_name, self.model_config_file)) self.trainer.compile(self.cfgs) self.compile_inference() if not self.trainer.init: self.io.print_error('Error with model complication') self.init = False return self.io.print_info('{0} Model compiled'.format(self.trainer_name)) self.init = True def compile_inference(self): """ This compile version is to be used for testing only. Since its optimized for predictions """ if self.model_type == 'Sequential': self.inference = K.function([self.trainer.model.get_input(train=False)], [self.trainer.model.layers[-1].get_output(train=False)]) elif self.model_type == 'Graph': self.inference = K.function([self.trainer.model.get_input(train=False)], [self.trainer.model.nodes[node].get_output(train=False) for node in self.cfgs['output_node_name']['test']]) else: self.io.print_error('Say whaaaat ?! This model type is not supported : {}'.format(self.model_type)) sel.init = False self.io.print_info('Inference model compiled') def setup_training_data(self): # TODO [HSS]: Reduce the argument lenght of set_params # This bit of set-up is not clean to read self.train_lmdb = LMDBParser(self.cfgs['train_file'], log_dir=self.cfgs['log_dir']) self.val_lmdb = None if 'val_file' in self.cfgs.keys(): self.val_lmdb = LMDBParser(self.cfgs['val_file'], log_dir=self.cfgs['log_dir']) self.label_image_mapping = None self.label_embedding = None self.label_vectors = None # if 'label_file' in self.cfgs.keys(): # import ipdb; ipdb.set_trace() # self.label_image_mapping = self.io.read_npz_file(self.cfgs['label_file'], 'info').item() # #self.label_image_mapping =None # self.io.print_info('Read label/image lookup file {0}'.format(self.cfgs['label_file'])) if 'label_file' in self.cfgs.keys(): images, mapping, vocab, vocab_mapping = self.io.read_npz_file(self.cfgs['label_file'], ['images', 'mapping', 'vocabulary', 'vocabulary_mapping']) self.label_image_mapping = {'images': list(images), 'mapping': list(mapping), 'vocabulary': list(vocab), 'vocabulary_mapping': vocab_mapping.item()} self.io.print_info('Read label/image lookup file {0}'.format(self.cfgs['label_file'])) if 'label_embedding_file' in self.cfgs.keys(): self.label_embedding = self.io.read_npz_file(self.cfgs['label_embedding_file'], 'matrix') self.label_vectors = self.io.read_npz_file(self.cfgs['label_embedding_file'], 'vectors') self.io.print_info('Read label embedding file {0}'.format(self.cfgs['label_embedding_file'])) self.train_lmdb.set_params((self.im_h, self.im_w), self.mean_pixel_value, self.cfgs['train_lmdb_keep'], self.channel_swap, self.pixel_scaling, self.cfgs['label_sampling'], self.concepts, self.label_image_mapping, image_shuffle=self.image_shuffle, label_embedding=self.label_embedding) self.val_lmdb.set_params((self.im_h, self.im_w), self.mean_pixel_value, self.cfgs['val_lmdb_keep'], self.channel_swap, self.pixel_scaling, 'None', self.concepts, None, image_shuffle=self.image_shuffle, label_embedding=self.label_embedding) if 'captions' in self.cfgs['model_name']: self.train_lmdb.set_captions_params(self.trainer.LUT, self.trainer.cfgs['max_caption_len'], self.trainer.vocabulary, self.cfgs['label_sampling']) self.val_lmdb.set_captions_params(self.trainer.LUT, self.trainer.cfgs['max_caption_len'], self.trainer.vocabulary, 'captions') def setup_loss_function(self): self.trainer.setup_loss_function(self.train_lmdb.concepts_cdf) def load_mean_file(self): if not self.cfgs['mean_file'] == 'None': pfile = open(self.cfgs['mean_file'], 'r') m = np.load(pfile) pfile.close() m = np.mean(np.mean(m, axis=1), axis=1) self.mean_pixel_value = np.array([m[2], m[1], m[0]], dtype=np.float32) else: self.mean_pixel_value = np.array([128.0, 128.0, 128.0]) self.pixel_scaling = self.cfgs['pixel_scaling']['train'] if 'pixel_scaling' in self.cfgs.keys() else None self.channel_swap = self.cfgs['channel_swap']['train'] if 'channel_swap' in self.cfgs.keys() else None self.image_shuffle = self.cfgs['image_shuffle']['train'] if 'image_shuffle' in self.cfgs.keys() else None def load_synset_file(self): pfile = open(self.cfgs['synset_file'], 'r') concepts = pfile.readlines() pfile.close() self.concepts = [p.strip() for p in concepts] def run(self): if not self.init: self.io.print_error('Error initializing trainer') return self.show_train_info() self.save_model_def() prefix = self.cfgs['snapshot_prefix'] model_name = self.trainer.cfgs['model_weights_file'] # # HSS : Shorter argument list ? # if prefix in model_name: start_iter = int(model_name.split(prefix)[-1].split('.')[0][1:]) num_epocs = start_iter / self.cfgs['epoc_size'] lr_rates = [self.trainer.update_optimizer(self.cfgs['decay']) for i in range(num_epocs)] self.io.print_info('Learning rate updated for snapshot {},{}'.format(model_name, lr_rates)) else: start_iter = -1 #import ipdb; ipdb.set_trace() for n_iter in range(start_iter + 1, self.cfgs['maximum_iteration']): self.io.print_info('At iteration {0} of {1}'.format(n_iter, self.cfgs['maximum_iteration'])) if n_iter % self.cfgs['stepsize'] == 0 and not n_iter == 0: self.update_lr() file_names, images, labels_train, labels_test = self.train_lmdb.get_batch[self.cfgs['label_sampling']]() orig_images=self.fetch_images(file_names) train_logs = LogHistory() try: # if self.cfgs['resize_images'] == False: # for img,lab_train in zip(orig_images,labels_train): # self.fit([img], [lab_train], batch_size=self.cfgs['batch_size'], nb_epoch=self.cfgs['nb_epocs'], callbacks=[train_logs], verbose=1) # else: self.fit(orig_images, labels_train, batch_size=self.cfgs['batch_size'], nb_epoch=self.cfgs['nb_epocs'], callbacks=[train_logs], verbose=1) self.eval(n_iter, orig_images, labels_test) except Exception as e: self.io.print_info('Skipped iteration {0}, {1}'.format(n_iter, e)) if n_iter % self.cfgs['display'] == 0 and not n_iter == 0: self.save_train_logs(n_iter, train_logs) if n_iter % self.cfgs['test_interval'] == 0 and not n_iter == 0: self.run_validation(n_iter) if n_iter % self.cfgs['snapshot'] == 0 and not n_iter == 0: self.save_snapshot(n_iter) if n_iter % self.cfgs['epoc_size'] == 0 and not n_iter == 0: last_lr = self.trainer.opt.get_config()['lr'] self.trainer.update_optimizer(self.cfgs['decay']) current_lr = self.trainer.opt.get_config()['lr'] self.io.print_info('[LR] update : last lr {}, current lr {}'.format(last_lr, current_lr)) self.close_lmdbs() def run_threaded(self): if not self.init: self.io.print_error('Error initializing trainer') return self.show_train_info() self.save_model_def() prefix = self.cfgs['snapshot_prefix'] model_name = self.trainer.cfgs['model_weights_file'] # # HSS : Shorter argument list ? # if prefix in model_name: start_iter = int(model_name.split(prefix)[-1].split('.')[0][1:]) num_epocs = start_iter / self.cfgs['epoc_size'] lr_rates = [self.trainer.update_optimizer(self.cfgs['decay']) for i in range(num_epocs)] self.io.print_info('Learning rate updated for snapshot {},{}'.format(model_name, lr_rates)) else: start_iter = -1 _, images, labels_train, labels_test = self.train_lmdb.get_batch[self.cfgs['label_sampling']]() for n_iter in range(start_iter + 1, self.cfgs['maximum_iteration']): self.io.print_info('At iteration {0} of {1}'.format(n_iter, self.cfgs['maximum_iteration'])) if n_iter % self.cfgs['stepsize'] == 0 and not n_iter == 0: self.update_lr() async_result = pool.apply_async(self.train_lmdb.get_batch[self.cfgs['label_sampling']]) train_logs = LogHistory() try: t1 = time() self.fit(images, labels_train, batch_size=self.cfgs['batch_size'], nb_epoch=self.cfgs['nb_epocs'], callbacks=[train_logs], verbose=1) t2 = time() - t1 if self.cfgs['logging_for_profiling'] is True: self.io.print_info('fit took {}s'.format(round(t2, 2))) t1 = time() self.eval(n_iter, images, labels_test) t2 = time() - t1 if self.cfgs['logging_for_profiling']: self.io.print_info('eval took {}s'.format(round(t2, 2))) except Exception as e: self.io.print_info('Skipped iteration {0}, {1}'.format(n_iter, e)) if n_iter % self.cfgs['display'] == 0 and not n_iter == 0: self.save_train_logs(n_iter, train_logs) if n_iter % self.cfgs['test_interval'] == 0 and not n_iter == 0: self.run_validation(n_iter) if n_iter % self.cfgs['snapshot'] == 0 and not n_iter == 0: self.save_snapshot(n_iter) if n_iter % self.cfgs['epoc_size'] == 0 and not n_iter == 0: last_lr = self.trainer.opt.get_config()['lr'] self.trainer.update_optimizer(self.cfgs['decay']) current_lr = self.trainer.opt.get_config()['lr'] self.io.print_info('[LR] update : last lr {}, current lr {}'.format(last_lr, current_lr)) t1 = time() _, images, labels_train, labels_test = async_result.get() t2 = time() - t1 if self.cfgs['logging_for_profiling']: self.io.print_info('waiting for batch took {}s'.format(round(t2, 2))) self.close_lmdbs() def fit(self, images, labels, **kwargs): if self.model_type == 'Sequential': self.trainer.model.fit(images, labels, **kwargs) elif self.model_type == 'Graph': self.trainer.model.fit({'input': images, 'output': labels}, **kwargs) else: self.io.print_error('Say whaaaat ?! This model type is not supported : {}'.format(self.model_type)) def eval(self, n_iter, images, labels, tag='TRAIN'): img_batches = [images[i:i + self.cfgs['batch_size']] for i in range(0, len(images), self.cfgs['batch_size'])] img_labels = [labels[i:i + self.cfgs['batch_size']] for i in range(0, len(labels), self.cfgs['batch_size'])] acc = 0.0 for img, lab in zip(img_batches, img_labels): (pred, feat) = self.inference([img]) """ if self.model_type == 'Sequential': pred = self.trainer.model.predict(img) elif self.model_type == 'Graph': pred = self.trainer.model.predict({'input':img})[self.cfgs['output_node_name']['train']] else: self.io.print_error('Say whaaaat ?! This model type is not supported : {}'.format(self.model_type)) #if """ acc += self.compute_accuracy(pred, lab) acc /= len(img_batches) self.io.print_info('{3} At {0} n_iter {2} {1}'.format(n_iter, acc, self.cfgs['metric'], tag)) def compute_accuracy(self, predictions, gt): if self.label_vectors is not None: # Need for L2 normalization of predictions and if need self.label_vectors predictions /= np.linalg.norm(predictions, axis=1)[:, np.newaxis] predictions = np.dot(predictions, self.label_vectors) pred_labels = np.zeros(predictions.shape) if self.cfgs['metric'] in ['precision']: pred_labels[np.where(predictions > self.cfgs['global_threshold'])] = 1.0 elif self.cfgs['metric'] in ['top-k accuracy']: good_labels = np.argsort(-predictions, axis=1)[:, :self.cfgs['top_k']['train']] for idx in range(pred_labels.shape[0]): pred_labels[idx, good_labels[idx]] = 1 else: self.io.print_warning('Accuracy not set for {0}'.format(self.cfgs['loss'])) return 0.0 g = pred_labels * gt + 0.0 acc = np.nan_to_num(g.sum(axis=1) / (pred_labels.sum(axis=1))).mean() return acc def update_lr(self): pass def build_k_hot(self, svec, getty_dictionary): if svec == []: return [] vec = np.zeros(len(getty_dictionary), dtype=float) idx = np.array([getty_dictionary.index(d) for d in svec if d in getty_dictionary]) if idx.size == 0: return [] vec[idx] = 1.0 return vec def show_model_info(self): self.io.print_warning('BEGIN MODEL INFO') model = self.trainer.model if self.model_type == 'Sequential': for idx, layer in enumerate(model.layers): layer_info = layer.get_config() self.io.print_info('Layer {0} : {1}'.format(idx, layer_info)) elif self.model_type == 'Graph': pass self.io.print_warning('END MODEL INFO') def show_train_info(self): self.io.print_warning('BEGIN TRAINING INFO') for var in self.cfgs['train_info']: try: value = self.cfgs[var] self.io.print_info('{0} : {1}'.format(var, value)) except Exception as e: self.io.print_warning('Could not find requested attribute {0}'.format(var, e)) self.io.print_warning('END TRAINING INFO') def save_model_def(self): yaml_string = self.trainer.model.to_yaml() model_definition_file = os.path.join(self.cfgs['snapshot_dir'], self.trainer_name + '_def.yaml') pfile = open(model_definition_file, 'w') pfile.write(yaml_string) pfile.close() def save_snapshot(self, n_iter): weights_file = os.path.join(self.cfgs['snapshot_dir'], '{0}_{2}_{1}.hdf5'.format(self.trainer_name, n_iter, self.cfgs['snapshot_prefix'])) self.trainer.model.save_weights(weights_file, overwrite=True) self.io.print_info('Wrote snapshot to {0}'.format(weights_file)) def run_validation(self, n_iter): self.io.print_info('Running validation') keys, val_images, _, val_labels = self.val_lmdb.get_batch['None']() """ if self.model_type == 'Sequential': val_logs = self.trainer.model.evaluate(val_images, val_labels, batch_size=1) elif self.model_type == 'Graph': val_logs = self.trainer.model.evaluate({'input':val_images,self.cfgs['output_node_name']['train']:val_labels}, batch_size=1) else: self.io.print_error('Say whaaaat ?! This model type is not supported : {}'.format(self.model_type)) #if """ self.eval(n_iter, val_images, val_labels, tag='VALIDATION') self.io.print_info('End of validation') def save_train_logs(self, n_iter, train_logs): self.io.print_info('[{1}] TRAINING : loss:{0}'.format(np.mean(train_logs.losses), n_iter)) def save_val_logs(self, n_iter, val_logs): self.io.print_info('[{1}] VALIDATION : loss:{0}'.format(val_logs, n_iter)) def close_lmdbs(self): if self.train_lmdb is not None: self.train_lmdb.close() if self.val_lmdb is not None: self.val_lmdb.close()