class ConvNetFeatureExtractor(FeatureExtractor): def __init__( self, feature_layer='fc7_cudanet_out', pretrained_params='imagenet.decafnet.epoch90', pretrained_meta='imagenet.decafnet.meta', center_only=True ): """ :param feature_layer: The ConvNet layer that's used for feature extraction. Defaults to `fc7_cudanet_out`. A description of all available layers for the ImageNet-1k-pretrained ConvNet is found in the DeCAF wiki. They are: - `pool5_cudanet_out` - `fc6_cudanet_out` - `fc6_neuron_cudanet_out` - `fc7_cudanet_out` - `fc7_neuron_cudanet_out` :param pretrained_params: This must point to the file with the pretrained parameters. Defaults to `imagenet.decafnet.epoch90`. For the ImageNet-1k-pretrained ConvNet this file can be obtained from here: http://www.eecs.berkeley.edu/~jiayq/decaf_pretrained/ :param pretrained_meta: Similar to `pretrained_params`, this must file to the file with the pretrained parameters' metadata. Defaults to `imagenet.decafnet.meta`. :param center_only: Use the center patch of the image only when extracting features. If `False`, use four corners, the image center and flipped variants and average a total of 10 feature vectors, which will usually yield better results. Defaults to `True`. """ super(ConvNetFeatureExtractor, self).__init__() self.feature_layer = feature_layer self.pretrained_params = pretrained_params self.pretrained_meta = pretrained_meta self.center_only = center_only self.convnet = DecafNet( self.pretrained_params, self.pretrained_meta ) def _extract(self, img): """ :param cv2 image :return: np.array with shape (4096,) """ img = self.convnet.oversample(img, center_only=self.center_only) self.convnet.classify_direct(img) feat = self.convnet.feature(self.feature_layer) if not self.center_only: feat = feat.mean(0) return feat[0]
class NetworkDecaf(Network): """ Implementation for the Decaf library. """ def __init__(self, model_spec_filename, model_filename=None,\ wnid_words_filename=None, center_only=False, wnid_subset = []): """ *** PRIVATE CONSTRUCTOR *** """ # the following is just an hack to allow retro-compatibility # with existing code if isinstance(model_spec_filename, NetworkDecafParams): params = model_spec_filename model_spec_filename = params.model_spec_filename model_filename = params.model_filename wnid_words_filename = params.wnid_words_filename center_only = params.center_only wnid_subset = params.wnid_subset if wnid_subset != []: print 'Warning: subset of labels not supported yet' else: assert isinstance(model_spec_filename, str) assert model_filename != None assert wnid_words_filename != None # load Decaf model self.net_ = DecafNet(model_filename, model_spec_filename) self.center_only_ = center_only # build a dictionary label --> description self.dict_label_desc_ = {} dict_desc_label = {} fd = open(wnid_words_filename) for line in fd: temp = line.strip().split('\t') wnid = temp[1].strip() self.dict_label_desc_[wnid] = temp[2].strip() dict_desc_label[temp[2].split(',')[0]] = wnid fd.close() # build a dictionary label --> label_id self.dict_label_id_ = {} self.labels_ = [] for i, desc in enumerate(self.net_.label_names): self.dict_label_id_[dict_desc_label[desc]] = i self.labels_.append(dict_desc_label[desc]) # Load the mean vector from file # mean of 3 channels self.net_.mean_img = np.mean(np.mean(self.net_._data_mean, axis=1), axis=0) # it is in BGR convert in RGB #self.net_.mean_img = self.net_.mean_img[::-1] def get_mean_img(self): return self.net_.mean_img def get_input_dim(self): return decaf.scripts.imagenet.INPUT_DIM def get_label_id(self, label): return self.dict_label_id_[label] def get_label_desc(self, label): return self.dict_label_desc_[label] def get_labels(self): return self.labels_ def evaluate(self, img, layer_name='softmax'): # for now only center_only is supported assert self.center_only_ == True # first, extract the 227x227 center dim = decaf.scripts.imagenet.INPUT_DIM image = util.crop_image_center(decaf.util.transform.as_rgb(img)) image = skimage.transform.resize(image, (dim, dim)) # convert to [0,255] float32 image = image.astype(np.float32) * 255. assert np.max(image) <= 255 # Flip the image if necessary, maintaining the c_contiguous order if decaf.scripts.imagenet._JEFFNET_FLIP: image = image[::-1, :].copy() # subtract the mean, cropping the 256x256 mean image xoff = (self.net_._data_mean.shape[1] - dim) / 2 yoff = (self.net_._data_mean.shape[0] - dim) / 2 image -= self.net_._data_mean[yoff + yoff + dim, xoff:xoff + dim] # make sure the data in contiguous in memory images = np.ascontiguousarray(image[np.newaxis], dtype=np.float32) # classify predictions = self.net_.classify_direct(images) scores = predictions.mean(0) # look at the particular layer if layer_name == 'softmax': return scores elif layer_name == 'fc7_relu': layer_name = 'fc7_neuron_cudanet_out' elif layer_name == 'fc7': layer_name = 'fc7_cudanet_out' elif layer_name == 'fc6_relu': layer_name = 'fc6_neuron_cudanet_out' elif layer_name == 'fc6': layer_name = 'fc6_cudanet_out' elif layer_name == 'pool5': layer_name = 'pool5_cudanet_out' else: raise ValueError('layer_name not supported') return self.net_.feature(layer_name)
class ConvNetFeatures(BaseEstimator): """Extract features from images using a pretrained ConvNet. Based on Yangqing Jia and Jeff Donahue's `DeCAF <https://github.com/UCB-ICSI-Vision-Group/decaf-release/wiki>`_. Please make sure you read and accept DeCAF's license before you use this class. If ``classify_direct=False``, expects its input X to be a list of image filenames or arrays as produced by `np.array(Image.open(filename))`. """ verbose = 0 def __init__( self, feature_layer='fc7_cudanet_out', pretrained_params='imagenet.decafnet.epoch90', pretrained_meta='imagenet.decafnet.meta', center_only=True, classify_direct=False, verbose=0, ): """ :param feature_layer: The ConvNet layer that's used for feature extraction. Defaults to `fc7_cudanet_out`. A description of all available layers for the ImageNet-1k-pretrained ConvNet is found in the DeCAF wiki. They are: - `pool5_cudanet_out` - `fc6_cudanet_out` - `fc6_neuron_cudanet_out` - `fc7_cudanet_out` - `fc7_neuron_cudanet_out` - `probs_cudanet_out` :param pretrained_params: This must point to the file with the pretrained parameters. Defaults to `imagenet.decafnet.epoch90`. For the ImageNet-1k-pretrained ConvNet this file can be obtained from here: http://www.eecs.berkeley.edu/~jiayq/decaf_pretrained/ :param pretrained_meta: Similar to `pretrained_params`, this must file to the file with the pretrained parameters' metadata. Defaults to `imagenet.decafnet.meta`. :param center_only: Use the center patch of the image only when extracting features. If `False`, use four corners, the image center and flipped variants and average a total of 10 feature vectors, which will usually yield better results. Defaults to `True`. :param classify_direct: When `True`, assume that input X is an array of shape (num x 256 x 256 x 3) as returned by `prepare_image`. """ self.feature_layer = feature_layer self.pretrained_params = pretrained_params self.pretrained_meta = pretrained_meta self.center_only = center_only self.classify_direct = classify_direct self.net_ = None if (not os.path.exists(pretrained_params) or not os.path.exists(pretrained_meta)): raise ValueError( "Pre-trained ConvNet parameters not found. You may" "need to download the files from " "http://www.eecs.berkeley.edu/~jiayq/decaf_pretrained/ and " "pass the path to the two files as `pretrained_params` and " "`pretrained_meta` to the `{}` estimator.".format( self.__class__.__name__)) def fit(self, X=None, y=None): from decaf.scripts.imagenet import DecafNet # soft dep if self.net_ is None: self.net_ = DecafNet( self.pretrained_params, self.pretrained_meta, ) return self @cache.cached(_transform_cache_key) def transform(self, X): features = [] for img in X: if self.classify_direct: images = self.net_.oversample( img, center_only=self.center_only) self.net_.classify_direct(images) else: if isinstance(img, str): import Image # soft dep img = np.array(Image.open(img)) self.net_.classify(img, center_only=self.center_only) feat = None for layer in self.feature_layer.split(','): val = self.net_.feature(layer) if feat is None: feat = val else: feat = np.hstack([feat, val]) if not self.center_only: feat = feat.flatten() features.append(feat) if self.verbose: sys.stdout.write( "\r[ConvNet] %d%%" % (100. * len(features) / len(X))) sys.stdout.flush() if self.verbose: sys.stdout.write('\n') return np.vstack(features) def prepare_image(self, image): """Returns image of shape `(256, 256, 3)`, as expected by `transform` when `classify_direct = True`. """ from decaf.util import transform # soft dep _JEFFNET_FLIP = True # first, extract the 256x256 center. image = transform.scale_and_extract(transform.as_rgb(image), 256) # convert to [0,255] float32 image = image.astype(np.float32) * 255. if _JEFFNET_FLIP: # Flip the image if necessary, maintaining the c_contiguous order image = image[::-1, :].copy() # subtract the mean image -= self.net_._data_mean return image
class NetworkDecaf(Network): """ Implementation for the Decaf library. """ def __init__(self, model_spec_filename, model_filename=None,\ wnid_words_filename=None, center_only=False, wnid_subset = []): """ *** PRIVATE CONSTRUCTOR *** """ # the following is just an hack to allow retro-compatibility # with existing code if isinstance(model_spec_filename, NetworkDecafParams): params = model_spec_filename model_spec_filename = params.model_spec_filename model_filename = params.model_filename wnid_words_filename = params.wnid_words_filename center_only = params.center_only wnid_subset = params.wnid_subset if wnid_subset!=[]: print 'Warning: subset of labels not supported yet' else: assert isinstance(model_spec_filename, str) assert model_filename != None assert wnid_words_filename != None # load Decaf model self.net_ = DecafNet(model_filename, model_spec_filename) self.center_only_ = center_only # build a dictionary label --> description self.dict_label_desc_ = {} dict_desc_label = {} fd = open(wnid_words_filename) for line in fd: temp = line.strip().split('\t') wnid = temp[1].strip() self.dict_label_desc_[wnid] = temp[2].strip() dict_desc_label[temp[2].split(',')[0]] = wnid fd.close() # build a dictionary label --> label_id self.dict_label_id_ = {} self.labels_ = [] for i, desc in enumerate(self.net_.label_names): self.dict_label_id_[dict_desc_label[desc]] = i self.labels_.append(dict_desc_label[desc]) # Load the mean vector from file # mean of 3 channels self.net_.mean_img =np.mean(np.mean(self.net_._data_mean,axis=1),axis=0) # it is in BGR convert in RGB #self.net_.mean_img = self.net_.mean_img[::-1] def get_mean_img(self): return self.net_.mean_img def get_input_dim(self): return decaf.scripts.imagenet.INPUT_DIM def get_label_id(self, label): return self.dict_label_id_[label] def get_label_desc(self, label): return self.dict_label_desc_[label] def get_labels(self): return self.labels_ def evaluate(self, img, layer_name = 'softmax'): # for now only center_only is supported assert self.center_only_ == True # first, extract the 227x227 center dim = decaf.scripts.imagenet.INPUT_DIM image = util.crop_image_center(decaf.util.transform.as_rgb(img)) image = skimage.transform.resize(image, (dim, dim)) # convert to [0,255] float32 image = image.astype(np.float32) * 255. assert np.max(image) <= 255 # Flip the image if necessary, maintaining the c_contiguous order if decaf.scripts.imagenet._JEFFNET_FLIP: image = image[::-1, :].copy() # subtract the mean, cropping the 256x256 mean image xoff = (self.net_._data_mean.shape[1] - dim)/2 yoff = (self.net_._data_mean.shape[0] - dim)/2 image -= self.net_._data_mean[yoff+yoff+dim, xoff:xoff+dim] # make sure the data in contiguous in memory images = np.ascontiguousarray(image[np.newaxis], dtype=np.float32) # classify predictions = self.net_.classify_direct(images) scores = predictions.mean(0) # look at the particular layer if layer_name == 'softmax': return scores elif layer_name == 'fc7_relu': layer_name = 'fc7_neuron_cudanet_out' elif layer_name == 'fc7': layer_name = 'fc7_cudanet_out' elif layer_name == 'fc6_relu': layer_name = 'fc6_neuron_cudanet_out' elif layer_name == 'fc6': layer_name = 'fc6_cudanet_out' elif layer_name == 'pool5': layer_name = 'pool5_cudanet_out' else: raise ValueError('layer_name not supported') return self.net_.feature(layer_name)