Example #1
0
def model_fromfile(path):
    if path.endswith('.js.gz'):
        return call_import(json.loads(zlib.decompress(open(path).read())))
    elif path.endswith('.msgpack.gz'):
        return call_import(msgpack.load(zlib.decompress(open(path).read())))
    elif path.endswith('.pkl.gz'):
        return pickle.loads(zlib.decompress(open(path).read()))
    else:
        raise ValueError('Unknown model type[%s]' % path)
Example #2
0
def image_classifier_fromstring(classifier_ser):
    cp = Classifier()
    cp.ParseFromString(classifier_ser)
    loader = lambda x, y: pickle.loads(y) if x == cp.PICKLE else call_import(json.loads(y))
    feature = loader(cp.feature_format, cp.feature)
    classifier = loader(cp.classifier_format, cp.classifier)
    return lambda image: float(classifier.decision_function(feature(image)).flat[0])
Example #3
0
def model_fromfile(path):
    if path.endswith('.js.gz'):
        return call_import(json.loads(zlib.decompress(open(path).read())))
    elif path.endswith('.pkl.gz'):
        return pickle.loads(zlib.decompress(open(path).read()))
    else:
        raise ValueError('Unknown model type[%s]' % path)
 def __init__(self):
     self._feats = pickle.load(open(os.environ['FEATURES_FN']))
     self._feats = [call_import(x) if isinstance(x, dict) else x
                    for x in self._feats]
     self.frame_skip = 30
     self.max_frames_per_video = float(os.environ.get('MAX_FRAMES_PER_VIDEO', float('inf')))
     self.remove_bars = imfeat.BlackBars()
 def __init__(self):
     self._feat = pickle.load(open(os.environ['FEATURE_FN']))
     if isinstance(self._feat, dict):
         self._feat = call_import(self._feat)
     self.frame_skip = 30
     self.max_outputs_per_video = float(os.environ.get('MAX_OUTPUTS_PER_VIDEO', float('inf')))
     self.max_frames_per_video = float(os.environ.get('MAX_FRAMES_PER_VIDEO', float('inf')))
     self.output_frame = int(os.environ.get('OUTPUT_FRAME', 1))
     self.remove_bars = imfeat.BlackBars()
Example #6
0
 def index_train(model_dict, model_param, inputs):
     index = call_import(model_dict)
     row_cols = hadoopy_hbase.scanner(thrift, self.table,
                                      columns=[inputs['hash'], inputs['meta']], start_row=start_row, stop_row=stop_row)
     metadata, hashes = zip(*[(json.dumps([cols[inputs['meta']], base64.urlsafe_b64encode(row)]), cols[inputs['hash']])
                              for row, cols in row_cols])
     hashes = np.ascontiguousarray(np.asfarray([np.fromstring(h, dtype=np.uint8) for h in hashes]))
     index = index.store_hashes(hashes, np.arange(len(metadata), dtype=np.uint64))
     index.metadata = metadata
     return index
Example #7
0
 def kmeans_cluster_mfeat(model_dict, model_param, inputs):
     # TODO: This needs to be finished, determine if we want quantizer level or cluster level
     clusterer = call_import(model_dict)
     features = []
     row_cols = hadoopy_hbase.scanner(thrift, self.table,
                                      columns=[inputs['multi_feature']], start_row=start_row, stop_row=stop_row)
     # TODO: We'll want to check that we aren't clustering too much data by placing constraints
     for row, columns in row_cols:
         features.append(picarus.api.np_fromstring(columns[inputs['multi_feature']]))
     features = np.vstack(features)
     return clusterer.cluster(features)
Example #8
0
 def __init__(self):
     self._feat = pickle.load(open(os.environ['FEATURE_FN']))
     if isinstance(self._feat, dict):
         self._feat = call_import(self._feat)
     self.frame_skip = 30
     self.max_outputs_per_video = float(
         os.environ.get('MAX_OUTPUTS_PER_VIDEO', float('inf')))
     self.max_frames_per_video = float(
         os.environ.get('MAX_FRAMES_PER_VIDEO', float('inf')))
     self.output_frame = int(os.environ.get('OUTPUT_FRAME', 1))
     self.remove_bars = imfeat.BlackBars()
Example #9
0
 def load(self, proto_data, load_feature=True, load_hasher=True, load_index=True):
     si = picarus.api.SearchIndex()
     si.ParseFromString(proto_data)
     loader = lambda x, y: pickle.loads(y) if x == si.PICKLE else call_import(json.loads(y))
     self.metadata = np.array(si.metadata)
     if load_index:
         self.index = loader(si.index_format, si.index)
     if load_hasher:
         self.hasher = loader(si.hash_format, si.hash)
     if load_feature:
         f = loader(si.feature_format, si.feature)
         self.feature = lambda y: f(imfeat.resize_image_max_side(y, self.max_side))
     return self
Example #10
0
 def load(self,
          proto_data,
          load_feature=True,
          load_hasher=True,
          load_index=True):
     si = picarus.api.SearchIndex()
     si.ParseFromString(proto_data)
     loader = lambda x, y: pickle.loads(
         y) if x == si.PICKLE else call_import(json.loads(y))
     self.metadata = np.array(si.metadata)
     if load_index:
         self.index = loader(si.index_format, si.index)
     if load_hasher:
         self.hasher = loader(si.hash_format, si.hash)
     if load_feature:
         f = loader(si.feature_format, si.feature)
         self.feature = lambda y: f(
             imfeat.resize_image_max_side(y, self.max_side))
     return self
Example #11
0
def _classifier_frompb(c, feature_input=False):
    loader = lambda x, y: pickle.loads(y) if x == c.PICKLE else call_import(json.loads(y))
    preprocessor = loader(c.preprocessor_format, c.preprocessor)
    feature = loader(c.feature_format, c.feature)
    classifier = loader(c.classifier_format, c.classifier)
    if c.classifier_type == c.CLASS_DISTANCE_LIST:
        classifier_func = classifier
    else:
        classifier_func = lambda feature: float(classifier.decision_function(feature).flat[0])
    if c.feature_type == c.FEATURE:
        feature_func = feature
    elif c.feature_type == c.MULTI_FEATURE:
        feature_func = lambda image: feature.compute_dense(image)
    else:
        raise ValueError('Unknown feature type')
    if feature_input:
        return classifier_func
    else:
        return lambda image: classifier_func(feature_func(preprocessor.asarray(image)))
 def __init__(self):
     feature = open(os.environ['FEATURE']).read()
     try:
         self._feat = features.select_feature(feature)
     except KeyError:
         self._feat = call_import(json.loads(zlib.decompress(feature)))
 def __init__(self):
     feature = open(os.environ["FEATURE"]).read()
     try:
         self._feat = features.select_feature(feature)
     except KeyError:
         self._feat = call_import(json.loads(zlib.decompress(feature)))
Example #14
0
 def hasher_train(model_dict, model_param, inputs):
     hasher = call_import(model_dict)
     features = hadoopy_hbase.scanner_column(thrift, self.table, inputs['feature'],
                                             start_row=start_row, stop_row=stop_row)
     return hasher.train(picarus.api.np_fromstring(x) for x in features)