def model_fromfile(path):
if path.endswith('.js.gz'):
return call_import(json.loads(zlib.decompress(open(path).read())))
elif path.endswith('.msgpack.gz'):
return call_import(msgpack.load(zlib.decompress(open(path).read())))
elif path.endswith('.pkl.gz'):
return pickle.loads(zlib.decompress(open(path).read()))
else:
raise ValueError('Unknown model type[%s]' % path)
def image_classifier_fromstring(classifier_ser):
cp = Classifier()
cp.ParseFromString(classifier_ser)
loader = lambda x, y: pickle.loads(y) if x == cp.PICKLE else call_import(json.loads(y))
feature = loader(cp.feature_format, cp.feature)
classifier = loader(cp.classifier_format, cp.classifier)
return lambda image: float(classifier.decision_function(feature(image)).flat[0])
def model_fromfile(path):
if path.endswith('.js.gz'):
return call_import(json.loads(zlib.decompress(open(path).read())))
elif path.endswith('.pkl.gz'):
return pickle.loads(zlib.decompress(open(path).read()))
else:
raise ValueError('Unknown model type[%s]' % path)
def __init__(self):
self._feats = pickle.load(open(os.environ['FEATURES_FN']))
self._feats = [call_import(x) if isinstance(x, dict) else x
for x in self._feats]
self.frame_skip = 30
self.max_frames_per_video = float(os.environ.get('MAX_FRAMES_PER_VIDEO', float('inf')))
self.remove_bars = imfeat.BlackBars()
def __init__(self):
self._feat = pickle.load(open(os.environ['FEATURE_FN']))
if isinstance(self._feat, dict):
self._feat = call_import(self._feat)
self.frame_skip = 30
self.max_outputs_per_video = float(os.environ.get('MAX_OUTPUTS_PER_VIDEO', float('inf')))
self.max_frames_per_video = float(os.environ.get('MAX_FRAMES_PER_VIDEO', float('inf')))
self.output_frame = int(os.environ.get('OUTPUT_FRAME', 1))
self.remove_bars = imfeat.BlackBars()
def index_train(model_dict, model_param, inputs):
index = call_import(model_dict)
row_cols = hadoopy_hbase.scanner(thrift, self.table,
columns=[inputs['hash'], inputs['meta']], start_row=start_row, stop_row=stop_row)
metadata, hashes = zip(*[(json.dumps([cols[inputs['meta']], base64.urlsafe_b64encode(row)]), cols[inputs['hash']])
for row, cols in row_cols])
hashes = np.ascontiguousarray(np.asfarray([np.fromstring(h, dtype=np.uint8) for h in hashes]))
index = index.store_hashes(hashes, np.arange(len(metadata), dtype=np.uint64))
index.metadata = metadata
return index
def kmeans_cluster_mfeat(model_dict, model_param, inputs):
# TODO: This needs to be finished, determine if we want quantizer level or cluster level
clusterer = call_import(model_dict)
features = []
row_cols = hadoopy_hbase.scanner(thrift, self.table,
columns=[inputs['multi_feature']], start_row=start_row, stop_row=stop_row)
# TODO: We'll want to check that we aren't clustering too much data by placing constraints
for row, columns in row_cols:
features.append(picarus.api.np_fromstring(columns[inputs['multi_feature']]))
features = np.vstack(features)
return clusterer.cluster(features)
def __init__(self):
self._feat = pickle.load(open(os.environ['FEATURE_FN']))
if isinstance(self._feat, dict):
self._feat = call_import(self._feat)
self.frame_skip = 30
self.max_outputs_per_video = float(
os.environ.get('MAX_OUTPUTS_PER_VIDEO', float('inf')))
self.max_frames_per_video = float(
os.environ.get('MAX_FRAMES_PER_VIDEO', float('inf')))
self.output_frame = int(os.environ.get('OUTPUT_FRAME', 1))
self.remove_bars = imfeat.BlackBars()
def load(self, proto_data, load_feature=True, load_hasher=True, load_index=True):
si = picarus.api.SearchIndex()
si.ParseFromString(proto_data)
loader = lambda x, y: pickle.loads(y) if x == si.PICKLE else call_import(json.loads(y))
self.metadata = np.array(si.metadata)
if load_index:
self.index = loader(si.index_format, si.index)
if load_hasher:
self.hasher = loader(si.hash_format, si.hash)
if load_feature:
f = loader(si.feature_format, si.feature)
self.feature = lambda y: f(imfeat.resize_image_max_side(y, self.max_side))
return self
def load(self,
proto_data,
load_feature=True,
load_hasher=True,
load_index=True):
si = picarus.api.SearchIndex()
si.ParseFromString(proto_data)
loader = lambda x, y: pickle.loads(
y) if x == si.PICKLE else call_import(json.loads(y))
self.metadata = np.array(si.metadata)
if load_index:
self.index = loader(si.index_format, si.index)
if load_hasher:
self.hasher = loader(si.hash_format, si.hash)
if load_feature:
f = loader(si.feature_format, si.feature)
self.feature = lambda y: f(
imfeat.resize_image_max_side(y, self.max_side))
return self
def _classifier_frompb(c, feature_input=False):
loader = lambda x, y: pickle.loads(y) if x == c.PICKLE else call_import(json.loads(y))
preprocessor = loader(c.preprocessor_format, c.preprocessor)
feature = loader(c.feature_format, c.feature)
classifier = loader(c.classifier_format, c.classifier)
if c.classifier_type == c.CLASS_DISTANCE_LIST:
classifier_func = classifier
else:
classifier_func = lambda feature: float(classifier.decision_function(feature).flat[0])
if c.feature_type == c.FEATURE:
feature_func = feature
elif c.feature_type == c.MULTI_FEATURE:
feature_func = lambda image: feature.compute_dense(image)
else:
raise ValueError('Unknown feature type')
if feature_input:
return classifier_func
else:
return lambda image: classifier_func(feature_func(preprocessor.asarray(image)))
def __init__(self):
feature = open(os.environ['FEATURE']).read()
try:
self._feat = features.select_feature(feature)
except KeyError:
self._feat = call_import(json.loads(zlib.decompress(feature)))
def __init__(self):
feature = open(os.environ["FEATURE"]).read()
try:
self._feat = features.select_feature(feature)
except KeyError:
self._feat = call_import(json.loads(zlib.decompress(feature)))
def hasher_train(model_dict, model_param, inputs):
hasher = call_import(model_dict)
features = hadoopy_hbase.scanner_column(thrift, self.table, inputs['feature'],
start_row=start_row, stop_row=stop_row)
return hasher.train(picarus.api.np_fromstring(x) for x in features)