def hasher_spherical(row_cols, params):
features = []
for row, columns in row_cols:
cur_feature = msgpack.loads(columns['feature'])
features.append(np.array(cur_feature[0]))
features = np.asfarray(features)
out = picarus_takeout.spherical_hasher_train(features, params['num_pivots'], params['eps_m'], params['eps_s'], params['max_iters'])
out = {'pivots': out['pivots'].ravel().tolist(),
'threshs': out['threshs'].tolist()}
model_link = {'name': 'picarus.SphericalHasher', 'kw': out}
return 'feature', 'hash', model_link
def hasher_spherical(row_cols, params):
features = []
for row, columns in row_cols:
cur_feature = msgpack.loads(columns['feature'])
features.append(np.array(cur_feature[0]))
features = np.asfarray(features)
out = picarus_takeout.spherical_hasher_train(features,
params['num_pivots'],
params['eps_m'],
params['eps_s'],
params['max_iters'])
out = {
'pivots': out['pivots'].ravel().tolist(),
'threshs': out['threshs'].tolist()
}
model_link = {'name': 'picarus.SphericalHasher', 'kw': out}
return 'feature', 'hash', model_link
def hasher_spherical(queue, params, inputs, schema, start_stop_rows, table, owner):
thrift, manager, slices, inputsb64 = _setup(start_stop_rows, inputs)
features = []
for start_row, stop_row in start_stop_rows:
row_cols = hadoopy_hbase.scanner(thrift, table,
columns=[inputs['feature']],
start_row=start_row, stop_row=stop_row)
for row, cols in row_cols:
cur_feature = msgpack.loads(cols[inputs['feature']])
features.append(np.array(cur_feature[0]))
print('num_features[%d]' % len(features))
features = np.asfarray(features)
out = picarus_takeout.spherical_hasher_train(features, params['num_pivots'], params['eps_m'], params['eps_s'], params['max_iters'])
out = {'pivots': out['pivots'].ravel().tolist(),
'threshs': out['threshs'].tolist()}
#out = picarus.modules.spherical_hash.train_takeout(features, params['num_pivots'], params['eps_m'], params['eps_s'], params['max_iters'])
factory_info = {'slices': slices, 'num_features': len(features), 'data': 'slices', 'params': params, 'inputs': inputsb64}
model_link = {'name': 'picarus.SphericalHasher', 'kw': out}
model_chain = tables._takeout_model_chain_from_key(manager, inputs['feature']) + [model_link]
queue.put(manager.input_model_param_to_key(**{'input': inputs['feature'], 'model_link': model_link, 'model_chain': model_chain, 'input_type': 'feature',
'output_type': 'hash', 'email': owner, 'name': manager.model_to_name(model_link),
'factory_info': json.dumps(factory_info)}))