def test_hbase(self):
import hadoopy_hbase
client = hadoopy_hbase.connect()
try:
client.createTable('testtable',
[hadoopy_hbase.ColumnDescriptor('colfam1:')])
except:
pass
for x in xrange(100):
client.mutateRow('testtable', str(x), [
hadoopy_hbase.Mutation(column='colfam1:col%d' % y,
value=str(x)) for y in range(10)
])
ds = data_sources.HBaseDataSource({'mydata': 'colfam1:col0'},
'testtable')
print list(ds.rows())
print list(ds.columns(list(ds.rows())[0]))
print list(ds.column_values(list(ds.rows())[0]))
print[(x, list(y)) for x, y in ds.row_columns()]
print[(x, dict(y)) for x, y in ds.row_column_values()]
print ds.uri
ds = data_sources.data_source_from_uri(ds.uri)
print list(ds.rows())
print list(ds.columns(list(ds.rows())[0]))
print list(ds.column_values(list(ds.rows())[0]))
print[(x, list(y)) for x, y in ds.row_columns()]
print[(x, dict(y)) for x, y in ds.row_column_values()]
print ds.uri
def annotation_masks_to_hbase(self):
import redis
import ast
import imfeat
import cv2
r = redis.StrictRedis(port=6381, db=6)
responses = [r.hgetall(x) for x in r.keys()]
print('Total Responses[%d]' % len(responses))
image_class_segments = {} # [row][class_name] = segments
for x in responses:
if 'user_data' in x:
x['user_data'] = ast.literal_eval(x['user_data'])
if x['user_data']['segments']:
row_key = x['image']
image_class_segments.setdefault(row_key, {}).setdefault(x['user_data']['name'], []).append(x['user_data']['segments'])
for row_key, class_segments in image_class_segments.items():
columns = dict((x, y.value) for x, y in self.hb.getRowWithColumns(self.images_table, row_key, [self.image_column, self.superpixel_column])[0].columns.items())
image = imfeat.image_fromstring(columns[self.image_column])
segments = json.loads(columns[self.superpixel_column])
class_masks = {}
class_masks = np.zeros((image.shape[0], image.shape[1], self.texton_num_classes))
for name, user_segment_groups in class_segments.items():
cur_mask = np.zeros(image.shape[:2], dtype=np.uint8)
for user_segment_group in user_segment_groups:
for user_segment in user_segment_group:
hull = segments[user_segment]
hull = np.asarray(hull).astype(np.int32).reshape(1, -1, 2)
mask_color = 255
cv2.drawContours(cur_mask, hull, -1, mask_color, -1)
cv2.drawContours(cur_mask, hull, -1, mask_color, 2)
class_masks[:, :, self.texton_classes[name]['mask_num']] = cur_mask / 255.
class_masks_ser = picarus.api.np_tostring(class_masks)
print('Storing row [%s]' % repr(row_key))
self.hb.mutateRow(self.images_table, row_key, [hadoopy_hbase.Mutation(column=self.masks_gt_column, value=class_masks_ser)])
def hbase_loader(prefix, dataset, thrift_server, thrift_port, verbose=False):
import hadoopy_hbase
dataset = DATASETS[dataset]()
client = hadoopy_hbase.connect(thrift_server, thrift_port)
for split, name, columns in dataset.images():
row = hadoopy_hbase.hash_key(name, prefix=prefix + split, suffix=name, hash_bytes=4)
if verbose:
print(repr(row))
mutations = [hadoopy_hbase.Mutation(column=x, value=y) for x, y in columns.items()]
client.mutateRow(TABLE, row, mutations)
#!/usr/bin/env python
from thrift.transport.TSocket import TSocket
from thrift.transport.TTransport import TBufferedTransport
from thrift.protocol import TBinaryProtocol
from thrift_bench import random_string, remove_table
import hadoopy_hbase
client = hadoopy_hbase.connect('localhost')
remove_table(client, 'testtable')
client.createTable('testtable', [hadoopy_hbase.ColumnDescriptor('colfam1:')])
for x in xrange(100):
client.mutateRow('testtable', str(x), [hadoopy_hbase.Mutation(column='colfam1:col%d' % y, value=random_string(5)) for y in range(10)])
print(client.getRow('testtable', '0'))
def __delitem__(self, key):
assert isinstance(key, str)
self._db.mutateRow(
self._table, key,
[hadoopy_hbase.Mutation(column=self._col, isDelete=True)])
def __setitem__(self, key, value):
assert isinstance(key, str)
assert isinstance(value, str)
self._db.mutateRow(
self._table, key,
[hadoopy_hbase.Mutation(column=self._col, value=value)])
def _data(table, row, col):
row = base64.urlsafe_b64decode(row)
col = base64.urlsafe_b64decode(col)
method = bottle.request.method.upper()
#print_request()
# TODO Check authentication per table
if table not in ('images', 'testtable'):
raise ValueError('Only images/testtable allowed for now!')
if method == 'GET':
if table and row and col:
result = THRIFT.get(table, row, col)
if not result:
raise ValueError('Cell not found!')
return {'data': base64.b64encode(result[0].value)}
elif table and row:
result = THRIFT.getRow(table, row)
if not result:
raise ValueError('Row not found!')
return {
'data':
dict((x, base64.b64encode(y.value))
for x, y in result[0].columns.items())
}
elif method == 'PUT':
mutations = []
if col:
if 'function' in bottle.request.params:
# TODO: Handle row=*
input_table, input_row, input_col = bottle.request.params[
'input'].split()
result = THRIFT.get(table, row, col)
if not result:
raise ValueError('Cell not found!')
image = result[0].value
# Remove internal parameters
params = dict(bottle.request.params)
del params['input']
del params['function']
output = _action_handle(bottle.request.params['function'],
params, image)
mutations.append(
hadoopy_hbase.Mutation(column=col, value=output))
else:
if 'data' in bottle.request.files:
mutations.append(
hadoopy_hbase.Mutation(
column=col,
value=bottle.request.files['data'].file.read()))
elif 'data' in bottle.request.params:
mutations.append(
hadoopy_hbase.Mutation(
column=col, value=bottle.request.params['data']))
else:
raise ValueError('"data" must be specified!')
else:
for x in bottle.request.files:
THRIFT.mutateRow(table, row, [
hadoopy_hbase.Mutation(
column=x, value=bottle.request.files[x].file.read())
])
for x in set(bottle.request.params) - set(bottle.request.files):
mutations.append(
hadoopy_hbase.Mutation(column=x,
value=bottle.request.params[x]))
if not mutations and len(bottle.request.files) == 0:
raise ValueError('No columns specified!')
if mutations:
THRIFT.mutateRow(table, row, mutations)
return {}
elif method == 'DELETE':
mutations = []
if col:
mutations.append(hadoopy_hbase.Mutation(column=col, isDelete=True))
else:
THRIFT.deleteAllRow(table, row)
return {}
else:
raise ValueError
def delete_column(self, table, row, column):
self._thrift.mutateRow(
table, row, [hadoopy_hbase.Mutation(column=column, isDelete=True)])
def mutate_row(self, table, row, mutations):
mutations = [
hadoopy_hbase.Mutation(column=x, value=y)
for x, y in mutations.items()
]
self._thrift.mutateRow(table, row, mutations)
