def test_hbase(self): import hadoopy_hbase client = hadoopy_hbase.connect() try: client.createTable('testtable', [hadoopy_hbase.ColumnDescriptor('colfam1:')]) except: pass for x in xrange(100): client.mutateRow('testtable', str(x), [ hadoopy_hbase.Mutation(column='colfam1:col%d' % y, value=str(x)) for y in range(10) ]) ds = data_sources.HBaseDataSource({'mydata': 'colfam1:col0'}, 'testtable') print list(ds.rows()) print list(ds.columns(list(ds.rows())[0])) print list(ds.column_values(list(ds.rows())[0])) print[(x, list(y)) for x, y in ds.row_columns()] print[(x, dict(y)) for x, y in ds.row_column_values()] print ds.uri ds = data_sources.data_source_from_uri(ds.uri) print list(ds.rows()) print list(ds.columns(list(ds.rows())[0])) print list(ds.column_values(list(ds.rows())[0])) print[(x, list(y)) for x, y in ds.row_columns()] print[(x, dict(y)) for x, y in ds.row_column_values()] print ds.uri
def annotation_masks_to_hbase(self): import redis import ast import imfeat import cv2 r = redis.StrictRedis(port=6381, db=6) responses = [r.hgetall(x) for x in r.keys()] print('Total Responses[%d]' % len(responses)) image_class_segments = {} # [row][class_name] = segments for x in responses: if 'user_data' in x: x['user_data'] = ast.literal_eval(x['user_data']) if x['user_data']['segments']: row_key = x['image'] image_class_segments.setdefault(row_key, {}).setdefault(x['user_data']['name'], []).append(x['user_data']['segments']) for row_key, class_segments in image_class_segments.items(): columns = dict((x, y.value) for x, y in self.hb.getRowWithColumns(self.images_table, row_key, [self.image_column, self.superpixel_column])[0].columns.items()) image = imfeat.image_fromstring(columns[self.image_column]) segments = json.loads(columns[self.superpixel_column]) class_masks = {} class_masks = np.zeros((image.shape[0], image.shape[1], self.texton_num_classes)) for name, user_segment_groups in class_segments.items(): cur_mask = np.zeros(image.shape[:2], dtype=np.uint8) for user_segment_group in user_segment_groups: for user_segment in user_segment_group: hull = segments[user_segment] hull = np.asarray(hull).astype(np.int32).reshape(1, -1, 2) mask_color = 255 cv2.drawContours(cur_mask, hull, -1, mask_color, -1) cv2.drawContours(cur_mask, hull, -1, mask_color, 2) class_masks[:, :, self.texton_classes[name]['mask_num']] = cur_mask / 255. class_masks_ser = picarus.api.np_tostring(class_masks) print('Storing row [%s]' % repr(row_key)) self.hb.mutateRow(self.images_table, row_key, [hadoopy_hbase.Mutation(column=self.masks_gt_column, value=class_masks_ser)])
def hbase_loader(prefix, dataset, thrift_server, thrift_port, verbose=False): import hadoopy_hbase dataset = DATASETS[dataset]() client = hadoopy_hbase.connect(thrift_server, thrift_port) for split, name, columns in dataset.images(): row = hadoopy_hbase.hash_key(name, prefix=prefix + split, suffix=name, hash_bytes=4) if verbose: print(repr(row)) mutations = [hadoopy_hbase.Mutation(column=x, value=y) for x, y in columns.items()] client.mutateRow(TABLE, row, mutations)
#!/usr/bin/env python from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport from thrift.protocol import TBinaryProtocol from thrift_bench import random_string, remove_table import hadoopy_hbase client = hadoopy_hbase.connect('localhost') remove_table(client, 'testtable') client.createTable('testtable', [hadoopy_hbase.ColumnDescriptor('colfam1:')]) for x in xrange(100): client.mutateRow('testtable', str(x), [hadoopy_hbase.Mutation(column='colfam1:col%d' % y, value=random_string(5)) for y in range(10)]) print(client.getRow('testtable', '0'))
def __delitem__(self, key): assert isinstance(key, str) self._db.mutateRow( self._table, key, [hadoopy_hbase.Mutation(column=self._col, isDelete=True)])
def __setitem__(self, key, value): assert isinstance(key, str) assert isinstance(value, str) self._db.mutateRow( self._table, key, [hadoopy_hbase.Mutation(column=self._col, value=value)])
def _data(table, row, col): row = base64.urlsafe_b64decode(row) col = base64.urlsafe_b64decode(col) method = bottle.request.method.upper() #print_request() # TODO Check authentication per table if table not in ('images', 'testtable'): raise ValueError('Only images/testtable allowed for now!') if method == 'GET': if table and row and col: result = THRIFT.get(table, row, col) if not result: raise ValueError('Cell not found!') return {'data': base64.b64encode(result[0].value)} elif table and row: result = THRIFT.getRow(table, row) if not result: raise ValueError('Row not found!') return { 'data': dict((x, base64.b64encode(y.value)) for x, y in result[0].columns.items()) } elif method == 'PUT': mutations = [] if col: if 'function' in bottle.request.params: # TODO: Handle row=* input_table, input_row, input_col = bottle.request.params[ 'input'].split() result = THRIFT.get(table, row, col) if not result: raise ValueError('Cell not found!') image = result[0].value # Remove internal parameters params = dict(bottle.request.params) del params['input'] del params['function'] output = _action_handle(bottle.request.params['function'], params, image) mutations.append( hadoopy_hbase.Mutation(column=col, value=output)) else: if 'data' in bottle.request.files: mutations.append( hadoopy_hbase.Mutation( column=col, value=bottle.request.files['data'].file.read())) elif 'data' in bottle.request.params: mutations.append( hadoopy_hbase.Mutation( column=col, value=bottle.request.params['data'])) else: raise ValueError('"data" must be specified!') else: for x in bottle.request.files: THRIFT.mutateRow(table, row, [ hadoopy_hbase.Mutation( column=x, value=bottle.request.files[x].file.read()) ]) for x in set(bottle.request.params) - set(bottle.request.files): mutations.append( hadoopy_hbase.Mutation(column=x, value=bottle.request.params[x])) if not mutations and len(bottle.request.files) == 0: raise ValueError('No columns specified!') if mutations: THRIFT.mutateRow(table, row, mutations) return {} elif method == 'DELETE': mutations = [] if col: mutations.append(hadoopy_hbase.Mutation(column=col, isDelete=True)) else: THRIFT.deleteAllRow(table, row) return {} else: raise ValueError
def delete_column(self, table, row, column): self._thrift.mutateRow( table, row, [hadoopy_hbase.Mutation(column=column, isDelete=True)])
def mutate_row(self, table, row, mutations): mutations = [ hadoopy_hbase.Mutation(column=x, value=y) for x, y in mutations.items() ] self._thrift.mutateRow(table, row, mutations)