def hbase_enum(target, port):
print(colored(
"\n[!] Enumeration Module For NoSQL Framework H-Base Launched.....", 'yellow'))
print(colored("[-] Enumerating Cluster Version and Cluster Status", 'blue'))
try:
c = Connection(target, port)
print(colored("[-] Cluster Version: %s" %
(str(c.cluster_version)), 'green'))
v = c.cluster_status
print(colored("[-] Cluster Status ", 'green'))
for key, value in v.iteritems():
print(colored("\t [-] "+str(key)+":"+str(value), 'green'))
print(colored("[-] Enumerating JVM and Box Details", 'blue'))
for key, value in c.version.iteritems():
print(colored("\t[-] "+str(key)+":"+str(value), 'green'))
print(colored("[-] Tables Available", 'blue'))
for i in c.tables():
print(colored("\t[-] "+i, 'green'))
print(colored("Would you like to enumerate columns", 'blue'))
choice = raw_input()
if choice == 'y':
tab = raw_input(colored("[-] Enter tables name ", 'blue'))
if tab in c.tables():
print(colored("[-] Enumerating Columns", 'blue'))
t = c.table(tab)
for i in t.columns():
print(colored("\t[-] "+str(i), 'green'))
else:
print(colored("[-] No such table Exists ", 'red'))
except Exception as e:
print(colored("[-] Error Occured while connection %s " %
(str(e)), 'red'))
def hbase_enum(target, port):
print colored(
"\n[!] Enumeration Module For NoSQL Framework H-Base Launched.....",
'yellow')
print colored("[-] Enumerating Cluster Version and Cluster Status", 'blue')
try:
c = Connection(target, port)
print colored("[-] Cluster Version: %s" % (str(c.cluster_version)),
'green')
v = c.cluster_status
print colored("[-] Cluster Status ", 'green')
for key, value in v.iteritems():
print colored("\t [-] " + str(key) + ":" + str(value), 'green')
print colored("[-] Enumerating JVM and Box Details", 'blue')
for key, value in c.version.iteritems():
print colored("\t[-] " + str(key) + ":" + str(value), 'green')
print colored("[-] Tables Available", 'blue')
for i in c.tables():
print colored("\t[-] " + i, 'green')
print colored("Would you like to enumerate columns", 'blue')
choice = raw_input()
if choice == 'y':
tab = raw_input(colored("[-] Enter tables name ", 'blue'))
if tab in c.tables():
print colored("[-] Enumerating Columns", 'blue')
t = c.table(tab)
for i in t.columns():
print colored("\t[-] " + str(i), 'green')
else:
print colored("[-] No such table Exists ", 'red')
except Exception, e:
print colored("[-] Error Occured while connection %s " % (str(e)),
'red')
def stargate(tablename,column_name,insert_data,added_row_num):
c = Connection()
c = Connection(host='127.0.0.1', port=7060)
print c.tables()
# request = requests.get(baseurl + "/" + tablename + "/schema")
# tablename = 'SRAtest'
# column_name = 'EXPERIMENT_ID:
# count_cmd = 'ls '
print added_row_num
# added_row_num = 64043
row_key = 'row' +str(added_row_num)
t = c.table(tablename)
print row_key
# t.insert(rowkey,
# {
# 'read':'ACGT'}
# )
if column_name.find(':') == -1:
t.insert(row_key,
{
column_name:
{ '':insert_data}
}
)
else:
column = column_name.split(':')[0]
key = column_name.split(':')[1]
t.insert(row_key,
{
column:
{ key:insert_data}
}
)
print 'insert finished '
print t.fetch(row_key,[column_name])
class StarbaseClient01ConnectionTest(unittest.TestCase):
"""
Starbase Connection tests.
"""
#@print_info
def setUp(self):
self.connection = Connection(HOST, PORT, content_type='json')
self.table = self.connection.table(TABLE_NAME)
@print_info
def test_01_version(self):
res = self.connection.version
self.assertTrue(isinstance(res, dict))
return res
@print_info
def test_02_cluster_version(self):
res = self.connection.cluster_version
self.assertTrue(isinstance(res, text_type))
return res
@print_info
def test_03_cluster_status(self):
res = self.connection.cluster_status
self.assertTrue(isinstance(res, dict))
return res
if TEST_DELETE_TABLE:
@print_info
def test_04_drop_table_schema(self):
"""
Delete table schema. Deleting the table if it exists. After that checking if table still exists.
"""
# First testing for non-existent table
non_existent_res = self.connection.table('non-existent-table').drop()
self.assertEqual(503, non_existent_res)
res = None
if self.connection.table_exists(TABLE_NAME):
res = self.connection.table(TABLE_NAME).drop()
self.assertEqual(200, res) # Checking the status code
self.assertTrue(not self.connection.table_exists(TABLE_NAME)) # Checking for physical existence
return non_existent_res, res
if TEST_CREATE_TABLE:
@print_info
def test_05_create_table_schema(self):
"""
Create table schema. After creating the table we just check if it exists.
"""
# Success tests
res = None
if not self.connection.table_exists(TABLE_NAME):
columns = [COLUMN_FROM_USER, COLUMN_TO_USER, COLUMN_MESSAGE]
res = self.connection.table(TABLE_NAME).create(*columns)
self.assertTrue(self.connection.table_exists(TABLE_NAME))
# Now trying to create a table even if it exists.
columns = [COLUMN_FROM_USER, COLUMN_TO_USER, COLUMN_MESSAGE]
res_fail = self.connection.table(TABLE_NAME).create(*columns)
self.assertEqual(res_fail, False)
return res, res_fail
@print_info
def test_06_get_table_schema(self):
"""
Get table schema.
"""
# First testing for non existent table
non_existent_table = self.connection.table('non-existent-table')
self.assertTrue(non_existent_table.schema() is None)
# Now for existing one
res = self.table.schema()
self.assertTrue(res is not None)
return non_existent_table, res
@print_info
def test_07_table_list(self):
res = self.connection.tables()
self.assertTrue(isinstance(res, list))
self.assertTrue(TABLE_NAME in res)
return res
class StarbaseClient01ConnectionTest(unittest.TestCase):
"""
Starbase Connection tests.
"""
#@print_info
def setUp(self):
self.connection = Connection(HOST, PORT, content_type='json')
self.table = self.connection.table(TABLE_NAME)
@print_info
def test_01_version(self):
res = self.connection.version
self.assertTrue(isinstance(res, dict))
return res
@print_info
def test_02_cluster_version(self):
res = self.connection.cluster_version
self.assertTrue(isinstance(res, text_type))
return res
@print_info
def test_03_cluster_status(self):
res = self.connection.cluster_status
self.assertTrue(isinstance(res, dict))
return res
if TEST_DELETE_TABLE:
@print_info
def test_04_drop_table_schema(self):
"""
Delete table schema. Deleting the table if it exists. After that checking if table still exists.
"""
# First testing for non-existent table
non_existent_res = self.connection.table(
'non-existent-table').drop()
self.assertEqual(503, non_existent_res)
res = None
if self.connection.table_exists(TABLE_NAME):
res = self.connection.table(TABLE_NAME).drop()
self.assertEqual(200, res) # Checking the status code
self.assertTrue(not self.connection.table_exists(TABLE_NAME)
) # Checking for physical existence
return non_existent_res, res
if TEST_CREATE_TABLE:
@print_info
def test_05_create_table_schema(self):
"""
Create table schema. After creating the table we just check if it exists.
"""
# Success tests
res = None
if not self.connection.table_exists(TABLE_NAME):
columns = [COLUMN_FROM_USER, COLUMN_TO_USER, COLUMN_MESSAGE]
res = self.connection.table(TABLE_NAME).create(*columns)
self.assertTrue(self.connection.table_exists(TABLE_NAME))
# Now trying to create a table even if it exists.
columns = [COLUMN_FROM_USER, COLUMN_TO_USER, COLUMN_MESSAGE]
res_fail = self.connection.table(TABLE_NAME).create(*columns)
self.assertEqual(res_fail, False)
return res, res_fail
@print_info
def test_06_get_table_schema(self):
"""
Get table schema.
"""
# First testing for non existent table
non_existent_table = self.connection.table('non-existent-table')
self.assertTrue(non_existent_table.schema() is None)
# Now for existing one
res = self.table.schema()
self.assertTrue(res is not None)
return non_existent_table, res
@print_info
def test_07_table_list(self):
res = self.connection.tables()
self.assertTrue(isinstance(res, list))
self.assertTrue(TABLE_NAME in res)
return res
class HbaseConnection:
def __init__(self):
# self.connection = Connection(host="kdna.edison.re.kr", port="9090", user="******", password="******", secure=True, verify_ssl=False, retries=3, retry_delay=10)
self.connection = Connection(host="kdna.edison.re.kr", port="9090", user="******", password="******", secure=True, verify_ssl=False, retries=3, retry_delay=10)
def get_talbeList(self):
return self.connection.tables()
def create_table(self, table_name, *column_list):
table = self.connection.table(table_name)
if not table.exists():
table.create(*column_list)
# for item in table.columns():
# print(item)
return True
else:
return False
def add_columns(self, table_name, column_list):
table = self.connection.table(table_name)
if not table.exists():
return False
else:
table.add_columns(column_list)
# print(table.columns())
return True
# def batch_upload(self):
def drop_talbe(self, table_name):
table = self.connection.table(table_name)
if table.exists():
table.drop()
# def insert_table(self, table_name, data):
# table = self.connection.table(table_name)
# response = table.insert(data)
# print("insert one data : ", data, response)
def insert_batch(self, table_name, key_list, data_list):
# print("data size", len(key_list),len(data_list))
table = self.connection.table(table_name)
batch = table.batch()
if batch and len(key_list)>0 and len(data_list)>0:
for i in range(len(data_list)):
# print(i, " : ", key_list[i], data_list[i] )
batch.update(key_list[i], data_list[i])
response_return = batch.commit(finalize=True)
print(response_return)
def search_data_rowkey(self, table_name, rowkey):
table = self.connection.table(table_name)
result = table.fetch(rowkey)
# print(type(result))
# print(len(result))
# print(result)
return result
def search_data_rowkey_with_filter(self, table_name, start_rowkey, end_rowkey):
table = self.connection.table(table_name)
filter_configuration = {}
filter_configuration["type"] = "FilterList"
filter_configuration["op"] = "MUST_PASS_ALL"
hbase_filter1 = {}
hbase_filter1["type"] = "RowFilter"
hbase_filter1["op"] = "EQUAL"
comparator1 = {}
comparator1["type"] = "RegexStringComparator"
# comparator1["type"] = "BinaryComparator"
comparator1["value"] = start_rowkey
hbase_filter1["comparator"] = comparator1
hbase_filter2 = {}
hbase_filter2["type"] = "RowFilter"
hbase_filter2["op"] = "LESS_OR_EQUAL"
comparator2 = {}
comparator2["type"] = "RegexStringComparator"
comparator2["value"] = end_rowkey
hbase_filter2["comparator"] = comparator2
filter_configuration["filters"] = []
filter_configuration["filters"].append(hbase_filter1)
print(json.dumps(filter_configuration))
# f_string = '{"type": "RowFilter", "op": "GREATER_OR_EQUAL", "comparator": {"type": "BinaryComparator", "value": "1-9000000-A-A"}}'
# f_string = '{"type": "RowFilter", "op": "GREATER_OR_EQUAL", "comparator": { "type": "BinaryComparator", "value": "MTNfMTAwMTE3MjM4X1RfRw=="} }'
# f_string = '{"type": "FamilyFilter", "op": "EQUAL", "comparator": {"type": "ColumnPrefixFilter", "value": "SAMPLE:HG00566" } }'
# result = table.fetch_all_rows(with_row_id=True, filter_string=f_string)
# table._scanner(batch_size=5000)
result = table.fetch_all_rows(with_row_id=True, filter_string=json.dumps(filter_configuration))
# result = table.fetch_all_rows(with_row_id=True, filter_string=f_string)
print("scan filter end...")
return result
def search_data_columnkey(self, table_name, columnkey):
table = self.connection.table(table_name)
hbase_filter = {}
hbase_filter["type"] = "FamilyFilter"
hbase_filter["value"] = "SAMPLE:"+columnkey
f_string = '{"type": "FamilyFilter", "op": "EQUAL", "comparator": { "type": "BinaryComparator", "value": "HG00566"} }'
# f_string = '{"type": "FamilyFilter", "op": "EQUAL", "comparator": {"type": "ColumnPrefixFilter", "value": "SAMPLE:HG00566" } }'
result = table.fetch_all_rows(with_row_id=True, filter_string=f_string)
print(type(result))
print(next(result))
def alldata(self, table_name):
table = self.connection.table(table_name)
return table.fetch_all_rows(with_row_id=True, perfect_dict=True)
class HbaseClient:
#
def __init__(self):
self.connection = Connection(host=hbase_conf.hbase_host,
port=hbase_conf.hbase_port)
def table_list(self):
self.connection.tables()
def set_current_table(self, table_name):
self.table_name = table_name
self.current_table = self.connection.table(self.table_name)
# 删除表
def table_drop(self, table_name):
self.table_name = table_name
self.current_table = self.connection.table(self.table_name)
self.current_table.drop()
return True
# 创建表
def table_create(self, table_name, column):
self.table_name = table_name
self.current_table = self.connection.table(self.table_name)
self.current_table.create(column)
return True
# columns_qualifiers_list : "f:xxx" ,("f:xxx","f:zzz") ,["f:xxx","f:zzz"]
# 返回字典
def fetch(self, row_key, columns_qualifiers_list=None):
result = {}
if row_key is not None:
if columns_qualifiers_list is None:
result = self.current_table.fetch(row_key, perfect_dict=False)
else:
result = self.current_table.fetch(
row_key,
columns=columns_qualifiers_list,
perfect_dict=False)
if result is None:
return {}
return result
# 返回字符串或者None
def fetch_one_qualifier(self, row_key, columns_qualifiers):
if isinstance(columns_qualifiers, str) is False:
return None
return self.fetch(row_key,
columns_qualifiers).get(columns_qualifiers, None)
def remove(self, row_key, column=None, qualifier=None):
if row_key is None:
return False
elif column is None:
result = self.current_table.remove(row_key)
elif qualifier is None:
result = self.current_table.remove(row_key, column)
else:
result = self.current_table.remove(row_key, column, qualifier)
if result == 200:
return True
else:
return False
def insert(self, row_key, column, qualifier, value):
if row_key is None or column is None or qualifier is None:
return False
value_dict = {'%s:%s' % (column, qualifier): value}
result = self.current_table.insert(row_key, value_dict)
if result == 200:
return True
else:
return False
def increase(self, row_key, column, qualifier, num):
if row_key is None or column is None or qualifier is None:
return 0
column_qualifier = '%s:%s' % (column, qualifier)
old_num = struct.unpack(
'>Q',
bytes(self.fetch(row_key, column_qualifier)[column_qualifier]))[0]
new_num = old_num + num
self.insert(row_key, column, qualifier, struct.pack('>Q', new_num))
return new_num
def issuccessful(request):
if 200 <= request.status_code and request.status_code <= 299:
return True
else:
return False
tablename = 'fda_twitter_table'
baseurl = 'ec2-174-129-50-11.compute-1.amazonaws.com'
#baseurl = 'localhost'
connection = Connection(host='127.0.0.1', port=8080)
table = connection.table(tablename)
connection.tables()
print str(table.exists())
print table.columns()
quit()
# Delete table if it exists
#request = requests.get(baseurl + "/" + tablename + "/schema")
#print str(request.text)
#if issuccessful(request):
# print "Deleted table " + tablename
#else:
# print "Errored out. Status code was " + str(request.status_code) + "\n" + request.text
#quit()
# Create Messages Table
"""
Logging HTTP requests.
"""
import logging
logging.basicConfig(level=logging.DEBUG)
from starbase import Connection
c = Connection()
c.tables()
t = c.table('table4')
t.create('column1', 'column2', 'column3')
t.exists()
t.add_columns('column4', 'column5', 'column6', 'column7')
t.drop_columns('column6', 'column7')
t.insert(
'my-key-1',
{
'column1': {'key11': 'value 11', 'key12': 'value 12',
'key13': 'value 13'},
'column2': {'key21': 'value 21', 'key22': 'value 22'},
'column3': {'key32': 'value 31', 'key32': 'value 32'}
}
class HbaseIndex:
default_host = "kdna.edison.re.kr"
default_port = "9090"
default_userId = "tuser"
default_password = "******"
default_secure = True
default_verify_ssl = False
default_retries = 3
default_retry_deply = 10
def __init__(self, host, port, user, password):
if host == None:
self.host = self.default_host
if port == None:
self.port = self.default_port
if user == None:
self.user = self.default_userId
if password == None:
self.password = self.default_password
self.connection = Connection(self.host,
self.port,
self.user,
self.password,
secure=self.default_secure,
verify_ssl=self.default_verify_ssl,
retries=self.default_retries,
retry_delay=self.default_retry_deply)
def __del__(self):
print('finalizing')
self.flush()
def index(self, table_name, key_list, data_list):
table = self.connection.table(table_name)
self.batch = table.batch()
if self.batch and len(key_list) > 0 and len(data_list) > 0:
for i in range(len(data_list)):
self.batch.update(key_list[i], data_list[i])
def flush(self):
response_return = self.batch.commit(finalize=True)
print(response_return)
def tables(self):
return self.connection.tables()
def close(self):
print('OK')
self.flush()
def create_table(self, table_name, *column_list):
table = self.connection.table(table_name)
if not table.exists():
table.create(*column_list)
# for item in table.columns():
# print(item)
return True
else:
return False
def add_columns(self, table_name, column_list):
table = self.connection.table(table_name)
if not table.exists():
return False
else:
table.add_columns(column_list)
# print(table.columns())
return True
def drop(self, table_name):
table = self.connection.table(table_name)
if table.exists():
table.drop()
def search_data_rowkey(self, table_name, rowkey):
table = self.connection.table(table_name)
result = table.fetch(rowkey)
# print(type(result))
# print(len(result))
# print(result)
return result
def search_data_rowkey_with_filter(self, table_name, start_rowkey,
end_rowkey):
table = self.connection.table(table_name)
filter_configuration = {}
filter_configuration["type"] = "FilterList"
filter_configuration["op"] = "MUST_PASS_ALL"
hbase_filter1 = {}
hbase_filter1["type"] = "RowFilter"
hbase_filter1["op"] = "GREATER_OR_EQUAL"
comparator1 = {}
comparator1["type"] = "BinaryComparator"
# comparator["value"] = base64.b64encode(start_rowkey.encode("UTF_8"))
comparator1["value"] = start_rowkey
hbase_filter1["comparator"] = comparator1
hbase_filter2 = {}
hbase_filter2["type"] = "RowFilter"
hbase_filter2["op"] = "LESS_OR_EQUAL"
comparator2 = {}
comparator2["type"] = "BinaryComparator"
comparator2["value"] = end_rowkey
hbase_filter2["comparator"] = comparator2
filter_configuration["filters"] = []
filter_configuration["filters"].append(hbase_filter1)
# filter_configuration["filters"].append(hbase_filter2)
print(json.dumps(filter_configuration))
# f_string = '{"type": "RowFilter", "op": "GREATER_OR_EQUAL", "comparator": { "type": "BinaryComparator", "value": "MTNfMTAwMTE3MjM4X1RfRw=="} }'
# f_string = '{"type": "FamilyFilter", "op": "EQUAL", "comparator": {"type": "ColumnPrefixFilter", "value": "SAMPLE:HG00566" } }'
# result = table.fetch_all_rows(with_row_id=True, filter_string=f_string)
result = table.fetch_all_rows(
with_row_id=True, filter_string=json.dumps(filter_configuration))
print("scan filter end...")
return result
def search_data_columnkey(self, table_name, columnkey):
table = self.connection.table(table_name)
hbase_filter = {}
hbase_filter["type"] = "FamilyFilter"
hbase_filter["value"] = "SAMPLE:" + columnkey
f_string = '{"type": "FamilyFilter", "op": "EQUAL", "comparator": { "type": "BinaryComparator", "value": "HG00566"} }'
# f_string = '{"type": "FamilyFilter", "op": "EQUAL", "comparator": {"type": "ColumnPrefixFilter", "value": "SAMPLE:HG00566" } }'
result = table.fetch_all_rows(with_row_id=True, filter_string=f_string)
print(type(result))
print(next(result))
def alldata(self, table_name):
table = self.connection.table(table_name)
return table.fetch_all_rows(with_row_id=True, perfect_dict=True)
#!/usr/bin/env python
# this does work but requires rest interface to be turned on
# bin/hbase rest start -p 8070
# import cProfile
from starbase import Connection
import time
c = Connection(host='cloudsmall1',port=8070)
print str(c.tables())
t = c.table('speedtest:test0')
print str(t.columns())
keys = [ 'key test %d' % i for i in range(0, 5000) ]
values = [ "value-%(id)d %(ts)f" % { 'id': i, 'ts': time.time() } for i in range(0, 5000) ]
def build():
b = t.batch()
for i in range(0, 50):
key = keys[i]
value = values[i]
print key+" => "+value
b.insert(key, { 'f1': { 'x': value }, 'f2': { 'y': value }, 'f3': { 'z': value } })
b.commit()
def read():
for i in range(0, 50):
key = keys[i]
row = t.fetch(key)
print key+" is "+str(row)
startt = time.time()
build()
wstartt = time.time()
writet = time.time() - startt
