def creat_table(table_name="l_test_table"):
transport.open()
content_1 = Hbase.ColumnDescriptor(name='person:', maxVersions=2)
content_2 = Hbase.ColumnDescriptor(name='content:', maxVersions=2)
client.createTable(table_name, [content_1, content_2])
print client.getTableNames()
transport.close()
def open_spider(self, spider):
column_families = (Hbase.ColumnDescriptor(name=self.cf_basic,
maxVersions=1),
Hbase.ColumnDescriptor(name=self.cf_price,
maxVersions=1,
timeToLive=365 * 24 * 60 *
60))
self.hbase.create_table_if_not_exists(column_families)
def __create_table(self, table):
""" create table in hbase with column families
"""
columnFamilies = []
for columnFamily in self.columnFamilies:
name = Hbase.ColumnDescriptor(name=columnFamily)
columnFamilies.append(name)
self.client.createTable(table, columnFamilies)
def __init__(self, settings):
self.host = settings.get('HBASE_HOST')
self.port = settings.get('HBASE_PORT')
self.table = settings.get('HISTORY_TABLE')
self.hbase = HbaseWrapper(self.host, self.port, self.table)
column_families = (Hbase.ColumnDescriptor(name=self.column_family,
maxVersions=1,
timeToLive=86400), )
self.hbase.create_table_if_not_exists(column_families)
def createTable(self, columnFamilies):
#判断 表释放存在 存在不创建 , 不存在 创建
tables = self.client.getTableNames()
if self.tableName not in tables:
# 定义 列族 user, address,
columns = []
for column in columnFamilies:
columns.append(Hbase.ColumnDescriptor(column))
self.client.createTable(self.tableName, columns)
print('创建成功')
else:
print("表明已经创建了")
print(self.client.isTableEnabled(self.tableName))
def __create_table(self, table):
"""
create table in hbase with column families
:param table: fr_test_hbase:fr_test
:return:
"""
columnFamilies = []
for columnFamily in self.columnFamilies:
name = Hbase.ColumnDescriptor(name=columnFamily)
columnFamilies.append(name)
table = table.encode('utf-8')
print(type(table), type(columnFamilies))
self.client.createTable(table, columnFamilies)
def createTable(self, columnFamilies):
try:
tables = self.client.getTableNames()
if self.dbname in tables:
return False
TempcolumnFamilies = []
for columnFamily in columnFamilies:
name = Hbase.ColumnDescriptor(name=columnFamily)
TempcolumnFamilies.append(name)
self.client.createTable(self.dbname, TempcolumnFamilies)
return True
except (Hbase.IOError, Hbase.TException, Hbase.TApplicationException,
Hbase.IllegalArgument) as e:
logInfo('createTable')
logInfo(e)
print(e)
return False
def __init__(self, table, column_families, file_lock, put_num=0):
"""
初始化一个 HBase Table
:param table: 表的名字,比如 b'hb_charts'
:param column_families: 表的列族,比如 [b'cf1', b'cf2']
:param host: host地址
:param port: thrift服务端口
"""
conf = configparser.ConfigParser()
conf.read(CONFIG_FILE)
self.host = conf.get("hbase", "host")
self.port = int(conf.get("hbase", "port"))
# Connect to HBase Thrift server
self.transport = TTransport.TBufferedTransport(
TSocket.TSocket(self.host, self.port))
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
# Create and open the client connection
self.client = Hbase.Client(self.protocol)
self.transport.open()
self.file_lock = file_lock
self.put_num = put_num
# set table and column families
self.table = table
self.columnFamilies = column_families
tables = self.client.getTableNames()
if self.table not in tables:
cf = []
for columnFamily in self.columnFamilies:
name = Hbase.ColumnDescriptor(name=columnFamily)
cf.append(name)
self.client.createTable(table, cf)
try:
transport = TSocket.TSocket(sys.argv[1], int(sys.argv[2]))
transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = Hbase.Client(protocol)
transport.open()
tableName = 'AdventureWorks_Person_Contact'
tableExists = False
tableNames = client.getTableNames()
for table in tableNames:
if table == tableName:
tableExists = True
if tableExists != True:
columnFamilies = []
columnFamilies.append(Hbase.ColumnDescriptor(name='info'))
columnFamilies.append(Hbase.ColumnDescriptor(name='contact'))
columnFamilies.append(Hbase.ColumnDescriptor(name='others'))
client.createTable(tableName, columnFamilies)
mutationsbatch = []
mutations = [
Hbase.Mutation(column='info:FULLNAME', value='Gustavo Achong'),
Hbase.Mutation(column='info:AGE', value='38'),
Hbase.Mutation(column='contact:EMAILID',
value='*****@*****.**'),
Hbase.Mutation(column='contact:PHONE', value='398-555-0132'),
Hbase.Mutation(column='others:MODIFIEDDATE',
value='5/16/2005 4:33:33 PM')
]
mutationsbatch.append(Hbase.BatchMutation(row='1',
mutations=mutations))
transport.open()
#########################################
# Define Table Name and Column Family Name Here
#########################################
tablename = 'haztable5'
cfname = 'cf1'
tableexists = 0
tables = client.getTableNames()
for table in tables:
if table == tablename.encode('ascii'):
print("TABLE ALREADY EXISTS")
tableexists = 1
if tableexists == 0:
# Create a new table if it does not exist
client.createTable(tablename.encode('ascii'),
[Hbase.ColumnDescriptor(name=cfname.encode('ascii'))])
print("Created table", tablename, "with column familiy", cfname)
# Get list of tables from HBase and print
tables = client.getTableNames()
print("List of current user tables in HBase:")
for table in tables:
print("Table: ", table.decode('ascii'))
# This closes up the socket and frees up the resources on the Thrift server.
transport.close()
def open_spider(self, spider):
column_families = (Hbase.ColumnDescriptor(name=self.cf_basic,
maxVersions=1), )
self.hbase.create_table_if_not_exists(column_families)
# Disable and delete table if table exists already
tables = client.getTableNames()
found = False
for table in tables:
if table == tablename:
found = True
if found == True:
client.disableTable(tablename)
client.deleteTable(tablename)
# Create the table
client.createTable(tablename, [Hbase.ColumnDescriptor(name=cfname)])
# Create a message for every line in every work of Shakespeare
sourceDir = "shakespeare"
for filename in os.listdir(sourceDir):
shakespeare = open(os.path.join(sourceDir, filename), "rb")
linenumber = 0
# Create a list of mutations per work of Shakespeare
mutationsbatch = []
for line in shakespeare:
rowkey = username + "-" + filename + "-" + str(linenumber).zfill(6)
