def init_client(self):
transport = TSocket.TSocket(self.address, self.port)
transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
transport.open()
client = THBaseService.Client(protocol)
return client
def connect():
transport = TTransport.TBufferedTransport(TSocket.TSocket('127.0.0.1', 9090))
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = THBaseService.Client(protocol)
transport.open()
try:
yield client
finally:
transport.close()
def _hbaseClient(self):
transport = TSocket.TSocket(self.host, self.port)
transport.setTimeout(self.timeout)
transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = THBaseService.Client(protocol)
transport.open()
return client, transport
def __init__(self,host,table):
host = host.split(':')
self.port = int(host[1])
self.host = host[0]
self.table = table
#初始化Thrift连接
transport = TSocket.TSocket(self.host, self.port) #建立socket连接
#transport.setTimeout(5000)
self.transport = TTransport.TBufferedTransport(transport) #设置缓存，非常重要
protocol = TBinaryProtocol.TBinaryProtocol(transport) #包装在协议中
self.client = THBaseService.Client(protocol) #创建客户端使用规定的解码协议
#连接成功
print "Connect to {0}:{1}".format(self.host, self.port)
transport.open()
def __init__(self, ip="localhost", port=9090, rest_port=8080, conf=None):
if conf is None:
self.port = port
self.ip = ip
self.rest_port = rest_port
else:
self.port = conf["port"]
self.ip = conf["host"]
self.rest_port = conf["rest_port"]
self.transport = TSocket.TSocket(self.ip, self.port)
self.transport = TTransport.TBufferedTransport(self.transport)
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = THBaseService.Client(self.protocol)
self.transport.open()
self.base_url = "http://%s:%d" % (self.ip, self.rest_port)
def __init__(self, settings):
self.DB_URI = settings['HBASE_URI']
self.DB_PORT = settings['HBASE_PORT']
self.TB_INFO = settings['TB_INFO'].encode()
self.TB_NEWS = settings['TB_NEWS'].encode()
# 连接数据库表
socket = TSocket.TSocket(self.DB_URI, self.DB_PORT)
self.transport = TTransport.TFramedTransport(socket)
protocol = TCompactProtocol.TCompactProtocol(self.transport)
self.client = THBaseService.Client(protocol)
self.transport.open()
# 将爬虫开始的信息存入数据库
self.spider_info_row_key, start_put = gen_start_spider_info()
self.client.put(self.TB_INFO, start_put)
def createConnection():
socket = TSocket.TSocket(host, port)
if framed:
transport = TTransport.TFramedTransport(socket)
else:
if secureMode:
transport = TTransport.TSaslClientTransport(
socket, host, saslServiceName)
else:
transport = TTransport.TBufferedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = THBaseService.Client(protocol)
transport.open()
return client, transport
def __init__(self, conf):
print "init dataprocess"
self._section_name = "DataProcess"
self.thrift_host = conf.get(self._section_name, "HBASE_THRIFT_HOST", required=True)
self.port = conf.getint(self._section_name, "HBASE_THRIFT_PORT", required=True)
self.transport = TSocket.TSocket(self.thrift_host, self.port)
self.transport = TTransport.TBufferedTransport(self.transport)
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = THBaseService.Client(self.protocol)
self.table_name = conf.get(self._section_name, "HBASE_TABLE_ID", required=True)
self.id_family = conf.get(self._section_name, "HBASE_FAMILY_ID")
self.id_qualifier = conf.get(self._section_name, "HBASE_QUALIFIER_ID")
try:
self.transport.open()
except TTransport.TTransportException:
raise Exception('Error to open thrift service on {host} at port {port}'.format(host=self.thrift_host, port=self.port))
def checkAndPut(self, response):
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
client = THBaseService.Client(protocol)
self.transport.open()
rowKey = response.url
get = TGet(rowKey)
result = client.get(self.hbaseTable, get)
update = True
havePage = False
for col in result.columnValues:
if col.family == self.hbaseTableCf and col.qualifier == self.hbaseColQualifier:
havePage = True
dom = soupparser.fromstring(col.value)
colvalue = etree.tostring(dom.xpath("//body")[0])
dom = soupparser.fromstring(response.body)
responsevalue = etree.tostring(dom.xpath("//body")[0])
if colvalue == responsevalue:
update = False
print "Url not update " + response.url
else:
col.value = response.body
if update:
# if result.columnValues :
# if havePage:
# client.put(self.hbaseTable, TPut(rowKey,result.columnValues))
# else:
# val = TColumnValue(self.hbaseTableCf, self.hbaseColQualifier, response.body)
# client.put(self.hbaseTable, TPut(rowKey,result.columnValues + [val]))
#
# else:
# val = TColumnValue(self.hbaseTableCf, self.hbaseColQualifier, response.body)
# client.put(self.hbaseTable, TPut(rowKey,[val]))
val = TColumnValue(self.hbaseTableCf, self.hbaseColQualifier,
response.body)
client.put(self.hbaseTable, TPut(rowKey, [val]))
self.transport.close()
return response
def _connect(self):
try:
self._current_host_port = random.choice(
self._host_ports_pool).split(':')
self._sock = TSocket.TSocket(host=self._current_host_port[0],
port=self._current_host_port[1])
self._transport = TTransport.TFramedTransport(self._sock)
self._protocol = TCompactProtocol(self._transport)
self._client = THBaseService.Client(self._protocol)
self._transport.open()
except Exception, e:
TDDCLogging.error(e)
current_host_port = ':'.join(self._current_host_port)
self._host_ports_pool.remove(current_host_port)
if len(self._host_ports_pool) > 0:
TDDCLogging.warning(
'HBase Server Exception. Now Is Reconnecting.')
else:
TDDCLogging.warning(
'HBase Server Fatal Error. Please Check It.')
gevent.sleep(30)
self._host_ports_pool = list(self._host_ports)
TDDCLogging.warning('Retry Connecting HHase.')
self._reconnect()
def get_client(self):
transport = TSocket.TSocket(HBASE_LOG_ADDERSS, HBASE_LOG_PORT)
transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
transport.open()
self.client = THBaseService.Client(protocol)
from hbase.ttypes import *
print "Thrift2 Demo"
print "This demo assumes you have a table called \"example\" with a column family called \"family1\""
host = "localhost"
port = 9090
framed = False
socket = TSocket.TSocket(host, port)
if framed:
transport = TTransport.TFramedTransport(socket)
else:
transport = TTransport.TBufferedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = THBaseService.Client(protocol)
transport.open()
table = "example"
put = TPut(row="row1",
columnValues=[
TColumnValue(family="family1",
qualifier="qualifier1",
value="value1")
])
print "Putting:", put
client.put(table, put)
get = TGet(row="row1")
def OnRtnDepthMarketData(self, pDepthMarketData):
#print pDepthMarketData.InstrumentID,pDepthMarketData.UpdateTime , pDepthMarketData.UpdateMillisec ,pDepthMarketData.LastPrice ,pDepthMarketData.Volume ,pDepthMarketData.AskPrice1 ,pDepthMarketData.AskVolume1 ,pDepthMarketData.BidPrice1 ,pDepthMarketData.BidVolume1
############################################################################################################################################################################
try:
##
## 这里是把行情保存到hbase数据库中，
## 请修改下
### host port
### table 请在hbase shell中自行创建
##
## 补充说明：如果上生产系统，此处需要修改下，每次都创建socket close效率不高，把它移到外面去。（此处只是演示，so...）
##
##
host = "127.0.0.1" # todo 改成hbase的地址与端口
port = 9090 # todo
framed = False
socket = TSocket.TSocket(host, port)
if framed:
transport = TTransport.TFramedTransport(socket)
else:
transport = TTransport.TBufferedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = THBaseService.Client(protocol)
transport.open()
table = "marketdata" # todo
rowkey = pDepthMarketData.InstrumentID + pDepthMarketData.TradingDay + pDepthMarketData.UpdateTime + '_' + str(
self.rowkeyindex)
self.rowkeyindex += 1
put = TPut(
row=rowkey,
columnValues=[
TColumnValue(family="data",
qualifier="InstrumentID",
value=pDepthMarketData.InstrumentID),
TColumnValue(family="data",
qualifier="TradingDay",
value=pDepthMarketData.TradingDay),
TColumnValue(family="data",
qualifier="UpdateTime",
value=pDepthMarketData.UpdateTime),
TColumnValue(family="data",
qualifier="UpdateMillisec",
value=str(pDepthMarketData.UpdateMillisec)),
TColumnValue(family="data",
qualifier="LastPrice",
value=str(pDepthMarketData.LastPrice)),
TColumnValue(family="data",
qualifier="Volume",
value=str(pDepthMarketData.Volume)),
TColumnValue(family="data",
qualifier="AskPrice1",
value=str(pDepthMarketData.AskPrice1)),
TColumnValue(family="data",
qualifier="AskVolume1",
value=str(pDepthMarketData.AskVolume1)),
TColumnValue(family="data",
qualifier="BidPrice1",
value=str(pDepthMarketData.BidPrice1)),
TColumnValue(family="data",
qualifier="BidPrice1",
value=str(pDepthMarketData.BidVolume1)),
])
client.put(table, put)
transport.close()
except Exception, e:
print e