def _connect(self, host, port) -> Client:
socket = TSocket(host, port)
transport = TFramedTransport(socket)
protocol = TCompactProtocol(transport)
client = Client(protocol)
transport.open()
return client
def connect_to_thrudex(self):
socket = TSocket('localhost', THRUDEX_PORT)
transport = TFramedTransport(socket)
protocol = TBinaryProtocol(transport)
self.thrudex = Thrudex.Client(protocol)
transport.open()
self.thrudex.admin("create_index", THRUDEX_INDEX)
def connect_to_thrudoc(self):
socket = TSocket('localhost', THRUDOC_PORT)
transport = TFramedTransport(socket)
protocol = TBinaryProtocol(transport)
self.thrudoc = Thrudoc.Client(protocol)
transport.open()
class ThriftClient(object):
def __init__(self, host, port, service_class, **kwargs):
self._host = host
self._port = port
self._connection_timeout = kwargs.get('connection_time', DEFAULT_TCONNECTION_TIMEOUT)
self._receive_timeout = kwargs.get('receive_timeout', DEFAULT_TRECEIVE_TIMEOUT)
self._send_timeout = kwargs.get('send_timeout', DEFAULT_TSEND_TIMEOUT)
self._transport_class = kwargs.get('transport_class', TFRAMED_TRANSPORT_NAME)
self._service_class = service_class
self._connect = False
def get_service(self):
self.__connect_if_necessary()
return self._client
def __connect_if_necessary(self):
if self._connect:
return
self._socket = TSocket(self._host, self._port)
if (self._transport_class == TBUFFERED_TRANSPORT_NAME):
self._transport = TBufferedTransport(self._socket)
else:
self._transport = TFramedTransport(self._socket)
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._client = self._service_class.Client(self._protocol)
self._connect = True
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(
buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[
prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError, msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError,msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
class ThriftHbase(object):
def __init__(self):
__socket = TSocket.TSocket(host=HBASE_HOST, port=HBASE_PORT)
__socket.setTimeout(50000)
self.transport = TFramedTransport(__socket)
# transport = TTransport.TBufferedTransport(socket)
__protocol = TCompactProtocol.TCompactProtocol(self.transport)
# protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.client = Hbase.Client(__protocol)
self.transport.open()
def create_table_hbase(self, table_name, column_list=None):
column_families = list()
# 定义列族
if column_list:
for each in column_list:
column = ColumnDescriptor(name=each)
column_families.append(column)
else:
column = ColumnDescriptor(name=HBASE_COLUM_FAMILY)
column_families.append(column)
# 创建表
self.client.createTable(table_name, column_families)
def delete_table_hbase(self, table_name):
self.client.disableTable(table_name)
self.client.deleteTable(table_name)
def insert_row_into_hbase(self, table_name, data):
mutation_list = list()
row_key = str(data["_id"])
# del data["_id"]
if "d" in data:
del data["d"]
for key, value in data.items():
if key == "_id":
continue
else:
mutation = Mutation(column="{}:{}".format(
HBASE_COLUM_FAMILY, str(key)),
value=str(value))
mutation_list.append(mutation)
self.client.mutateRow(table_name, row_key, mutation_list)
# batch_mutation1 = BatchMutation("ROW1", mutation_list)
# batch_mutation2 = BatchMutation("ROW1", mutation_list)
# batch_mutation_list = [batch_mutation1, batch_mutation2]
# client.mutateRows(table_name, batch_mutation_list)
def delete_row_from_hbase(self, table_name, row):
self.client.deleteAllRow(table_name, row)
def get_row_from_hbase(self, table_name, row):
return self.client.getRow(table_name, row)
def Init(self):
try:
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
self.service = DedupService.Client(protocol)
return True
except Exception as ex:
print 'Exception:%s' % (ex)
return False
def Extract(url_template, template_type, depth, body_file):
try:
global SVR_PORT
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = ExtractorService.Client(protocol)
extract_item = ExtractItem()
extract_item.url = 'http://api.wap.58.com/api/info/infolist/bj/zufang/1/25/?pic=1'
extract_item.url_template = url_template
extract_item.depth = depth
extract_item.template_type = template_type
file_in = body_file
f = open(file_in, 'r')
extract_item.body = f.read()
f.close()
matched_result_item = service.extract_sync(extract_item)
print 'len(sub_result_list):%d' % len(
matched_result_item.sub_result_list)
transport.close()
if matched_result_item.is_ok == False:
print 'Err:%s' % (matched_result_item.err_info)
for key, value in matched_result_item.self_result.iteritems():
for v in value:
print '[%s]\t%s' % (key, v)
for item in matched_result_item.sub_result_list:
print '-------------------'
for key, value in item.iteritems():
print '[%s]\t%s' % (key, value[0])
'''
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = StaticLinkBaseService.Client(protocol)
service.upload_extract(extract_item, matched_result_item)
transport.close()
'''
return True
except Exception as ex:
print "Error:%s" % (ex)
return False
def LoadMongoDbTask(task_id, is_add_task):
try:
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = StaticLinkBaseService.Client(protocol)
service.load_mongodb_task(task_id, is_add_task)
transport.close()
return True
except Exception as ex:
print "Error:%s" % (ex)
return True
def LoadSeedByUrl(seed_url, is_add_link):
try:
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = LinkBaseService.Client(protocol)
service.load_seed_by_url(seed_url, is_add_link)
transport.close()
return True
except Exception as ex:
print "Error:%s" % (ex)
return True
def LoadTemplate(url_template, template_type):
try:
global SVR_PORT
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = ExtractorService.Client(protocol)
ret = service.load_template(url_template, template_type)
transport.close()
return True
except Exception as ex:
print "Error:%s" % (ex)
return True
def UploadBody():
try:
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = StaticLinkBaseService.Client(protocol)
req_item = DownloadReqItem(url = 'http://www.ganji.com')
prop_item = DownloadPropItem(seed_url = 'http://so.iautos.cn/so.jsp?modeltype2=%D0%A1%D0%CD%B3%B5&pageindex=1', depth = 0)
downloaded_body_item = DownloadedBodyItem(req_item, prop_item, is_ok = True, body = '123')
service.upload_body(downloaded_body_item)
transport.close()
return True
except Exception as ex:
print "Error:%s" % (ex)
return True
class ThriftClient(threading.local):
@Argument('interface', type=ArsInterface)
@Argument('transport_factory', type=FunctionType)
def __init__(self, interface, transport_factory):
super(ThriftClient, self).__init__()
self._interface = interface
self._transport_factory = transport_factory
self._started = False
def _wrap_procedure(self, procedure):
names = [parameter.name for parameter in procedure.parameters]
sign = Signature(names)
for param in procedure.parameters:
if param.optional:
sign.arguments[param.name].mode = ArgumentMode.OPTIONAL
values_type = sign.Values
def proc(*args, **kwargs):
if not self._started:
self.start()
values = values_type(*args, **kwargs)
try:
return self._processor.call(procedure, values.named,
self._protocol, self._protocol)
except TTransportException:
self.stop()
self.start()
return self._processor.call(procedure, values.named,
self._protocol, self._protocol)
except IOError as e:
if e[0] == errno.EPIPE:
self.stop()
self.start()
return self._processor.call(procedure, values.named,
self._protocol, self._protocol)
else:
raise
proc.func_name = procedure.name
return proc
def wrap_all(self):
for service in self._interface.services:
for procedure in service.procedures:
procedure.implementation = self._wrap_procedure(procedure)
def unwrap_all(self):
for service in self._interface.services:
for procedure in service.procedures:
procedure.implementation = None
def start(self):
if self._started:
self.stop()
self._transport = TFramedTransport(self._transport_factory())
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._processor = ThriftProcessor(self._interface)
self._started = True
def stop(self):
if self._started:
self._transport.close()
self._started = False
sys.path.append('../gen-py')
sys.path.append('/usr/lib/python2.6/site-packages/')
from conf_crawler import DownloaderService
from conf_crawler import DCService
from conf_crawler.ttypes import *
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.transport.TTransport import TFramedTransport
transport = TSocket.TSocket('localhost', 44002)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
client = DCService.Client(protocol)
def PushDownloadTask():
try:
url = 'http://www.coo8.com/interfaces/showReviewsByGoodsId.action<%param%>flag=all&goodsId=P145484&pageIndex=1'
referer = "http://www.coo8.com/product/145484.html"
download_task = DownloadTask()
download_task.req_item = DownloadReqItem()
download_task.req_item.url = url
download_task.req_item.referer = referer
download_task.req_item.time_out = 1000
download_task.prop_item = DownloadPropItem()
class ThriftClient(threading.local):
@Argument('interface', type=ArsInterface)
@Argument('transport_factory', type=FunctionType)
def __init__(self, interface, transport_factory):
super(ThriftClient, self).__init__()
self._interface = interface
self._transport_factory = transport_factory
self._started = False
def _wrap_procedure(self, procedure):
names = [parameter.name for parameter in procedure.parameters]
sign = Signature(names)
for param in procedure.parameters:
if param.optional:
sign.arguments[param.name].mode = ArgumentMode.OPTIONAL
values_type = sign.Values
def proc(*args, **kwargs):
if not self._started:
self.start()
values = values_type(*args, **kwargs)
try:
return self._processor.call(procedure, values.named, self._protocol, self._protocol)
except TTransportException:
self.stop()
self.start()
return self._processor.call(procedure, values.named, self._protocol, self._protocol)
except IOError as e:
if e[0] == errno.EPIPE:
self.stop()
self.start()
return self._processor.call(procedure, values.named, self._protocol, self._protocol)
else:
raise
proc.func_name = procedure.name
return proc
def wrap_all(self):
for service in self._interface.services:
for procedure in service.procedures:
procedure.implementation = self._wrap_procedure(procedure)
def unwrap_all(self):
for service in self._interface.services:
for procedure in service.procedures:
procedure.implementation = None
def start(self):
if self._started:
self.stop()
self._transport = TFramedTransport(self._transport_factory())
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._processor = ThriftProcessor(self._interface)
self._started = True
def stop(self):
if self._started:
self._transport.close()
self._started = False