def connect_to_thrudex(self):
socket = TSocket('localhost', THRUDEX_PORT)
transport = TFramedTransport(socket)
protocol = TBinaryProtocol(transport)
self.thrudex = Thrudex.Client(protocol)
transport.open()
self.thrudex.admin("create_index", THRUDEX_INDEX)
def connect_to_thrudoc(self):
socket = TSocket('localhost', THRUDOC_PORT)
transport = TFramedTransport(socket)
protocol = TBinaryProtocol(transport)
self.thrudoc = Thrudoc.Client(protocol)
transport.open()
def _connect(self, host, port) -> Client:
socket = TSocket(host, port)
transport = TFramedTransport(socket)
protocol = TCompactProtocol(transport)
client = Client(protocol)
transport.open()
return client
def serve(self):
self._server_transport.listen()
while True:
with self._stopped_mu:
if self._stopped:
return
client = self._server_transport.accept()
framed = TFramedTransport(client)
iprot = self._protocol_factory.get_protocol(framed)
oprot = self._protocol_factory.get_protocol(framed)
processor = self._processor_factory.get_processor(framed)
try:
while True:
with self._stopped_mu:
if self._stopped:
break
processor.process(iprot, oprot)
except TTransportException:
continue
except Exception as e:
print(e)
break
framed.close()
class ThriftClient(object):
def __init__(self, host, port, service_class, **kwargs):
self._host = host
self._port = port
self._connection_timeout = kwargs.get('connection_time', DEFAULT_TCONNECTION_TIMEOUT)
self._receive_timeout = kwargs.get('receive_timeout', DEFAULT_TRECEIVE_TIMEOUT)
self._send_timeout = kwargs.get('send_timeout', DEFAULT_TSEND_TIMEOUT)
self._transport_class = kwargs.get('transport_class', TFRAMED_TRANSPORT_NAME)
self._service_class = service_class
self._connect = False
def get_service(self):
self.__connect_if_necessary()
return self._client
def __connect_if_necessary(self):
if self._connect:
return
self._socket = TSocket(self._host, self._port)
if (self._transport_class == TBUFFERED_TRANSPORT_NAME):
self._transport = TBufferedTransport(self._socket)
else:
self._transport = TFramedTransport(self._socket)
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._client = self._service_class.Client(self._protocol)
self._connect = True
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(
buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[
prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError, msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
def __init__(self):
__socket = TSocket.TSocket(host=HBASE_HOST, port=HBASE_PORT)
__socket.setTimeout(50000)
self.transport = TFramedTransport(__socket)
# transport = TTransport.TBufferedTransport(socket)
__protocol = TCompactProtocol.TCompactProtocol(self.transport)
# protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.client = Hbase.Client(__protocol)
self.transport.open()
def start(self):
if self._started:
self.stop()
self._transport = TFramedTransport(self._transport_factory())
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._processor = ThriftProcessor(self._interface)
self._started = True
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError,msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
class ThriftHbase(object):
def __init__(self):
__socket = TSocket.TSocket(host=HBASE_HOST, port=HBASE_PORT)
__socket.setTimeout(50000)
self.transport = TFramedTransport(__socket)
# transport = TTransport.TBufferedTransport(socket)
__protocol = TCompactProtocol.TCompactProtocol(self.transport)
# protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.client = Hbase.Client(__protocol)
self.transport.open()
def create_table_hbase(self, table_name, column_list=None):
column_families = list()
# 定义列族
if column_list:
for each in column_list:
column = ColumnDescriptor(name=each)
column_families.append(column)
else:
column = ColumnDescriptor(name=HBASE_COLUM_FAMILY)
column_families.append(column)
# 创建表
self.client.createTable(table_name, column_families)
def delete_table_hbase(self, table_name):
self.client.disableTable(table_name)
self.client.deleteTable(table_name)
def insert_row_into_hbase(self, table_name, data):
mutation_list = list()
row_key = str(data["_id"])
# del data["_id"]
if "d" in data:
del data["d"]
for key, value in data.items():
if key == "_id":
continue
else:
mutation = Mutation(column="{}:{}".format(
HBASE_COLUM_FAMILY, str(key)),
value=str(value))
mutation_list.append(mutation)
self.client.mutateRow(table_name, row_key, mutation_list)
# batch_mutation1 = BatchMutation("ROW1", mutation_list)
# batch_mutation2 = BatchMutation("ROW1", mutation_list)
# batch_mutation_list = [batch_mutation1, batch_mutation2]
# client.mutateRows(table_name, batch_mutation_list)
def delete_row_from_hbase(self, table_name, row):
self.client.deleteAllRow(table_name, row)
def get_row_from_hbase(self, table_name, row):
return self.client.getRow(table_name, row)
def __connect_if_necessary(self):
if self._connect:
return
self._socket = TSocket(self._host, self._port)
if (self._transport_class == TBUFFERED_TRANSPORT_NAME):
self._transport = TBufferedTransport(self._socket)
else:
self._transport = TFramedTransport(self._socket)
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._client = self._service_class.Client(self._protocol)
self._connect = True
def Init(self):
try:
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
self.service = DedupService.Client(protocol)
return True
except Exception as ex:
print 'Exception:%s' % (ex)
return False
def Extract(url_template, template_type, depth, body_file):
try:
global SVR_PORT
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = ExtractorService.Client(protocol)
extract_item = ExtractItem()
extract_item.url = 'http://api.wap.58.com/api/info/infolist/bj/zufang/1/25/?pic=1'
extract_item.url_template = url_template
extract_item.depth = depth
extract_item.template_type = template_type
file_in = body_file
f = open(file_in, 'r')
extract_item.body = f.read()
f.close()
matched_result_item = service.extract_sync(extract_item)
print 'len(sub_result_list):%d' % len(
matched_result_item.sub_result_list)
transport.close()
if matched_result_item.is_ok == False:
print 'Err:%s' % (matched_result_item.err_info)
for key, value in matched_result_item.self_result.iteritems():
for v in value:
print '[%s]\t%s' % (key, v)
for item in matched_result_item.sub_result_list:
print '-------------------'
for key, value in item.iteritems():
print '[%s]\t%s' % (key, value[0])
'''
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = StaticLinkBaseService.Client(protocol)
service.upload_extract(extract_item, matched_result_item)
transport.close()
'''
return True
except Exception as ex:
print "Error:%s" % (ex)
return False
def main(argv):
p = argparse.ArgumentParser()
add_common_args(p)
# Since THeaderTransport acts as framed transport when detected frame, we
# cannot use --transport=framed as it would result in 2 layered frames.
p.add_argument('--override-transport')
p.add_argument('--override-protocol')
args = p.parse_args()
assert args.protocol == 'header'
assert args.transport == 'buffered'
assert not args.ssl
sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
if not args.override_transport or args.override_transport == 'buffered':
trans = TBufferedTransport(sock)
elif args.override_transport == 'framed':
print('TFRAMED')
trans = TFramedTransport(sock)
else:
raise ValueError('invalid transport')
trans.open()
if not args.override_protocol or args.override_protocol == 'binary':
proto = TBinaryProtocol(trans)
elif args.override_protocol == 'compact':
proto = TCompactProtocol(trans)
else:
raise ValueError('invalid transport')
test_void(proto)
test_void(proto)
trans.close()
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
if conf.transport_mode == 'http':
mode = THttpClient(conf.http_url)
mode.set_verify(conf.validate)
else:
if conf.use_ssl:
try:
from ssl import PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
try:
from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
mode = TSSLSocketWithWildcardSAN(conf.host, conf.port, validate=conf.validate, ca_certs=conf.ca_certs,
keyfile=conf.keyfile, certfile=conf.certfile, ssl_version=PROTOCOL_SSLv23)
else:
mode = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
mode.setTimeout(conf.timeout_seconds * 1000.0)
if conf.transport_mode == 'http':
if conf.use_sasl and conf.mechanism != 'PLAIN':
mode.set_kerberos_auth(service=conf.kerberos_principal)
else:
mode.set_basic_auth(conf.username, conf.password)
if conf.transport_mode == 'socket' and conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", str(conf.host))
saslc.setAttr("service", str(conf.kerberos_principal))
if conf.mechanism == 'PLAIN':
saslc.setAttr("username", str(conf.username))
saslc.setAttr("password", str(conf.password)) # Defaults to 'hue' for a non-empty string unless using LDAP
else:
saslc.setAttr("maxbufsize", SASL_MAX_BUFFER.get())
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, conf.mechanism, mode)
elif conf.transport == 'framed':
transport = TFramedTransport(mode)
else:
transport = TBufferedTransport(mode)
protocol = TBinaryProtocol(transport)
if conf.multiple:
protocol = TMultiplexedProtocol(protocol, conf.service_name)
service = conf.klass(protocol)
return service, protocol, transport
def LoadMongoDbTask(task_id, is_add_task):
try:
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = StaticLinkBaseService.Client(protocol)
service.load_mongodb_task(task_id, is_add_task)
transport.close()
return True
except Exception as ex:
print "Error:%s" % (ex)
return True
def LoadSeedByUrl(seed_url, is_add_link):
try:
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = LinkBaseService.Client(protocol)
service.load_seed_by_url(seed_url, is_add_link)
transport.close()
return True
except Exception as ex:
print "Error:%s" % (ex)
return True
def LoadTemplate(url_template, template_type):
try:
global SVR_PORT
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = ExtractorService.Client(protocol)
ret = service.load_template(url_template, template_type)
transport.close()
return True
except Exception as ex:
print "Error:%s" % (ex)
return True
def start(self):
if self._started:
self.stop()
self._transport = TFramedTransport(self._transport_factory())
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._processor = ThriftProcessor(self._interface)
self._started = True
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
if conf.use_ssl:
sock = TSSLSocket(conf.host,
conf.port,
validate=conf.validate,
ca_certs=conf.ca_certs,
keyfile=conf.keyfile,
certfile=conf.certfile)
else:
sock = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
sock.setTimeout(conf.timeout_seconds * 1000.0)
if conf.transport_mode == 'http':
sock = THttpClient(conf.http_url, cert_validate=conf.validate)
if conf.use_sasl and conf.mechanism != 'PLAIN':
sock.set_kerberos_auth()
else:
sock.set_basic_auth(conf.username, conf.password)
transport = TBufferedTransport(sock)
elif conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", str(conf.host))
saslc.setAttr("service", str(conf.kerberos_principal))
if conf.mechanism == 'PLAIN':
saslc.setAttr("username", str(conf.username))
saslc.setAttr(
"password", str(conf.password)
) # defaults to hue for a non-empty string unless using ldap
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, conf.mechanism, sock)
elif conf.transport_mode == 'framed':
transport = TFramedTransport(sock)
else:
transport = TBufferedTransport(sock)
protocol = TBinaryProtocol(transport)
if conf.multiple:
protocol = TMultiplexedProtocol(protocol, conf.service_name)
service = conf.klass(protocol)
return service, protocol, transport
def UploadBody():
try:
transport = TSocket.TSocket(SVR_IP, SVR_PORT)
transport.setTimeout(TIME_OUT)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
service = StaticLinkBaseService.Client(protocol)
req_item = DownloadReqItem(url = 'http://www.ganji.com')
prop_item = DownloadPropItem(seed_url = 'http://so.iautos.cn/so.jsp?modeltype2=%D0%A1%D0%CD%B3%B5&pageindex=1', depth = 0)
downloaded_body_item = DownloadedBodyItem(req_item, prop_item, is_ok = True, body = '123')
service.upload_body(downloaded_body_item)
transport.close()
return True
except Exception as ex:
print "Error:%s" % (ex)
return True
def _connect_socket(self, timeout=None):
if self.failed_state:
raise Exception("Unable to establish connection since server %s:%s is in failed state." % self.addr)
self.connected_sockets += 1 # This must be done before creating connection to avoid races
try:
host, port = self.addr
sock = TSocket.TSocket(host, port)
protocol = TBinaryProtocol(TFramedTransport(sock))
client = BackendService.Client(protocol)
sock.open()
return client
except:
self.connected_sockets -= 1
self.failed_state = True
raise
def main(argv):
p = argparse.ArgumentParser()
add_common_args(p)
# Since THeaderTransport acts as framed transport when detected frame, we
# cannot use --transport=framed as it would result in 2 layered frames.
p.add_argument('--override-transport')
args = p.parse_args()
assert args.protocol == 'header'
assert args.transport == 'buffered'
assert not args.ssl
sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
if not args.override_transport or args.override_transport == 'buffered':
trans = TBufferedTransport(sock)
elif args.override_transport == 'framed':
trans = TFramedTransport(sock)
else:
raise ValueError('invalid transport')
trans.open()
proto = TBinaryProtocol(trans)
proto.writeMessageBegin('testVoid', TMessageType.CALL, 3)
proto.writeStructBegin('testVoid_args')
proto.writeFieldStop()
proto.writeStructEnd()
proto.writeMessageEnd()
trans.flush()
_, mtype, _ = proto.readMessageBegin()
assert mtype == TMessageType.REPLY
proto.readStructBegin()
_, ftype, _ = proto.readFieldBegin()
assert ftype == TType.STOP
proto.readFieldEnd()
proto.readStructEnd()
proto.readMessageEnd()
trans.close()
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
if conf.transport_mode == 'http':
mode = THttpClient(conf.http_url)
mode.set_verify(conf.validate)
else:
if conf.use_ssl:
try:
from ssl import PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
try:
from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
mode = TSSLSocketWithWildcardSAN(conf.host,
conf.port,
validate=conf.validate,
ca_certs=conf.ca_certs,
keyfile=conf.keyfile,
certfile=conf.certfile,
ssl_version=PROTOCOL_SSLv23)
else:
mode = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
mode.setTimeout(conf.timeout_seconds * 1000.0)
if conf.transport_mode == 'http':
if conf.use_sasl and conf.mechanism != 'PLAIN':
mode.set_kerberos_auth(service=conf.kerberos_principal)
elif USE_THRIFT_HTTP_JWT.get():
from desktop.auth.backend import find_user, rewrite_user # Cyclic dependency
user = rewrite_user(find_user(conf.username))
if user is None:
raise Exception("JWT: User not found.")
if ENABLE_ORGANIZATIONS.get() and user.token:
token = user.token
elif user.profile.data.get('jwt_access_token'):
token = user.profile.data['jwt_access_token']
else:
raise Exception(
"JWT: Could not retrive saved token from user.")
mode.set_bearer_auth(token)
else:
mode.set_basic_auth(conf.username, conf.password)
if conf.transport_mode == 'socket' and conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", str(conf.host))
saslc.setAttr("service", str(conf.kerberos_principal))
if conf.mechanism == 'PLAIN':
saslc.setAttr("username", str(conf.username))
saslc.setAttr(
"password", str(conf.password)
) # Defaults to 'hue' for a non-empty string unless using LDAP
else:
saslc.setAttr("maxbufsize", SASL_MAX_BUFFER.get())
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, conf.mechanism, mode)
elif conf.transport == 'framed':
transport = TFramedTransport(mode)
else:
transport = TBufferedTransport(mode)
protocol = TBinaryProtocol(transport)
if conf.multiple:
protocol = TMultiplexedProtocol(protocol, conf.service_name)
service = conf.klass(protocol)
return service, protocol, transport
class ThriftClient(threading.local):
@Argument('interface', type=ArsInterface)
@Argument('transport_factory', type=FunctionType)
def __init__(self, interface, transport_factory):
super(ThriftClient, self).__init__()
self._interface = interface
self._transport_factory = transport_factory
self._started = False
def _wrap_procedure(self, procedure):
names = [parameter.name for parameter in procedure.parameters]
sign = Signature(names)
for param in procedure.parameters:
if param.optional:
sign.arguments[param.name].mode = ArgumentMode.OPTIONAL
values_type = sign.Values
def proc(*args, **kwargs):
if not self._started:
self.start()
values = values_type(*args, **kwargs)
try:
return self._processor.call(procedure, values.named, self._protocol, self._protocol)
except TTransportException:
self.stop()
self.start()
return self._processor.call(procedure, values.named, self._protocol, self._protocol)
except IOError as e:
if e[0] == errno.EPIPE:
self.stop()
self.start()
return self._processor.call(procedure, values.named, self._protocol, self._protocol)
else:
raise
proc.func_name = procedure.name
return proc
def wrap_all(self):
for service in self._interface.services:
for procedure in service.procedures:
procedure.implementation = self._wrap_procedure(procedure)
def unwrap_all(self):
for service in self._interface.services:
for procedure in service.procedures:
procedure.implementation = None
def start(self):
if self._started:
self.stop()
self._transport = TFramedTransport(self._transport_factory())
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._processor = ThriftProcessor(self._interface)
self._started = True
def stop(self):
if self._started:
self._transport.close()
self._started = False
import sys
sys.path.append('../gen-py')
sys.path.append('/usr/lib/python2.6/site-packages/')
from conf_crawler import DownloaderService
from conf_crawler import DCService
from conf_crawler.ttypes import *
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.transport.TTransport import TFramedTransport
transport = TSocket.TSocket('localhost', 44002)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)
client = DCService.Client(protocol)
def PushDownloadTask():
try:
url = 'http://www.coo8.com/interfaces/showReviewsByGoodsId.action<%param%>flag=all&goodsId=P145484&pageIndex=1'
referer = "http://www.coo8.com/product/145484.html"
download_task = DownloadTask()
download_task.req_item = DownloadReqItem()
download_task.req_item.url = url
download_task.req_item.referer = referer
download_task.req_item.time_out = 1000
def _create_connection(self):
self.transport = TTransport.TBufferedTransport(TFramedTransport(TSocket.TSocket(self.host, self.port)))
self.protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport)
self.client = Hbase.Client(self.protocol)
self.transport.open()
class ThriftClient(threading.local):
@Argument('interface', type=ArsInterface)
@Argument('transport_factory', type=FunctionType)
def __init__(self, interface, transport_factory):
super(ThriftClient, self).__init__()
self._interface = interface
self._transport_factory = transport_factory
self._started = False
def _wrap_procedure(self, procedure):
names = [parameter.name for parameter in procedure.parameters]
sign = Signature(names)
for param in procedure.parameters:
if param.optional:
sign.arguments[param.name].mode = ArgumentMode.OPTIONAL
values_type = sign.Values
def proc(*args, **kwargs):
if not self._started:
self.start()
values = values_type(*args, **kwargs)
try:
return self._processor.call(procedure, values.named,
self._protocol, self._protocol)
except TTransportException:
self.stop()
self.start()
return self._processor.call(procedure, values.named,
self._protocol, self._protocol)
except IOError as e:
if e[0] == errno.EPIPE:
self.stop()
self.start()
return self._processor.call(procedure, values.named,
self._protocol, self._protocol)
else:
raise
proc.func_name = procedure.name
return proc
def wrap_all(self):
for service in self._interface.services:
for procedure in service.procedures:
procedure.implementation = self._wrap_procedure(procedure)
def unwrap_all(self):
for service in self._interface.services:
for procedure in service.procedures:
procedure.implementation = None
def start(self):
if self._started:
self.stop()
self._transport = TFramedTransport(self._transport_factory())
self._transport.open()
self._protocol = TBinaryProtocol(self._transport)
self._processor = ThriftProcessor(self._interface)
self._started = True
def stop(self):
if self._started:
self._transport.close()
self._started = False
