Exemple #1
0
 def _connect(self, host, port) -> Client:
     socket = TSocket(host, port)
     transport = TFramedTransport(socket)
     protocol = TCompactProtocol(transport)
     client = Client(protocol)
     transport.open()
     return client
Exemple #2
0
 def connect_to_thrudex(self):
     socket = TSocket('localhost', THRUDEX_PORT)
     transport = TFramedTransport(socket)
     protocol = TBinaryProtocol(transport)
     self.thrudex = Thrudex.Client(protocol)
     transport.open()
     self.thrudex.admin("create_index", THRUDEX_INDEX)
Exemple #3
0
    def connect_to_thrudoc(self):
        socket = TSocket('localhost', THRUDOC_PORT)
        transport = TFramedTransport(socket)
        protocol = TBinaryProtocol(transport)
        self.thrudoc = Thrudoc.Client(protocol)

        transport.open()
Exemple #4
0
class ThriftClient(object):
    def __init__(self, host, port, service_class, **kwargs):
        self._host = host
        self._port = port
        self._connection_timeout = kwargs.get('connection_time', DEFAULT_TCONNECTION_TIMEOUT)
        self._receive_timeout = kwargs.get('receive_timeout', DEFAULT_TRECEIVE_TIMEOUT)
        self._send_timeout = kwargs.get('send_timeout', DEFAULT_TSEND_TIMEOUT)
        self._transport_class = kwargs.get('transport_class', TFRAMED_TRANSPORT_NAME)
        self._service_class = service_class
        self._connect = False

    def get_service(self):
        self.__connect_if_necessary()
        return self._client

    def __connect_if_necessary(self):
        if self._connect:
            return

        self._socket = TSocket(self._host, self._port)
        if (self._transport_class == TBUFFERED_TRANSPORT_NAME):
            self._transport = TBufferedTransport(self._socket)
        else:
            self._transport = TFramedTransport(self._socket)
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._client = self._service_class.Client(self._protocol)
        self._connect = True
Exemple #5
0
    def run(self):
        global work_mutex
        global work_numbers

        err_local = 0
        try:
            socket = TSocket(self.server_ip, int(self.server_port))
            transport = TFramedTransport(socket)
            transport.open()
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
            client = ThriftNeloEventServer.Client(protocol)

            stop_flag = True
            while stop_flag:
                #read thrift from file
                f = file(file_name, 'r')
                fd_transport = TFileObjectTransport(f)
                buffered_transport = TBufferedTransport(fd_transport)
                binary_protocol = TBinaryProtocol.TBinaryProtocol(
                    buffered_transport)
                fd_transport.open()
                stop_flag = False
                try:
                    evt = ThriftNeloEvent()
                    while True:
                        evt.read(binary_protocol)
                        #send the log to each project name
                        for prjName, logCnt in prj_dict.items():
                            try:
                                if logCnt_dict.has_key(prjName):
                                    if int(logCnt_dict[prjName]) < int(logCnt):
                                        evt.projectName = prjName
                                        evt.sendTime = int(time.time() * 1000)
                                        err = client.ackedAppend(evt)
                                        tps_remember(err)
                                        err_local += err
                                        logCnt_dict[
                                            prjName] = logCnt_dict[prjName] + 1
                                        stop_flag = True
                                else:
                                    evt.projectName = prjName
                                    err = client.ackedAppend(evt)
                                    tps_remember(err)
                                    err_local += err
                                    logCnt_dict[prjName] = 1
                                    stop_flag = True
                            except TException, msg:
                                print msg, prjName
                except EOFError, msg:
                    buffered_transport.close()  #close the transport
                    stop_flag = True

            work_mutex.acquire()
            work_numbers -= 1
            work_mutex.release()

            socket.close()
Exemple #6
0
    def run(self):
        global work_mutex
        global work_numbers
                
        err_local = 0
        try:
    	    socket = TSocket(self.server_ip, int(self.server_port))
            transport = TFramedTransport(socket)
            transport.open()
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
            client = ThriftNeloEventServer.Client(protocol)   


            stop_flag = True
            while stop_flag:
    	        #read thrift from file
                f = file(file_name, 'r')
                fd_transport = TFileObjectTransport(f)
                buffered_transport = TBufferedTransport(fd_transport)
                binary_protocol = TBinaryProtocol.TBinaryProtocol(buffered_transport)
                fd_transport.open()
                stop_flag = False
                try:
                    evt = ThriftNeloEvent()
                    while True:
                        evt.read(binary_protocol)
				        #send the log to each project name
                        for prjName, logCnt in prj_dict.items():
                            try:
                                if logCnt_dict.has_key(prjName):
                                    if int(logCnt_dict[prjName]) < int(logCnt):
                                        evt.projectName = prjName
                                        evt.sendTime = int(time.time() * 1000)
                                        err = client.ackedAppend(evt)
                                        tps_remember(err)
                                        err_local += err
                                        logCnt_dict[prjName] = logCnt_dict[prjName] + 1
                                        stop_flag = True
                                else:
                                    evt.projectName = prjName
                                    err = client.ackedAppend(evt)
                                    tps_remember(err)
                                    err_local += err
                                    logCnt_dict[prjName] = 1
                                    stop_flag = True
                            except TException, msg:
                                print msg, prjName
                except EOFError,msg:
                    buffered_transport.close() #close the transport
                    stop_flag = True

            work_mutex.acquire()
            work_numbers -= 1
            work_mutex.release()
 
            socket.close()
Exemple #7
0
class ThriftHbase(object):
    def __init__(self):
        __socket = TSocket.TSocket(host=HBASE_HOST, port=HBASE_PORT)
        __socket.setTimeout(50000)
        self.transport = TFramedTransport(__socket)
        # transport = TTransport.TBufferedTransport(socket)
        __protocol = TCompactProtocol.TCompactProtocol(self.transport)
        # protocol = TBinaryProtocol.TBinaryProtocol(transport)
        self.client = Hbase.Client(__protocol)
        self.transport.open()

    def create_table_hbase(self, table_name, column_list=None):
        column_families = list()
        # 定义列族
        if column_list:
            for each in column_list:
                column = ColumnDescriptor(name=each)
                column_families.append(column)
        else:
            column = ColumnDescriptor(name=HBASE_COLUM_FAMILY)
            column_families.append(column)
        # 创建表
        self.client.createTable(table_name, column_families)

    def delete_table_hbase(self, table_name):
        self.client.disableTable(table_name)
        self.client.deleteTable(table_name)

    def insert_row_into_hbase(self, table_name, data):
        mutation_list = list()
        row_key = str(data["_id"])
        # del data["_id"]
        if "d" in data:
            del data["d"]
        for key, value in data.items():
            if key == "_id":
                continue
            else:
                mutation = Mutation(column="{}:{}".format(
                    HBASE_COLUM_FAMILY, str(key)),
                                    value=str(value))
                mutation_list.append(mutation)

        self.client.mutateRow(table_name, row_key, mutation_list)

        # batch_mutation1 = BatchMutation("ROW1", mutation_list)
        # batch_mutation2 = BatchMutation("ROW1", mutation_list)
        # batch_mutation_list = [batch_mutation1, batch_mutation2]
        # client.mutateRows(table_name, batch_mutation_list)

    def delete_row_from_hbase(self, table_name, row):
        self.client.deleteAllRow(table_name, row)

    def get_row_from_hbase(self, table_name, row):
        return self.client.getRow(table_name, row)
Exemple #8
0
    def Init(self):
        try:
            transport = TSocket.TSocket(SVR_IP, SVR_PORT)
            transport.setTimeout(TIME_OUT)
            framed_transport = TFramedTransport(transport)
            framed_transport.open()
            protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

            self.service = DedupService.Client(protocol)

            return True
        except Exception as ex:
            print 'Exception:%s' % (ex)
            return False
Exemple #9
0
def Extract(url_template, template_type, depth, body_file):
    try:
        global SVR_PORT
        transport = TSocket.TSocket(SVR_IP, SVR_PORT)
        transport.setTimeout(TIME_OUT)
        framed_transport = TFramedTransport(transport)
        framed_transport.open()
        protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

        service = ExtractorService.Client(protocol)
        extract_item = ExtractItem()
        extract_item.url = 'http://api.wap.58.com/api/info/infolist/bj/zufang/1/25/?pic=1'
        extract_item.url_template = url_template
        extract_item.depth = depth
        extract_item.template_type = template_type
        file_in = body_file
        f = open(file_in, 'r')
        extract_item.body = f.read()
        f.close()
        matched_result_item = service.extract_sync(extract_item)
        print 'len(sub_result_list):%d' % len(
            matched_result_item.sub_result_list)
        transport.close()
        if matched_result_item.is_ok == False:
            print 'Err:%s' % (matched_result_item.err_info)
        for key, value in matched_result_item.self_result.iteritems():
            for v in value:
                print '[%s]\t%s' % (key, v)
        for item in matched_result_item.sub_result_list:
            print '-------------------'
            for key, value in item.iteritems():
                print '[%s]\t%s' % (key, value[0])
        '''
    transport = TSocket.TSocket(SVR_IP, SVR_PORT)
    transport.setTimeout(TIME_OUT)
    framed_transport = TFramedTransport(transport)
    framed_transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

    service = StaticLinkBaseService.Client(protocol)
    service.upload_extract(extract_item, matched_result_item)
    transport.close()
    '''

        return True

    except Exception as ex:
        print "Error:%s" % (ex)
        return False
Exemple #10
0
def LoadMongoDbTask(task_id, is_add_task):
  try:
    transport = TSocket.TSocket(SVR_IP, SVR_PORT)
    transport.setTimeout(TIME_OUT)
    framed_transport = TFramedTransport(transport)
    framed_transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

    service = StaticLinkBaseService.Client(protocol)
    service.load_mongodb_task(task_id, is_add_task)
    transport.close()

    return True

  except Exception as ex:
    print "Error:%s" % (ex)
    return True
Exemple #11
0
def LoadSeedByUrl(seed_url, is_add_link):
  try:
    transport = TSocket.TSocket(SVR_IP, SVR_PORT)
    transport.setTimeout(TIME_OUT)
    framed_transport = TFramedTransport(transport)
    framed_transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

    service = LinkBaseService.Client(protocol)
    service.load_seed_by_url(seed_url, is_add_link)
    transport.close()

    return True

  except Exception as ex:
    print "Error:%s" % (ex)
    return True
Exemple #12
0
def LoadTemplate(url_template, template_type):
    try:
        global SVR_PORT
        transport = TSocket.TSocket(SVR_IP, SVR_PORT)
        transport.setTimeout(TIME_OUT)
        framed_transport = TFramedTransport(transport)
        framed_transport.open()
        protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

        service = ExtractorService.Client(protocol)
        ret = service.load_template(url_template, template_type)
        transport.close()

        return True

    except Exception as ex:
        print "Error:%s" % (ex)
        return True
Exemple #13
0
def UploadBody():
  try:
    transport = TSocket.TSocket(SVR_IP, SVR_PORT)
    transport.setTimeout(TIME_OUT)
    framed_transport = TFramedTransport(transport)
    framed_transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

    service = StaticLinkBaseService.Client(protocol)
    req_item = DownloadReqItem(url = 'http://www.ganji.com')
    prop_item = DownloadPropItem(seed_url = 'http://so.iautos.cn/so.jsp?modeltype2=%D0%A1%D0%CD%B3%B5&pageindex=1', depth = 0)
    downloaded_body_item = DownloadedBodyItem(req_item, prop_item, is_ok = True, body = '123')
    service.upload_body(downloaded_body_item)
    transport.close()

    return True

  except Exception as ex:
    print "Error:%s" % (ex)
    return True
Exemple #14
0
class ThriftClient(threading.local):
    @Argument('interface', type=ArsInterface)
    @Argument('transport_factory', type=FunctionType)
    def __init__(self, interface, transport_factory):
        super(ThriftClient, self).__init__()
        self._interface = interface
        self._transport_factory = transport_factory
        self._started = False

    def _wrap_procedure(self, procedure):
        names = [parameter.name for parameter in procedure.parameters]
        sign = Signature(names)
        for param in procedure.parameters:
            if param.optional:
                sign.arguments[param.name].mode = ArgumentMode.OPTIONAL
        values_type = sign.Values

        def proc(*args, **kwargs):
            if not self._started:
                self.start()

            values = values_type(*args, **kwargs)
            try:
                return self._processor.call(procedure, values.named,
                                            self._protocol, self._protocol)
            except TTransportException:
                self.stop()
                self.start()
                return self._processor.call(procedure, values.named,
                                            self._protocol, self._protocol)
            except IOError as e:
                if e[0] == errno.EPIPE:
                    self.stop()
                    self.start()
                    return self._processor.call(procedure, values.named,
                                                self._protocol, self._protocol)
                else:
                    raise

        proc.func_name = procedure.name
        return proc

    def wrap_all(self):
        for service in self._interface.services:
            for procedure in service.procedures:
                procedure.implementation = self._wrap_procedure(procedure)

    def unwrap_all(self):
        for service in self._interface.services:
            for procedure in service.procedures:
                procedure.implementation = None

    def start(self):
        if self._started:
            self.stop()

        self._transport = TFramedTransport(self._transport_factory())
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._processor = ThriftProcessor(self._interface)
        self._started = True

    def stop(self):
        if self._started:
            self._transport.close()
            self._started = False
Exemple #15
0
sys.path.append('../gen-py')
sys.path.append('/usr/lib/python2.6/site-packages/')

from conf_crawler import DownloaderService
from conf_crawler import DCService
from conf_crawler.ttypes import *

from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.transport.TTransport import TFramedTransport

transport = TSocket.TSocket('localhost', 44002)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

client = DCService.Client(protocol)


def PushDownloadTask():
    try:
        url = 'http://www.coo8.com/interfaces/showReviewsByGoodsId.action<%param%>flag=all&goodsId=P145484&pageIndex=1'
        referer = "http://www.coo8.com/product/145484.html"
        download_task = DownloadTask()
        download_task.req_item = DownloadReqItem()
        download_task.req_item.url = url
        download_task.req_item.referer = referer
        download_task.req_item.time_out = 1000
        download_task.prop_item = DownloadPropItem()
Exemple #16
0
class ThriftClient(threading.local):
    @Argument('interface', type=ArsInterface)
    @Argument('transport_factory', type=FunctionType)
    def __init__(self, interface, transport_factory):
        super(ThriftClient, self).__init__()
        self._interface = interface
        self._transport_factory = transport_factory
        self._started = False

    def _wrap_procedure(self, procedure):
        names = [parameter.name for parameter in procedure.parameters]
        sign = Signature(names)
        for param in procedure.parameters:
            if param.optional:
                sign.arguments[param.name].mode = ArgumentMode.OPTIONAL
        values_type = sign.Values

        def proc(*args, **kwargs):
            if not self._started:
                self.start()

            values = values_type(*args, **kwargs)
            try:
                return self._processor.call(procedure, values.named, self._protocol, self._protocol)
            except TTransportException:
                self.stop()
                self.start()
                return self._processor.call(procedure, values.named, self._protocol, self._protocol)
            except IOError as e:
                if e[0] == errno.EPIPE:
                    self.stop()
                    self.start()
                    return self._processor.call(procedure, values.named, self._protocol, self._protocol)
                else:
                    raise

        proc.func_name = procedure.name
        return proc

    def wrap_all(self):
        for service in self._interface.services:
            for procedure in service.procedures:
                procedure.implementation = self._wrap_procedure(procedure)

    def unwrap_all(self):
        for service in self._interface.services:
            for procedure in service.procedures:
                procedure.implementation = None

    def start(self):
        if self._started:
            self.stop()

        self._transport = TFramedTransport(self._transport_factory())
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._processor = ThriftProcessor(self._interface)
        self._started = True

    def stop(self):
        if self._started:
            self._transport.close()
            self._started = False