Ejemplo n.º 1
0
 def connect_to_thrudex(self):
     socket = TSocket('localhost', THRUDEX_PORT)
     transport = TFramedTransport(socket)
     protocol = TBinaryProtocol(transport)
     self.thrudex = Thrudex.Client(protocol)
     transport.open()
     self.thrudex.admin("create_index", THRUDEX_INDEX)
Ejemplo n.º 2
0
    def connect_to_thrudoc(self):
        socket = TSocket('localhost', THRUDOC_PORT)
        transport = TFramedTransport(socket)
        protocol = TBinaryProtocol(transport)
        self.thrudoc = Thrudoc.Client(protocol)

        transport.open()
Ejemplo n.º 3
0
 def _connect(self, host, port) -> Client:
     socket = TSocket(host, port)
     transport = TFramedTransport(socket)
     protocol = TCompactProtocol(transport)
     client = Client(protocol)
     transport.open()
     return client
Ejemplo n.º 4
0
    def serve(self):
        self._server_transport.listen()
        while True:
            with self._stopped_mu:
                if self._stopped:
                    return

            client = self._server_transport.accept()
            framed = TFramedTransport(client)
            iprot = self._protocol_factory.get_protocol(framed)
            oprot = self._protocol_factory.get_protocol(framed)
            processor = self._processor_factory.get_processor(framed)

            try:
                while True:
                    with self._stopped_mu:
                        if self._stopped:
                            break

                    processor.process(iprot, oprot)
            except TTransportException:
                continue
            except Exception as e:
                print(e)
                break

            framed.close()
Ejemplo n.º 5
0
class ThriftClient(object):
    def __init__(self, host, port, service_class, **kwargs):
        self._host = host
        self._port = port
        self._connection_timeout = kwargs.get('connection_time', DEFAULT_TCONNECTION_TIMEOUT)
        self._receive_timeout = kwargs.get('receive_timeout', DEFAULT_TRECEIVE_TIMEOUT)
        self._send_timeout = kwargs.get('send_timeout', DEFAULT_TSEND_TIMEOUT)
        self._transport_class = kwargs.get('transport_class', TFRAMED_TRANSPORT_NAME)
        self._service_class = service_class
        self._connect = False

    def get_service(self):
        self.__connect_if_necessary()
        return self._client

    def __connect_if_necessary(self):
        if self._connect:
            return

        self._socket = TSocket(self._host, self._port)
        if (self._transport_class == TBUFFERED_TRANSPORT_NAME):
            self._transport = TBufferedTransport(self._socket)
        else:
            self._transport = TFramedTransport(self._socket)
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._client = self._service_class.Client(self._protocol)
        self._connect = True
Ejemplo n.º 6
0
    def run(self):
        global work_mutex
        global work_numbers

        err_local = 0
        try:
            socket = TSocket(self.server_ip, int(self.server_port))
            transport = TFramedTransport(socket)
            transport.open()
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
            client = ThriftNeloEventServer.Client(protocol)

            stop_flag = True
            while stop_flag:
                #read thrift from file
                f = file(file_name, 'r')
                fd_transport = TFileObjectTransport(f)
                buffered_transport = TBufferedTransport(fd_transport)
                binary_protocol = TBinaryProtocol.TBinaryProtocol(
                    buffered_transport)
                fd_transport.open()
                stop_flag = False
                try:
                    evt = ThriftNeloEvent()
                    while True:
                        evt.read(binary_protocol)
                        #send the log to each project name
                        for prjName, logCnt in prj_dict.items():
                            try:
                                if logCnt_dict.has_key(prjName):
                                    if int(logCnt_dict[prjName]) < int(logCnt):
                                        evt.projectName = prjName
                                        evt.sendTime = int(time.time() * 1000)
                                        err = client.ackedAppend(evt)
                                        tps_remember(err)
                                        err_local += err
                                        logCnt_dict[
                                            prjName] = logCnt_dict[prjName] + 1
                                        stop_flag = True
                                else:
                                    evt.projectName = prjName
                                    err = client.ackedAppend(evt)
                                    tps_remember(err)
                                    err_local += err
                                    logCnt_dict[prjName] = 1
                                    stop_flag = True
                            except TException, msg:
                                print msg, prjName
                except EOFError, msg:
                    buffered_transport.close()  #close the transport
                    stop_flag = True

            work_mutex.acquire()
            work_numbers -= 1
            work_mutex.release()

            socket.close()
Ejemplo n.º 7
0
 def __init__(self):
     __socket = TSocket.TSocket(host=HBASE_HOST, port=HBASE_PORT)
     __socket.setTimeout(50000)
     self.transport = TFramedTransport(__socket)
     # transport = TTransport.TBufferedTransport(socket)
     __protocol = TCompactProtocol.TCompactProtocol(self.transport)
     # protocol = TBinaryProtocol.TBinaryProtocol(transport)
     self.client = Hbase.Client(__protocol)
     self.transport.open()
Ejemplo n.º 8
0
    def start(self):
        if self._started:
            self.stop()

        self._transport = TFramedTransport(self._transport_factory())
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._processor = ThriftProcessor(self._interface)
        self._started = True
Ejemplo n.º 9
0
    def run(self):
        global work_mutex
        global work_numbers
                
        err_local = 0
        try:
    	    socket = TSocket(self.server_ip, int(self.server_port))
            transport = TFramedTransport(socket)
            transport.open()
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
            client = ThriftNeloEventServer.Client(protocol)   


            stop_flag = True
            while stop_flag:
    	        #read thrift from file
                f = file(file_name, 'r')
                fd_transport = TFileObjectTransport(f)
                buffered_transport = TBufferedTransport(fd_transport)
                binary_protocol = TBinaryProtocol.TBinaryProtocol(buffered_transport)
                fd_transport.open()
                stop_flag = False
                try:
                    evt = ThriftNeloEvent()
                    while True:
                        evt.read(binary_protocol)
				        #send the log to each project name
                        for prjName, logCnt in prj_dict.items():
                            try:
                                if logCnt_dict.has_key(prjName):
                                    if int(logCnt_dict[prjName]) < int(logCnt):
                                        evt.projectName = prjName
                                        evt.sendTime = int(time.time() * 1000)
                                        err = client.ackedAppend(evt)
                                        tps_remember(err)
                                        err_local += err
                                        logCnt_dict[prjName] = logCnt_dict[prjName] + 1
                                        stop_flag = True
                                else:
                                    evt.projectName = prjName
                                    err = client.ackedAppend(evt)
                                    tps_remember(err)
                                    err_local += err
                                    logCnt_dict[prjName] = 1
                                    stop_flag = True
                            except TException, msg:
                                print msg, prjName
                except EOFError,msg:
                    buffered_transport.close() #close the transport
                    stop_flag = True

            work_mutex.acquire()
            work_numbers -= 1
            work_mutex.release()
 
            socket.close()
Ejemplo n.º 10
0
class ThriftHbase(object):
    def __init__(self):
        __socket = TSocket.TSocket(host=HBASE_HOST, port=HBASE_PORT)
        __socket.setTimeout(50000)
        self.transport = TFramedTransport(__socket)
        # transport = TTransport.TBufferedTransport(socket)
        __protocol = TCompactProtocol.TCompactProtocol(self.transport)
        # protocol = TBinaryProtocol.TBinaryProtocol(transport)
        self.client = Hbase.Client(__protocol)
        self.transport.open()

    def create_table_hbase(self, table_name, column_list=None):
        column_families = list()
        # 定义列族
        if column_list:
            for each in column_list:
                column = ColumnDescriptor(name=each)
                column_families.append(column)
        else:
            column = ColumnDescriptor(name=HBASE_COLUM_FAMILY)
            column_families.append(column)
        # 创建表
        self.client.createTable(table_name, column_families)

    def delete_table_hbase(self, table_name):
        self.client.disableTable(table_name)
        self.client.deleteTable(table_name)

    def insert_row_into_hbase(self, table_name, data):
        mutation_list = list()
        row_key = str(data["_id"])
        # del data["_id"]
        if "d" in data:
            del data["d"]
        for key, value in data.items():
            if key == "_id":
                continue
            else:
                mutation = Mutation(column="{}:{}".format(
                    HBASE_COLUM_FAMILY, str(key)),
                                    value=str(value))
                mutation_list.append(mutation)

        self.client.mutateRow(table_name, row_key, mutation_list)

        # batch_mutation1 = BatchMutation("ROW1", mutation_list)
        # batch_mutation2 = BatchMutation("ROW1", mutation_list)
        # batch_mutation_list = [batch_mutation1, batch_mutation2]
        # client.mutateRows(table_name, batch_mutation_list)

    def delete_row_from_hbase(self, table_name, row):
        self.client.deleteAllRow(table_name, row)

    def get_row_from_hbase(self, table_name, row):
        return self.client.getRow(table_name, row)
Ejemplo n.º 11
0
    def __connect_if_necessary(self):
        if self._connect:
            return

        self._socket = TSocket(self._host, self._port)
        if (self._transport_class == TBUFFERED_TRANSPORT_NAME):
            self._transport = TBufferedTransport(self._socket)
        else:
            self._transport = TFramedTransport(self._socket)
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._client = self._service_class.Client(self._protocol)
        self._connect = True
Ejemplo n.º 12
0
    def Init(self):
        try:
            transport = TSocket.TSocket(SVR_IP, SVR_PORT)
            transport.setTimeout(TIME_OUT)
            framed_transport = TFramedTransport(transport)
            framed_transport.open()
            protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

            self.service = DedupService.Client(protocol)

            return True
        except Exception as ex:
            print 'Exception:%s' % (ex)
            return False
Ejemplo n.º 13
0
def Extract(url_template, template_type, depth, body_file):
    try:
        global SVR_PORT
        transport = TSocket.TSocket(SVR_IP, SVR_PORT)
        transport.setTimeout(TIME_OUT)
        framed_transport = TFramedTransport(transport)
        framed_transport.open()
        protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

        service = ExtractorService.Client(protocol)
        extract_item = ExtractItem()
        extract_item.url = 'http://api.wap.58.com/api/info/infolist/bj/zufang/1/25/?pic=1'
        extract_item.url_template = url_template
        extract_item.depth = depth
        extract_item.template_type = template_type
        file_in = body_file
        f = open(file_in, 'r')
        extract_item.body = f.read()
        f.close()
        matched_result_item = service.extract_sync(extract_item)
        print 'len(sub_result_list):%d' % len(
            matched_result_item.sub_result_list)
        transport.close()
        if matched_result_item.is_ok == False:
            print 'Err:%s' % (matched_result_item.err_info)
        for key, value in matched_result_item.self_result.iteritems():
            for v in value:
                print '[%s]\t%s' % (key, v)
        for item in matched_result_item.sub_result_list:
            print '-------------------'
            for key, value in item.iteritems():
                print '[%s]\t%s' % (key, value[0])
        '''
    transport = TSocket.TSocket(SVR_IP, SVR_PORT)
    transport.setTimeout(TIME_OUT)
    framed_transport = TFramedTransport(transport)
    framed_transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

    service = StaticLinkBaseService.Client(protocol)
    service.upload_extract(extract_item, matched_result_item)
    transport.close()
    '''

        return True

    except Exception as ex:
        print "Error:%s" % (ex)
        return False
Ejemplo n.º 14
0
def main(argv):
    p = argparse.ArgumentParser()
    add_common_args(p)
    # Since THeaderTransport acts as framed transport when detected frame, we
    # cannot use --transport=framed as it would result in 2 layered frames.
    p.add_argument('--override-transport')
    p.add_argument('--override-protocol')
    args = p.parse_args()
    assert args.protocol == 'header'
    assert args.transport == 'buffered'
    assert not args.ssl

    sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
    if not args.override_transport or args.override_transport == 'buffered':
        trans = TBufferedTransport(sock)
    elif args.override_transport == 'framed':
        print('TFRAMED')
        trans = TFramedTransport(sock)
    else:
        raise ValueError('invalid transport')
    trans.open()

    if not args.override_protocol or args.override_protocol == 'binary':
        proto = TBinaryProtocol(trans)
    elif args.override_protocol == 'compact':
        proto = TCompactProtocol(trans)
    else:
        raise ValueError('invalid transport')

    test_void(proto)
    test_void(proto)

    trans.close()
Ejemplo n.º 15
0
def connect_to_thrift(conf):
  """
  Connect to a thrift endpoint as determined by the 'conf' parameter.
  Note that this does *not* open the transport.

  Returns a tuple of (service, protocol, transport)
  """
  if conf.transport_mode == 'http':
    mode = THttpClient(conf.http_url)
    mode.set_verify(conf.validate)
  else:
    if conf.use_ssl:
      try:
        from ssl import PROTOCOL_TLS
        PROTOCOL_SSLv23 = PROTOCOL_TLS
      except ImportError:
        try:
          from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
          PROTOCOL_SSLv23 = PROTOCOL_TLS
        except ImportError:
          PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
      mode = TSSLSocketWithWildcardSAN(conf.host, conf.port, validate=conf.validate, ca_certs=conf.ca_certs,
                                       keyfile=conf.keyfile, certfile=conf.certfile, ssl_version=PROTOCOL_SSLv23)
    else:
      mode = TSocket(conf.host, conf.port)

  if conf.timeout_seconds:
    # Thrift trivia: You can do this after the fact with
    # _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
    mode.setTimeout(conf.timeout_seconds * 1000.0)

  if conf.transport_mode == 'http':
    if conf.use_sasl and conf.mechanism != 'PLAIN':
      mode.set_kerberos_auth(service=conf.kerberos_principal)
    else:
      mode.set_basic_auth(conf.username, conf.password)

  if conf.transport_mode == 'socket' and conf.use_sasl:
    def sasl_factory():
      saslc = sasl.Client()
      saslc.setAttr("host", str(conf.host))
      saslc.setAttr("service", str(conf.kerberos_principal))
      if conf.mechanism == 'PLAIN':
        saslc.setAttr("username", str(conf.username))
        saslc.setAttr("password", str(conf.password)) # Defaults to 'hue' for a non-empty string unless using LDAP
      else:
        saslc.setAttr("maxbufsize", SASL_MAX_BUFFER.get())
      saslc.init()
      return saslc
    transport = TSaslClientTransport(sasl_factory, conf.mechanism, mode)
  elif conf.transport == 'framed':
    transport = TFramedTransport(mode)
  else:
    transport = TBufferedTransport(mode)

  protocol = TBinaryProtocol(transport)
  if conf.multiple:
    protocol = TMultiplexedProtocol(protocol, conf.service_name)
  service = conf.klass(protocol)
  return service, protocol, transport
Ejemplo n.º 16
0
def LoadMongoDbTask(task_id, is_add_task):
  try:
    transport = TSocket.TSocket(SVR_IP, SVR_PORT)
    transport.setTimeout(TIME_OUT)
    framed_transport = TFramedTransport(transport)
    framed_transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

    service = StaticLinkBaseService.Client(protocol)
    service.load_mongodb_task(task_id, is_add_task)
    transport.close()

    return True

  except Exception as ex:
    print "Error:%s" % (ex)
    return True
Ejemplo n.º 17
0
def LoadSeedByUrl(seed_url, is_add_link):
  try:
    transport = TSocket.TSocket(SVR_IP, SVR_PORT)
    transport.setTimeout(TIME_OUT)
    framed_transport = TFramedTransport(transport)
    framed_transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

    service = LinkBaseService.Client(protocol)
    service.load_seed_by_url(seed_url, is_add_link)
    transport.close()

    return True

  except Exception as ex:
    print "Error:%s" % (ex)
    return True
Ejemplo n.º 18
0
def LoadTemplate(url_template, template_type):
    try:
        global SVR_PORT
        transport = TSocket.TSocket(SVR_IP, SVR_PORT)
        transport.setTimeout(TIME_OUT)
        framed_transport = TFramedTransport(transport)
        framed_transport.open()
        protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

        service = ExtractorService.Client(protocol)
        ret = service.load_template(url_template, template_type)
        transport.close()

        return True

    except Exception as ex:
        print "Error:%s" % (ex)
        return True
Ejemplo n.º 19
0
    def start(self):
        if self._started:
            self.stop()

        self._transport = TFramedTransport(self._transport_factory())
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._processor = ThriftProcessor(self._interface)
        self._started = True
Ejemplo n.º 20
0
def connect_to_thrift(conf):
    """
  Connect to a thrift endpoint as determined by the 'conf' parameter.
  Note that this does *not* open the transport.

  Returns a tuple of (service, protocol, transport)
  """

    if conf.use_ssl:
        sock = TSSLSocket(conf.host,
                          conf.port,
                          validate=conf.validate,
                          ca_certs=conf.ca_certs,
                          keyfile=conf.keyfile,
                          certfile=conf.certfile)
    else:
        sock = TSocket(conf.host, conf.port)
    if conf.timeout_seconds:
        # Thrift trivia: You can do this after the fact with
        # _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
        sock.setTimeout(conf.timeout_seconds * 1000.0)

    if conf.transport_mode == 'http':
        sock = THttpClient(conf.http_url, cert_validate=conf.validate)
        if conf.use_sasl and conf.mechanism != 'PLAIN':
            sock.set_kerberos_auth()
        else:
            sock.set_basic_auth(conf.username, conf.password)
        transport = TBufferedTransport(sock)

    elif conf.use_sasl:

        def sasl_factory():
            saslc = sasl.Client()
            saslc.setAttr("host", str(conf.host))
            saslc.setAttr("service", str(conf.kerberos_principal))
            if conf.mechanism == 'PLAIN':
                saslc.setAttr("username", str(conf.username))
                saslc.setAttr(
                    "password", str(conf.password)
                )  # defaults to hue for a non-empty string unless using ldap
            saslc.init()
            return saslc

        transport = TSaslClientTransport(sasl_factory, conf.mechanism, sock)
    elif conf.transport_mode == 'framed':
        transport = TFramedTransport(sock)
    else:
        transport = TBufferedTransport(sock)

    protocol = TBinaryProtocol(transport)
    if conf.multiple:
        protocol = TMultiplexedProtocol(protocol, conf.service_name)
    service = conf.klass(protocol)
    return service, protocol, transport
Ejemplo n.º 21
0
def UploadBody():
  try:
    transport = TSocket.TSocket(SVR_IP, SVR_PORT)
    transport.setTimeout(TIME_OUT)
    framed_transport = TFramedTransport(transport)
    framed_transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

    service = StaticLinkBaseService.Client(protocol)
    req_item = DownloadReqItem(url = 'http://www.ganji.com')
    prop_item = DownloadPropItem(seed_url = 'http://so.iautos.cn/so.jsp?modeltype2=%D0%A1%D0%CD%B3%B5&pageindex=1', depth = 0)
    downloaded_body_item = DownloadedBodyItem(req_item, prop_item, is_ok = True, body = '123')
    service.upload_body(downloaded_body_item)
    transport.close()

    return True

  except Exception as ex:
    print "Error:%s" % (ex)
    return True
Ejemplo n.º 22
0
    def _connect_socket(self, timeout=None):
        if self.failed_state:
            raise Exception("Unable to establish connection since server %s:%s is in failed state." % self.addr)

        self.connected_sockets += 1 # This must be done before creating connection to avoid races
        try:
            host, port = self.addr

            sock = TSocket.TSocket(host, port)
            protocol = TBinaryProtocol(TFramedTransport(sock))
            client = BackendService.Client(protocol)
            sock.open()

            return client
        except:
            self.connected_sockets -= 1
            self.failed_state = True
            raise
Ejemplo n.º 23
0
def main(argv):
  p = argparse.ArgumentParser()
  add_common_args(p)
  # Since THeaderTransport acts as framed transport when detected frame, we
  # cannot use --transport=framed as it would result in 2 layered frames.
  p.add_argument('--override-transport')
  args = p.parse_args()
  assert args.protocol == 'header'
  assert args.transport == 'buffered'
  assert not args.ssl

  sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
  if not args.override_transport or args.override_transport == 'buffered':
    trans = TBufferedTransport(sock)
  elif args.override_transport == 'framed':
    trans = TFramedTransport(sock)
  else:
    raise ValueError('invalid transport')
  trans.open()
  proto = TBinaryProtocol(trans)
  proto.writeMessageBegin('testVoid', TMessageType.CALL, 3)
  proto.writeStructBegin('testVoid_args')
  proto.writeFieldStop()
  proto.writeStructEnd()
  proto.writeMessageEnd()
  trans.flush()

  _, mtype, _ = proto.readMessageBegin()
  assert mtype == TMessageType.REPLY
  proto.readStructBegin()
  _, ftype, _ = proto.readFieldBegin()
  assert ftype == TType.STOP
  proto.readFieldEnd()
  proto.readStructEnd()
  proto.readMessageEnd()

  trans.close()
Ejemplo n.º 24
0
def connect_to_thrift(conf):
    """
  Connect to a thrift endpoint as determined by the 'conf' parameter.
  Note that this does *not* open the transport.

  Returns a tuple of (service, protocol, transport)
  """
    if conf.transport_mode == 'http':
        mode = THttpClient(conf.http_url)
        mode.set_verify(conf.validate)
    else:
        if conf.use_ssl:
            try:
                from ssl import PROTOCOL_TLS
                PROTOCOL_SSLv23 = PROTOCOL_TLS
            except ImportError:
                try:
                    from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
                    PROTOCOL_SSLv23 = PROTOCOL_TLS
                except ImportError:
                    PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
            mode = TSSLSocketWithWildcardSAN(conf.host,
                                             conf.port,
                                             validate=conf.validate,
                                             ca_certs=conf.ca_certs,
                                             keyfile=conf.keyfile,
                                             certfile=conf.certfile,
                                             ssl_version=PROTOCOL_SSLv23)
        else:
            mode = TSocket(conf.host, conf.port)

    if conf.timeout_seconds:
        # Thrift trivia: You can do this after the fact with
        # _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
        mode.setTimeout(conf.timeout_seconds * 1000.0)

    if conf.transport_mode == 'http':
        if conf.use_sasl and conf.mechanism != 'PLAIN':
            mode.set_kerberos_auth(service=conf.kerberos_principal)

        elif USE_THRIFT_HTTP_JWT.get():
            from desktop.auth.backend import find_user, rewrite_user  # Cyclic dependency
            user = rewrite_user(find_user(conf.username))

            if user is None:
                raise Exception("JWT: User not found.")

            if ENABLE_ORGANIZATIONS.get() and user.token:
                token = user.token
            elif user.profile.data.get('jwt_access_token'):
                token = user.profile.data['jwt_access_token']
            else:
                raise Exception(
                    "JWT: Could not retrive saved token from user.")

            mode.set_bearer_auth(token)
        else:
            mode.set_basic_auth(conf.username, conf.password)

    if conf.transport_mode == 'socket' and conf.use_sasl:

        def sasl_factory():
            saslc = sasl.Client()
            saslc.setAttr("host", str(conf.host))
            saslc.setAttr("service", str(conf.kerberos_principal))
            if conf.mechanism == 'PLAIN':
                saslc.setAttr("username", str(conf.username))
                saslc.setAttr(
                    "password", str(conf.password)
                )  # Defaults to 'hue' for a non-empty string unless using LDAP
            else:
                saslc.setAttr("maxbufsize", SASL_MAX_BUFFER.get())
            saslc.init()
            return saslc

        transport = TSaslClientTransport(sasl_factory, conf.mechanism, mode)
    elif conf.transport == 'framed':
        transport = TFramedTransport(mode)
    else:
        transport = TBufferedTransport(mode)

    protocol = TBinaryProtocol(transport)
    if conf.multiple:
        protocol = TMultiplexedProtocol(protocol, conf.service_name)
    service = conf.klass(protocol)
    return service, protocol, transport
Ejemplo n.º 25
0
class ThriftClient(threading.local):
    @Argument('interface', type=ArsInterface)
    @Argument('transport_factory', type=FunctionType)
    def __init__(self, interface, transport_factory):
        super(ThriftClient, self).__init__()
        self._interface = interface
        self._transport_factory = transport_factory
        self._started = False

    def _wrap_procedure(self, procedure):
        names = [parameter.name for parameter in procedure.parameters]
        sign = Signature(names)
        for param in procedure.parameters:
            if param.optional:
                sign.arguments[param.name].mode = ArgumentMode.OPTIONAL
        values_type = sign.Values

        def proc(*args, **kwargs):
            if not self._started:
                self.start()

            values = values_type(*args, **kwargs)
            try:
                return self._processor.call(procedure, values.named, self._protocol, self._protocol)
            except TTransportException:
                self.stop()
                self.start()
                return self._processor.call(procedure, values.named, self._protocol, self._protocol)
            except IOError as e:
                if e[0] == errno.EPIPE:
                    self.stop()
                    self.start()
                    return self._processor.call(procedure, values.named, self._protocol, self._protocol)
                else:
                    raise

        proc.func_name = procedure.name
        return proc

    def wrap_all(self):
        for service in self._interface.services:
            for procedure in service.procedures:
                procedure.implementation = self._wrap_procedure(procedure)

    def unwrap_all(self):
        for service in self._interface.services:
            for procedure in service.procedures:
                procedure.implementation = None

    def start(self):
        if self._started:
            self.stop()

        self._transport = TFramedTransport(self._transport_factory())
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._processor = ThriftProcessor(self._interface)
        self._started = True

    def stop(self):
        if self._started:
            self._transport.close()
            self._started = False
Ejemplo n.º 26
0
import sys
sys.path.append('../gen-py')
sys.path.append('/usr/lib/python2.6/site-packages/')

from conf_crawler import DownloaderService
from conf_crawler import DCService
from conf_crawler.ttypes import *

from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.transport.TTransport import TFramedTransport

transport = TSocket.TSocket('localhost', 44002)
framed_transport = TFramedTransport(transport)
framed_transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(framed_transport)

client = DCService.Client(protocol)


def PushDownloadTask():
    try:
        url = 'http://www.coo8.com/interfaces/showReviewsByGoodsId.action<%param%>flag=all&goodsId=P145484&pageIndex=1'
        referer = "http://www.coo8.com/product/145484.html"
        download_task = DownloadTask()
        download_task.req_item = DownloadReqItem()
        download_task.req_item.url = url
        download_task.req_item.referer = referer
        download_task.req_item.time_out = 1000
Ejemplo n.º 27
0
 def _create_connection(self):
     self.transport = TTransport.TBufferedTransport(TFramedTransport(TSocket.TSocket(self.host, self.port)))
     self.protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport)
     self.client = Hbase.Client(self.protocol)
     self.transport.open()
Ejemplo n.º 28
0
class ThriftClient(threading.local):
    @Argument('interface', type=ArsInterface)
    @Argument('transport_factory', type=FunctionType)
    def __init__(self, interface, transport_factory):
        super(ThriftClient, self).__init__()
        self._interface = interface
        self._transport_factory = transport_factory
        self._started = False

    def _wrap_procedure(self, procedure):
        names = [parameter.name for parameter in procedure.parameters]
        sign = Signature(names)
        for param in procedure.parameters:
            if param.optional:
                sign.arguments[param.name].mode = ArgumentMode.OPTIONAL
        values_type = sign.Values

        def proc(*args, **kwargs):
            if not self._started:
                self.start()

            values = values_type(*args, **kwargs)
            try:
                return self._processor.call(procedure, values.named,
                                            self._protocol, self._protocol)
            except TTransportException:
                self.stop()
                self.start()
                return self._processor.call(procedure, values.named,
                                            self._protocol, self._protocol)
            except IOError as e:
                if e[0] == errno.EPIPE:
                    self.stop()
                    self.start()
                    return self._processor.call(procedure, values.named,
                                                self._protocol, self._protocol)
                else:
                    raise

        proc.func_name = procedure.name
        return proc

    def wrap_all(self):
        for service in self._interface.services:
            for procedure in service.procedures:
                procedure.implementation = self._wrap_procedure(procedure)

    def unwrap_all(self):
        for service in self._interface.services:
            for procedure in service.procedures:
                procedure.implementation = None

    def start(self):
        if self._started:
            self.stop()

        self._transport = TFramedTransport(self._transport_factory())
        self._transport.open()
        self._protocol = TBinaryProtocol(self._transport)
        self._processor = ThriftProcessor(self._interface)
        self._started = True

    def stop(self):
        if self._started:
            self._transport.close()
            self._started = False