def checkRead(o): prot = TBinaryProtocol.TBinaryProtocol(TTransport.TMemoryBuffer()) o.write(prot) slow_version_binary = prot.trans.getvalue() prot = TBinaryProtocol.TBinaryProtocolAccelerated( TTransport.TMemoryBuffer(slow_version_binary)) c = o.__class__() c.read(prot) if c != o: print("copy: ") pprint(eval(repr(c))) print("orig: ") pprint(eval(repr(o))) prot = TBinaryProtocol.TBinaryProtocolAccelerated( TTransport.TBufferedTransport( TTransport.TMemoryBuffer(slow_version_binary))) c = o.__class__() c.read(prot) if c != o: print("copy: ") pprint(eval(repr(c))) print("orig: ") pprint(eval(repr(o)))
def _configure_scribe(self, host, port): self.socket = TSocket.TSocket(host=host, port=port) self.socket.setTimeout(1000) self.transport = TTransport.TFramedTransport(self.socket) self.protocol = TBinaryProtocol.TBinaryProtocolAccelerated( trans=self.transport, strictRead=False, strictWrite=False) self.client = scribe.Client(iprot=self.protocol, oprot=self.protocol)
def __init__(self, data_status): self.block_names = [ block_chain.block_ids[-1].split(':') for block_chain in data_status.data_blocks ] self.block_ids = [int(b[-1]) for b in self.block_names] self.transports = [ TTransport.TFramedTransport(TSocket.TSocket(b[0], int(b[1]))) for b in self.block_names ] self.protocols = [ TBinaryProtocol.TBinaryProtocolAccelerated(transport) for transport in self.transports ] self.clients = [ block_request_service.Client(protocol) for protocol in self.protocols ] for transport in self.transports: transport.open() self.notifications = Mailbox() self.controls = Mailbox() self.worker = SubscriptionWorker(self.protocols, self.notifications, self.controls) self.worker.start()
def configure(cls, ngram_table="ngrams", subst_table="typogram", wiki_anchors_table="wiki_anchors", wiki_urls_table="wiki_urls", wiki_edges_table="wiki_edges", wiki_pagelinks_title_table="TL", wiki_link_mention_cooccur="CC", wiki_link_cooccur_table="LL", hbase_host=None): cls.subst_table = subst_table cls.ngram_table = ngram_table cls.wiki_urls_table = wiki_urls_table cls.wiki_anchors_table = wiki_anchors_table cls.wiki_edges_table = wiki_edges_table cls.wiki_pagelinks_title_table = wiki_pagelinks_title_table cls.wiki_link_mention_cooccur_table = wiki_link_mention_cooccur cls.wiki_link_cooccur_table = wiki_link_cooccur_table # HBASE cls.h_transport = TTransport.TBufferedTransport( TSocket.TSocket(*hbase_host)) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(cls.h_transport) cls.h_client = Hbase.Client(protocol) cls.h_transport.open() cls.h_rate = 0 cls.h_start = time.time() cls.substitution_counts = cls.get_freq(SUBSTITUTION_TOKEN) cls.substitutions = sorted(cls.substitution_counts.keys())
def get_mention_from_wikilink_thrift_file(fn): f = open(args.thrift_data_dir + '/%03d' % fn) out_val = defaultdict(list) p = TBinaryProtocol.TBinaryProtocolAccelerated(TFileObjectTransport(f)) pp = WikiLinkItem() while True: try: pp.read(p) except EOFError: break for m in pp.mentions: c = m.context if c is not None: url = simplify_wiki_url(m.wiki_url) # Follow url redirect. try: url = redirect[hash(url)] except KeyError: pass if url in POOL: # if c.left.startswith('the Musical August 30th, 2009 | Author: operator Shrek the Musical is a musical with music by Jeanine Tesori and a book and lyrics'): # print 1, fn out_val[url].append([c.left, c.middle, c.right]) print fn, len(out_val), sum(len(e) for e in out_val.itervalues()) out_val.default_factory = None # FINALIZE out_val return out_val
def connect(self): """Creates a connection to an Impalad instance. Returns a tuple with the impala version string and the webserver address, otherwise raises an exception. If the client was already connected, closes the previous connection.""" self.close_connection() sock, self.transport = self._get_socket_and_transport() if self.client_connect_timeout_ms > 0: sock.setTimeout(self.client_connect_timeout_ms) self.transport.open() if self.verbose: print_to_stderr('Opened TCP connection to %s:%s' % (self.impalad_host, self.impalad_port)) # Setting a timeout of None disables timeouts on sockets sock.setTimeout(None) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport) self.imp_service = self._get_thrift_client(protocol) self.connected = True try: self._open_session() return self._ping_impala_service() except: # Ensure we are in a disconnected state if we failed above. self.close_connection() raise
def _get_client_by_transport(self, options, transport, socket=None): # Create the protocol and client if options.json: protocol = TJSONProtocol.TJSONProtocol(transport) elif options.compact: protocol = TCompactProtocol.TCompactProtocol(transport) # No explicit option about protocol is specified. Try to infer. elif options.framed or options.unframed: protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) elif socket is not None: # If json, compact, framed, and unframed are not specified, # THeaderProtocol is the default. Create a protocol using either # fuzzy or non-fuzzy transport depending on if options.fuzz is set. if options.fuzz is not None: transport = TFuzzyHeaderTransport(socket, fuzz_fields=options.fuzz, verbose=True) else: transport = THeaderTransport(socket) protocol = THeaderProtocol.THeaderProtocol(transport) else: self._exit(error_message=('No valid protocol ' 'specified for %s' % (type(self))), status=os.EX_USAGE) transport.open() self._transport = transport client = self.service_class.Client(protocol) return client
def __init__(self, keyspace, server, framed_transport=True, timeout=None, credentials=None, socket_factory=default_socket_factory, transport_factory=default_transport_factory): self.keyspace = None self.server = server server = server.split(':') if len(server) <= 1: port = 9160 else: port = server[1] host = server[0] socket = socket_factory(host, int(port)) if timeout is not None: socket.setTimeout(timeout * 1000.0) self.transport = transport_factory(socket, host, port) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport) Cassandra.Client.__init__(self, protocol) self.transport.open() self.set_keyspace(keyspace) if credentials is not None: request = AuthenticationRequest(credentials=credentials) self.login(request)
def get_thrift_client(host='127.0.0.1', port=9170): socket = TSocket.TSocket(host, port) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) client = Cassandra.Client(protocol) client.transport = transport return client
def __init__(self, node=None, host=None, port=None, ks_name='ks', cf_name='cf', cassandra_interface='11'): """ initializes the connection. - node: a ccm node. If supplied, the host and port, and cassandra_interface will be pulled from the node. - host, port: overwritten if node is supplied - ks_name, cf_name: all operations are done on the supplied ks and cf - cassandra_interface: '07' and '11' are currently supported. This is the thrift interface to cassandra. '11' suffices for now except when creating keyspaces against cassandra0.7, in which case 07 must be used. """ if node: host, port = node.network_interfaces['thrift'] self.node = node self.host = host self.port = port self.cassandra_interface = cassandra_interface # import the correct version of the cassandra thrift interface # and set self.Cassandra as the imported module module_name = 'cassandra-thrift.v%s' % cassandra_interface imp = __import__(module_name, globals(), locals(), ['Cassandra']) self.Cassandra = imp.Cassandra socket = TSocket.TSocket(host, port) self.transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport) self.client = self.Cassandra.Client(protocol) socket.open() self.open_socket = True self.ks_name = ks_name self.cf_name = cf_name
def __init__(self, keyspace, server, framed_transport=True, timeout=None, credentials=None, api_version=None): self.keyspace = None self.server = server server = server.split(':') if len(server) <= 1: port = 9160 else: port = server[1] host = server[0] socket = TSocket.TSocket(host, int(port)) if timeout is not None: socket.setTimeout(timeout * 1000.0) if framed_transport: self.transport = TTransport.TFramedTransport(socket) else: self.transport = TTransport.TBufferedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport) Cassandra.Client.__init__(self, protocol) self.transport.open() self.set_keyspace(keyspace) if credentials is not None: request = AuthenticationRequest(credentials=credentials) self.login(request)
def open(self, retry=5, timeout_ms=None, force=False): if not self.transport or force: timeout_ms = timeout_ms or self.timeout servers_count = len(self.servers) count = min(retry, servers_count) if count == 0: logger.warn('NO SERVERS -->group: %s, %s:%s ' % (self.GROUP, self.host, self.port)) retry_count = 1 while retry_count <= count: try: #TODO: investigate BufferedStream perf # Use sequential retry self.server = self.servers[retry_count - 1] self.host = self.server['host'] self.port = self.server['port'] thrift_socket = TSocket.TSocket(self.host, int(self.port)) thrift_socket.setTimeout(timeout_ms) self.transport = TTransport.TBufferedTransport(thrift_socket) self.protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport) self.client = self.client_cls(self.protocol) self.transport.open() logger.info("CONNECTED TO SUCCESS --> %s:%s." % (self.host, self.port)) return except Exception: logger.exception('CONNECT TO FAILED --> %s:%s retry: %d' % (self.host, self.port, retry_count)) retry_count += 1 raise ThriftException('FAILED TO CONNECT TO THRIFT SERVER --> servers:%s' % (self.servers))
def deserialize_body(self, body): trans = TTransport.TMemoryBuffer(body) proto = TBinaryProtocol.TBinaryProtocolAccelerated(trans) result = self.deserialize_type() result.read(proto) return result
def __init__(self, host, port, keyspace, user=None, password=None): """ Params: * host .........: hostname of Cassandra node. * port .........: port number to connect to. * keyspace .....: keyspace to connect to. * user .........: username used in authentication (optional). * password .....: password used in authentication (optional). """ self.host = host self.port = port self.keyspace = keyspace socket = TSocket.TSocket(host, port) self.transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport) self.client = Cassandra.Client(protocol) socket.open() self.open_socket = True if user and password: credentials = {"username": user, "password": password} self.client.login(AuthenticationRequest(credentials=credentials)) if keyspace: c = self.cursor() c.execute('USE %s;' % keyspace) c.close()
def __init__(self, keyspace, server, framed_transport, timeout, credentials, recycle): host, port = server.split(":") socket = TSocket.TSocket(host, int(port)) if timeout is not None: socket.setTimeout(timeout * 1000.0) if framed_transport: transport = TTransport.TFramedTransport(socket) else: transport = TTransport.TBufferedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) client = Cassandra.Client(protocol) transport.open() server_api_version = client.describe_version().split('.', 1) assert server_api_version[0] == API_VERSION[0], \ "Thrift API version mismatch. " \ "(Client: %s, Server: %s)" % (API_VERSION[0], server_api_version[0]) client.set_keyspace(keyspace) if credentials is not None: request = AuthenticationRequest(credentials=credentials) client.login(request) self.keyspace = keyspace self.client = client self.transport = transport if recycle: self.recycle = time.time() + recycle + random.uniform( 0, recycle * 0.1) else: self.recycle = None
def __init__(self, server, framed_transport, timeout, recycle): host, port = server.split(":") socket = TSocket.TSocket(host, int(port)) if timeout is not None: socket.setTimeout(timeout * 1000.0) if framed_transport: transport = TTransport.TFramedTransport(socket) else: transport = TTransport.TBufferedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) client = Rest.Client(protocol) transport.open() # server_api_version = client.describe_version().split('.', 1) # assert server_api_version[0] == API_VERSION[0], \ # "Thrift API version mismatch. " \ # "(Client: %s, Server: %s)" % (API_VERSION[0], server_api_version[0]) self.client = client self.transport = transport if recycle: self.recycle = time.time() + recycle + random.uniform( 0, recycle * 0.1) else: self.recycle = None
def __init__(self, uri=None, # type: Optional[str] user=None, # type: Optional[str] password=None, # type: Optional[str] host=None, # type: Optional[str] port=9091, # type: Optional[int] dbname=None, # type: Optional[str] protocol='binary', # type: Optional[str] ): # type: (...) -> None if uri is not None: if not all([user is None, password is None, host is None, port == 9091, dbname is None, protocol == 'binary']): raise TypeError("Cannot specify both URI and other arguments") user, password, host, port, dbname, protocol = _parse_uri(uri) if host is None: raise TypeError("`host` parameter is required.") if protocol in ("http", "https"): if not host.startswith(protocol): # the THttpClient expects http[s]://localhost host = protocol + '://' + host transport = THttpClient.THttpClient("{}:{}".format(host, port)) proto = TJSONProtocol.TJSONProtocol(transport) socket = None elif protocol == "binary": socket = TSocket.TSocket(host, port) transport = TTransport.TBufferedTransport(socket) proto = TBinaryProtocol.TBinaryProtocolAccelerated(transport) else: raise ValueError("`protocol` should be one of", " ['http', 'https', 'binary'],", " got {} instead".format(protocol)) self._user = user self._password = password self._host = host self._port = port self._dbname = dbname self._transport = transport self._protocol = protocol self._socket = socket self._closed = 0 self._tdf = None try: self._transport.open() except TTransportException as e: if e.NOT_OPEN: err = OperationalError("Could not connect to database") six.raise_from(err, e) else: raise self._client = Client(proto) try: self._session = self._client.connect(user, password, dbname) except TMapDException as e: six.raise_from(_translate_exception(e), e)
def thrift_client(self): socket = TSocket.TSocket(self.ip, self.port) transport = TTransport.TBufferedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) client = MapService.Client(protocol) transport.open() yield client transport.close()
def doTest(): checkWrite(hm) no_set = deepcopy(hm) no_set.contain = set() checkRead(no_set) checkWrite(rs) checkRead(rs) checkWrite(rshuge) checkRead(rshuge) checkWrite(my_zero) checkRead(my_zero) checkRead(Backwards({"first_tag2": 4, "second_tag1": 2})) try: checkWrite(my_nega) print "Hey, did this get fixed?" except AttributeError: # Sorry, doesn't work with negative tags. pass # One case where the serialized form changes, but only superficially. o = Backwards({"first_tag2": 4, "second_tag1": 2}) trans_fast = TTransport.TMemoryBuffer() trans_slow = TTransport.TMemoryBuffer() prot_fast = TBinaryProtocol.TBinaryProtocolAccelerated(trans_fast) prot_slow = TBinaryProtocol.TBinaryProtocol(trans_slow) o.write(prot_fast) o.write(prot_slow) ORIG = trans_slow.getvalue() MINE = trans_fast.getvalue() if ORIG == MINE: print "That shouldn't happen." prot = TBinaryProtocol.TBinaryProtocolAccelerated( TTransport.TMemoryBuffer()) o.write(prot) prot = TBinaryProtocol.TBinaryProtocol( TTransport.TMemoryBuffer(prot.trans.getvalue())) c = o.__class__() c.read(prot) if c != o: print "copy: " pprint(eval(repr(c))) print "orig: " pprint(eval(repr(o)))
def getLocalConnection(port=None): if port is None: port = conf.NETWORK_PORT socket = TSocket.TSocket("localhost", port) transport = TTransport.TFramedTransport(socket) transport.open() protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) service = RndNodeApi.Client(protocol) return (service, transport)
def __init__(self, socket, client_class): self.socket = None self.mutex = BoundedSemaphore(1) self.socket = TSocket.TSocket(unix_socket=socket) self.transport = TTransport.TBufferedTransport(self.socket) self.protocol = TBinaryProtocol.TBinaryProtocolAccelerated( self.transport) self.client = client_class(self.protocol) self.connected = False
def connect(server='localhost', port=9090, timeout=None): socket = TSocket(server, int(port)) if timeout is not None: socket.setTimeout(timeout) transport = TBufferedTransport(socket) transport.open() protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) client = Hbase.Client(protocol) return client
def serialize(msg): msg.validate() transport = TTransport.TMemoryBuffer() protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) msg.write(protocol) data = transport.getvalue() transport.close() return data
def connect(): transport = TTransport.TBufferedTransport(TSocket.TSocket('127.0.0.1', 9090)) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) client = THBaseService.Client(protocol) transport.open() try: yield client finally: transport.close()
def deserialize(msg, data): transport = TTransport.TMemoryBuffer(data) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) msg.read(protocol) msg.validate() remaining = data[transport.cstringio_buf.tell():] transport.close() return msg, remaining
def _create_thrift_connection(self): socket = TSocket.TSocket(self.host, self.port) if self.network_timeout > 0: socket.setTimeout(self.network_timeout) transport = TTransport.TBufferedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) connection = self.iface_cls(protocol) transport.open() return connection
def _refresh_thrift_client(self): """Refresh the Thrift socket, transport, and client.""" socket = TSocket(self.host, self.port) if self.timeout is not None: socket.setTimeout(self.timeout) self.transport = self._transport_class(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport) self.client = Hbase.Client(protocol)
def _getClient(self): # We can't just keep a connection because the app might fork # and we'll be left with the parent process's connection. # More robust to just open one for each flush. sock = TSocket.TSocket(host=self._remote_host, port=self._remote_port) sock.setTimeout(self._socket_timeout * 1000) transport = TTransport.TFramedTransport(sock) transport.open() protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) return scribe.Client(protocol)
def get_client(host='127.0.0.1', port=9160): socket = TSocket.TSocket(host, port) if options.unframed: transport = TTransport.TBufferedTransport(socket) else: transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) client = Cassandra.Client(protocol) client.transport = transport return client
def doTest(): checkWrite(hm) no_set = deepcopy(hm) no_set.contain = set() checkRead(no_set) checkRead(reserved) checkWrite(reserved) checkWrite(rs) checkRead(rs) checkWrite(rshuge) checkRead(rshuge) checkWrite(my_zero) checkRead(my_zero) checkRead(Backwards(first_tag2=4, second_tag1=2)) # One case where the serialized form changes, but only superficially. o = Backwards(first_tag2=4, second_tag1=2) trans_fast = TTransport.TMemoryBuffer() trans_slow = TTransport.TMemoryBuffer() prot_fast = TBinaryProtocol.TBinaryProtocolAccelerated(trans_fast) prot_slow = TBinaryProtocol.TBinaryProtocol(trans_slow) o.write(prot_fast) o.write(prot_slow) ORIG = trans_slow.getvalue() MINE = trans_fast.getvalue() if ORIG == MINE: print("That shouldn't happen.") prot = TBinaryProtocol.TBinaryProtocolAccelerated( TTransport.TMemoryBuffer()) o.write(prot) prot = TBinaryProtocol.TBinaryProtocol(TTransport.TMemoryBuffer( prot.trans.getvalue())) c = o.__class__() c.read(prot) if c != o: print("copy: ") pprint(eval(repr(c))) print("orig: ") pprint(eval(repr(o)))