def __init__(self, host, port, retry_interval, report_status=None, logging_timeout=None): # set up thrift and scribe objects timeout = logging_timeout if logging_timeout is not None else config.scribe_logging_timeout self.socket = thriftpy.transport.socket.TSocket( six.text_type(host), int(port)) if timeout: self.socket.set_timeout(timeout) self.transport = TFramedTransportFactory().get_transport(self.socket) protocol = TBinaryProtocolFactory(strict_read=False).get_protocol( self.transport) self.client = TClient(scribe_thrift.scribe, protocol) # our own bookkeeping for connection self.connected = False # whether or not we think we're currently connected to the scribe server self.last_connect_time = 0 # last time we got disconnected or failed to reconnect self.retry_interval = retry_interval self.report_status = report_status or get_default_reporter() self.__lock = threading.RLock() self._birth_pid = os.getpid()
def get_nimbus_client(env_config=None, host=None, port=None, timeout=7000): """Get a Thrift RPC client for Nimbus given project's config file. :param env_config: The project's parsed config. :type env_config: `dict` :param host: The host to use for Nimbus. If specified, `env_config` will not be consulted. :type host: `str` :param port: The port to use for Nimbus. If specified, `env_config` will not be consulted. :type port: `int` :param timeout: The time to wait (in milliseconds) for a response from Nimbus. :param timeout: `int` :returns: a ThriftPy RPC client to use to communicate with Nimbus """ if host is None: host, port = get_nimbus_host_port(env_config) nimbus_client = make_client( Nimbus, host=host, port=port, proto_factory=TBinaryProtocolFactory(), trans_factory=TFramedTransportFactory(), timeout=timeout, ) return nimbus_client
def __init__(self, host, port, category='zipkin', timeout=None): self.category = category self.host = host self.port = port self.timeout = timeout self.protocol_factory = TBinaryProtocolFactory(strict_read=False, strict_write=False) self.transport_factory = TFramedTransportFactory()
def blocking_call(): client = make_client( service=addressbook.AddressBookService, host=self.host, port=self.port, trans_factory=TFramedTransportFactory(), proto_factory=TBinaryProtocolFactory(), ) with closing_client(client): return getattr(client, method)(*args, **kwargs)
def make_client(service, host, port, proto_factory=TBinaryProtocolFactory(), trans_factory=TFramedTransportFactory()): socket = TNonBlockingSocket(host, port) transport = trans_factory.get_transport(socket) protocol = proto_factory.get_protocol(transport) transport.open() return TClient(service, protocol)
def __init__(self, ip="10.15.208.61", port=6800): self.ip = ip self.port = port try: self.client = make_client(CNN_i2i_thrift.CNNPredictService, ip, port,trans_factory=TFramedTransportFactory(), timeout=12000) ## timeout ms except Exception as err: print err traceback.print_exc() return
class FramedTransportTestCase(TestCase): TRANSPORT_FACTORY = TFramedTransportFactory() def mk_server(self): self.io_loop = ioloop.IOLoop() server = make_server(addressbook.AddressBookService, Dispatcher(self.io_loop), io_loop=self.io_loop) self.server = server sock = self.server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('localhost', 0)) sock.setblocking(0) self.port = sock.getsockname()[-1] self.server_thread = threading.Thread(target=self.listen) self.server_thread.setDaemon(True) self.server_thread.start() def listen(self): self.server_sock.listen(128) self.server.add_socket(self.server_sock) self.io_loop.start() def mk_client(self): return make_client(addressbook.AddressBookService, '127.0.0.1', self.port, trans_factory=self.TRANSPORT_FACTORY) def setUp(self): self.mk_server() time.sleep(0.1) self.client = self.mk_client() def test_able_to_communicate(self): dennis = addressbook.Person(name='Dennis Ritchie') success = self.client.add(dennis) assert success success = self.client.add(dennis) assert not success def test_zero_length_string(self): dennis = addressbook.Person(name='') success = self.client.add(dennis) assert success success = self.client.get(name='') assert success
def __init__(self, host, port, retry_interval, report_status=None, logging_timeout=None): # set up thrift and scribe objects timeout = logging_timeout if logging_timeout is not None else config.scribe_logging_timeout self.socket = thriftpy.transport.socket.TSocket(six.text_type(host), int(port)) if timeout: self.socket.set_timeout(timeout) self.transport = TFramedTransportFactory().get_transport(self.socket) protocol = TBinaryProtocolFactory(strict_read=False).get_protocol(self.transport) self.client = TClient(scribe_thrift.scribe, protocol) # our own bookkeeping for connection self.connected = False # whether or not we think we're currently connected to the scribe server self.last_connect_time = 0 # last time we got disconnected or failed to reconnect self.retry_interval = retry_interval self.report_status = report_status or get_default_reporter() self.__lock = threading.RLock() self._birth_pid = os.getpid()
def api_handler(username, service, version, path): runtime = Runtime.query.filter_by( username=username, service_name=service, version=version, obsolete=False, ).first() if runtime is None: return "service not found", 404 if request.method in ('POST', 'PUT'): body = request.content else: body = None client = make_client(okapi_thrift.InvokeService, runtime.full_name, 23241, trans_factory=TFramedTransportFactory()) result = client.InvokeAPI(path, request.method, request.args, request.headers, body) return result.body, result.code, result.headers
def DoPostProcess(self, datalst): reqlst = [] for data in datalst: req = CNN_i2i_thrift.Request() req.inputImage = data reqlst.append(req) ret = [] try: reslst = self.client.predict(reqlst) #print res for i in range(len(reslst)): roiimglst = [] ## get mask image maskres = reslst[i].maskResult roiimglst.append(maskres.outputImage) ## get roi image roilist = reslst[i].roiResults for k in range(len(roilist)): roi = roilist[k] val = roi.outputImage resc= roi.imageChannel resw= roi.imageWidth resh= roi.imageHeight roiimglst.append(val) ret.append(roiimglst) except Exception as err: try: self.client = make_client(CNN_i2i_thrift.CNNPredictService, self.ip, self.port,trans_factory=TFramedTransportFactory(), timeout=12000) ## timeout ms except Exception as err: traceback.print_exc() print err print err traceback.print_exc() return ret
class ScribeLogger(object): """Implementation that logs to a scribe server. If errors are encountered, drop lines and retry occasionally. :param host: hostname of the scribe server :param port: port number of the scribe server :param retry_interval: number of seconds to wait between retries :param report_status: a function `report_status(is_error, msg)` which is called to print out errors and status messages. The first argument indicates whether what is being printed is an error or not, and the second argument is the actual message. :param logging_timeout: milliseconds to time out scribe logging; "0" means blocking (no timeout) """ def __init__(self, host, port, retry_interval, report_status=None, logging_timeout=None): # set up thrift and scribe objects timeout = logging_timeout if logging_timeout is not None else config.scribe_logging_timeout self.socket = thriftpy.transport.socket.TSocket(six.text_type(host), int(port)) if timeout: self.socket.set_timeout(timeout) self.transport = TFramedTransportFactory().get_transport(self.socket) protocol = TBinaryProtocolFactory(strict_read=False).get_protocol(self.transport) self.client = TClient(scribe_thrift.scribe, protocol) # our own bookkeeping for connection self.connected = False # whether or not we think we're currently connected to the scribe server self.last_connect_time = 0 # last time we got disconnected or failed to reconnect self.retry_interval = retry_interval self.report_status = report_status or get_default_reporter() self.__lock = threading.RLock() self._birth_pid = os.getpid() def _maybe_reconnect(self): """Try (re)connecting to the server if it's been long enough since our last attempt. """ assert self.connected == False # don't retry too often now = time.time() if (now - self.last_connect_time) > self.retry_interval: try: self.transport.open() self.connected = True except TTransportException: self.last_connect_time = now self.report_status(True, 'yelp_clog failed to connect to scribe server') def _log_line_no_size_limit(self, stream, line): """Log a single line without size limit. It should not include any newline characters. Since this method is called in log_line, the line should be in utf-8 format and less than MAX_LINE_SIZE_IN_BYTES already. We don't limit traceback size. """ with self.__lock: if os.getpid() != self._birth_pid: raise ScribeIsNotForkSafeError if not self.connected: self._maybe_reconnect() if self.connected: log_entry = scribe_thrift.LogEntry(category=scribify(stream), message=line + b'\n') try: return self.client.Log(messages=[log_entry]) except Exception as e: try: self.report_status( True, 'yelp_clog failed to log to scribe server with ' ' exception: %s(%s)' % (type(e), six.text_type(e)) ) finally: self.close() self.last_connect_time = time.time() # Don't reconnect if report_status raises an exception self._maybe_reconnect() def log_line(self, stream, line): """Log a single line. It should not include any newline characters. If the line size is over 50 MB, an exception raises and the line will be dropped. If the line size is over 5 MB, a message consisting origin stream information will be recorded at WHO_CLOG_LARGE_LINE_STREAM (in json format). """ # log unicodes as their utf-8 encoded representation if isinstance(line, six.text_type): line = line.encode('UTF-8') # check log line size if len(line) <= WARNING_LINE_SIZE_IN_BYTES: self._log_line_no_size_limit(stream, line) elif len(line) <= MAX_LINE_SIZE_IN_BYTES: self._log_line_no_size_limit(stream, line) # log the origin of the stream with traceback to WHO_CLOG_LARGE_LINE_STREAM category origin_info = {} origin_info['stream'] = stream origin_info['line_size'] = len(line) origin_info['traceback'] = ''.join(traceback.format_stack()) log_line = json.dumps(origin_info).encode('UTF-8') self._log_line_no_size_limit(WHO_CLOG_LARGE_LINE_STREAM, log_line) self.report_status( False, 'The log line size is larger than %r bytes (monitored in \'%s\')' % (WARNING_LINE_SIZE_IN_BYTES, WHO_CLOG_LARGE_LINE_STREAM) ) else: # raise an exception if too large self.report_status( True, 'The log line is dropped (line size larger than %r bytes)' % MAX_LINE_SIZE_IN_BYTES ) raise LogLineIsTooLongError('The max log line size allowed is %r bytes' % MAX_LINE_SIZE_IN_BYTES) def close(self): self.transport.close() self.connected = False
def mk_client(self): return make_client(addressbook.AddressBookService, '127.0.0.1', self.port, transport_factory=TFramedTransportFactory())
def serve_forever(runtime, port, client): try: api_details = import_module(runtime.entrypoint) r = client.post("http://okapi-engine:5000/okapi/service/v1/%s/%s/spec" % (runtime.service_name, runtime.service_version), json = api_details) logging.debug("post service spec status: %s" % r.status_code) except: traceback.print_exc() try: logging.info("start to serve on port %s" % port) server = make_server(okapi_thrift.InvokeService, Dispatcher(), '0.0.0.0', port, trans_factory=TFramedTransportFactory()) server.serve() except Exception as e: traceback.print_exc() sys.exit(2) # serve fails (e.g. port already used) or serve exited (e.g. user press CTRL+C)
class ScribeLogger(object): """Implementation that logs to a scribe server. If errors are encountered, drop lines and retry occasionally. :param host: hostname of the scribe server :param port: port number of the scribe server :param retry_interval: number of seconds to wait between retries :param report_status: a function `report_status(is_error, msg)` which is called to print out errors and status messages. The first argument indicates whether what is being printed is an error or not, and the second argument is the actual message. :param logging_timeout: milliseconds to time out scribe logging; "0" means blocking (no timeout) """ def __init__(self, host, port, retry_interval, report_status=None, logging_timeout=None): # set up thrift and scribe objects timeout = logging_timeout if logging_timeout is not None else config.scribe_logging_timeout self.socket = thriftpy.transport.socket.TSocket( six.text_type(host), int(port)) if timeout: self.socket.set_timeout(timeout) self.transport = TFramedTransportFactory().get_transport(self.socket) protocol = TBinaryProtocolFactory(strict_read=False).get_protocol( self.transport) self.client = TClient(scribe_thrift.scribe, protocol) # our own bookkeeping for connection self.connected = False # whether or not we think we're currently connected to the scribe server self.last_connect_time = 0 # last time we got disconnected or failed to reconnect self.retry_interval = retry_interval self.report_status = report_status or get_default_reporter() self.__lock = threading.RLock() self._birth_pid = os.getpid() def _maybe_reconnect(self): """Try (re)connecting to the server if it's been long enough since our last attempt. """ assert self.connected == False # don't retry too often now = time.time() if (now - self.last_connect_time) > self.retry_interval: try: self.transport.open() self.connected = True except TTransportException: self.last_connect_time = now self.report_status( True, 'yelp_clog failed to connect to scribe server') def _log_line_no_size_limit(self, stream, line): """Log a single line without size limit. It should not include any newline characters. Since this method is called in log_line, the line should be in utf-8 format and less than MAX_LINE_SIZE_IN_BYTES already. We don't limit traceback size. """ with self.__lock: if os.getpid() != self._birth_pid: raise ScribeIsNotForkSafeError if not self.connected: self._maybe_reconnect() if self.connected: log_entry = scribe_thrift.LogEntry(category=scribify(stream), message=line + b'\n') try: return self.client.Log(messages=[log_entry]) except Exception as e: try: self.report_status( True, 'yelp_clog failed to log to scribe server with ' ' exception: %s(%s)' % (type(e), six.text_type(e))) finally: self.close() self.last_connect_time = time.time() # Don't reconnect if report_status raises an exception self._maybe_reconnect() def log_line(self, stream, line): """Log a single line. It should not include any newline characters. If the line size is over 50 MB, an exception raises and the line will be dropped. If the line size is over 5 MB, a message consisting origin stream information will be recorded at WHO_CLOG_LARGE_LINE_STREAM (in json format). """ # log unicodes as their utf-8 encoded representation if isinstance(line, six.text_type): line = line.encode('UTF-8') # check log line size if len(line) <= WARNING_LINE_SIZE_IN_BYTES: self._log_line_no_size_limit(stream, line) elif len(line) <= MAX_LINE_SIZE_IN_BYTES: self._log_line_no_size_limit(stream, line) # log the origin of the stream with traceback to WHO_CLOG_LARGE_LINE_STREAM category origin_info = {} origin_info['stream'] = stream origin_info['line_size'] = len(line) origin_info['traceback'] = ''.join(traceback.format_stack()) log_line = json.dumps(origin_info).encode('UTF-8') self._log_line_no_size_limit(WHO_CLOG_LARGE_LINE_STREAM, log_line) self.report_status( False, 'The log line size is larger than %r bytes (monitored in \'%s\')' % (WARNING_LINE_SIZE_IN_BYTES, WHO_CLOG_LARGE_LINE_STREAM)) else: # raise an exception if too large self.report_status( True, 'The log line is dropped (line size larger than %r bytes)' % MAX_LINE_SIZE_IN_BYTES) raise LogLineIsTooLongError( 'The max log line size allowed is %r bytes' % MAX_LINE_SIZE_IN_BYTES) def close(self): self.transport.close() self.connected = False