def __init__(self,
host,
port,
retry_interval,
report_status=None,
logging_timeout=None):
# set up thrift and scribe objects
timeout = logging_timeout if logging_timeout is not None else config.scribe_logging_timeout
self.socket = thriftpy.transport.socket.TSocket(
six.text_type(host), int(port))
if timeout:
self.socket.set_timeout(timeout)
self.transport = TFramedTransportFactory().get_transport(self.socket)
protocol = TBinaryProtocolFactory(strict_read=False).get_protocol(
self.transport)
self.client = TClient(scribe_thrift.scribe, protocol)
# our own bookkeeping for connection
self.connected = False # whether or not we think we're currently connected to the scribe server
self.last_connect_time = 0 # last time we got disconnected or failed to reconnect
self.retry_interval = retry_interval
self.report_status = report_status or get_default_reporter()
self.__lock = threading.RLock()
self._birth_pid = os.getpid()
def get_nimbus_client(env_config=None, host=None, port=None, timeout=7000):
"""Get a Thrift RPC client for Nimbus given project's config file.
:param env_config: The project's parsed config.
:type env_config: `dict`
:param host: The host to use for Nimbus. If specified, `env_config` will
not be consulted.
:type host: `str`
:param port: The port to use for Nimbus. If specified, `env_config` will
not be consulted.
:type port: `int`
:param timeout: The time to wait (in milliseconds) for a response from
Nimbus.
:param timeout: `int`
:returns: a ThriftPy RPC client to use to communicate with Nimbus
"""
if host is None:
host, port = get_nimbus_host_port(env_config)
nimbus_client = make_client(
Nimbus,
host=host,
port=port,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TFramedTransportFactory(),
timeout=timeout,
)
return nimbus_client
def __init__(self, host, port, category='zipkin', timeout=None):
self.category = category
self.host = host
self.port = port
self.timeout = timeout
self.protocol_factory = TBinaryProtocolFactory(strict_read=False,
strict_write=False)
self.transport_factory = TFramedTransportFactory()
def blocking_call():
client = make_client(
service=addressbook.AddressBookService,
host=self.host,
port=self.port,
trans_factory=TFramedTransportFactory(),
proto_factory=TBinaryProtocolFactory(),
)
with closing_client(client):
return getattr(client, method)(*args, **kwargs)
def make_client(service,
host,
port,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TFramedTransportFactory()):
socket = TNonBlockingSocket(host, port)
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def __init__(self, ip="10.15.208.61", port=6800):
self.ip = ip
self.port = port
try:
self.client = make_client(CNN_i2i_thrift.CNNPredictService,
ip, port,trans_factory=TFramedTransportFactory(), timeout=12000) ## timeout ms
except Exception as err:
print err
traceback.print_exc()
return
class FramedTransportTestCase(TestCase):
TRANSPORT_FACTORY = TFramedTransportFactory()
def mk_server(self):
self.io_loop = ioloop.IOLoop()
server = make_server(addressbook.AddressBookService,
Dispatcher(self.io_loop),
io_loop=self.io_loop)
self.server = server
sock = self.server_sock = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
sock.bind(('localhost', 0))
sock.setblocking(0)
self.port = sock.getsockname()[-1]
self.server_thread = threading.Thread(target=self.listen)
self.server_thread.setDaemon(True)
self.server_thread.start()
def listen(self):
self.server_sock.listen(128)
self.server.add_socket(self.server_sock)
self.io_loop.start()
def mk_client(self):
return make_client(addressbook.AddressBookService,
'127.0.0.1',
self.port,
trans_factory=self.TRANSPORT_FACTORY)
def setUp(self):
self.mk_server()
time.sleep(0.1)
self.client = self.mk_client()
def test_able_to_communicate(self):
dennis = addressbook.Person(name='Dennis Ritchie')
success = self.client.add(dennis)
assert success
success = self.client.add(dennis)
assert not success
def test_zero_length_string(self):
dennis = addressbook.Person(name='')
success = self.client.add(dennis)
assert success
success = self.client.get(name='')
assert success
def __init__(self, host, port, retry_interval, report_status=None, logging_timeout=None):
# set up thrift and scribe objects
timeout = logging_timeout if logging_timeout is not None else config.scribe_logging_timeout
self.socket = thriftpy.transport.socket.TSocket(six.text_type(host), int(port))
if timeout:
self.socket.set_timeout(timeout)
self.transport = TFramedTransportFactory().get_transport(self.socket)
protocol = TBinaryProtocolFactory(strict_read=False).get_protocol(self.transport)
self.client = TClient(scribe_thrift.scribe, protocol)
# our own bookkeeping for connection
self.connected = False # whether or not we think we're currently connected to the scribe server
self.last_connect_time = 0 # last time we got disconnected or failed to reconnect
self.retry_interval = retry_interval
self.report_status = report_status or get_default_reporter()
self.__lock = threading.RLock()
self._birth_pid = os.getpid()
def api_handler(username, service, version, path):
runtime = Runtime.query.filter_by(
username=username,
service_name=service,
version=version,
obsolete=False,
).first()
if runtime is None:
return "service not found", 404
if request.method in ('POST', 'PUT'):
body = request.content
else:
body = None
client = make_client(okapi_thrift.InvokeService,
runtime.full_name,
23241,
trans_factory=TFramedTransportFactory())
result = client.InvokeAPI(path, request.method, request.args,
request.headers, body)
return result.body, result.code, result.headers
def DoPostProcess(self, datalst):
reqlst = []
for data in datalst:
req = CNN_i2i_thrift.Request()
req.inputImage = data
reqlst.append(req)
ret = []
try:
reslst = self.client.predict(reqlst)
#print res
for i in range(len(reslst)):
roiimglst = []
## get mask image
maskres = reslst[i].maskResult
roiimglst.append(maskres.outputImage)
## get roi image
roilist = reslst[i].roiResults
for k in range(len(roilist)):
roi = roilist[k]
val = roi.outputImage
resc= roi.imageChannel
resw= roi.imageWidth
resh= roi.imageHeight
roiimglst.append(val)
ret.append(roiimglst)
except Exception as err:
try:
self.client = make_client(CNN_i2i_thrift.CNNPredictService,
self.ip, self.port,trans_factory=TFramedTransportFactory(), timeout=12000) ## timeout ms
except Exception as err:
traceback.print_exc()
print err
print err
traceback.print_exc()
return ret
class ScribeLogger(object):
"""Implementation that logs to a scribe server. If errors are encountered,
drop lines and retry occasionally.
:param host: hostname of the scribe server
:param port: port number of the scribe server
:param retry_interval: number of seconds to wait between retries
:param report_status: a function `report_status(is_error, msg)` which is
called to print out errors and status messages. The first
argument indicates whether what is being printed is an error or not,
and the second argument is the actual message.
:param logging_timeout: milliseconds to time out scribe logging; "0" means
blocking (no timeout)
"""
def __init__(self, host, port, retry_interval, report_status=None, logging_timeout=None):
# set up thrift and scribe objects
timeout = logging_timeout if logging_timeout is not None else config.scribe_logging_timeout
self.socket = thriftpy.transport.socket.TSocket(six.text_type(host), int(port))
if timeout:
self.socket.set_timeout(timeout)
self.transport = TFramedTransportFactory().get_transport(self.socket)
protocol = TBinaryProtocolFactory(strict_read=False).get_protocol(self.transport)
self.client = TClient(scribe_thrift.scribe, protocol)
# our own bookkeeping for connection
self.connected = False # whether or not we think we're currently connected to the scribe server
self.last_connect_time = 0 # last time we got disconnected or failed to reconnect
self.retry_interval = retry_interval
self.report_status = report_status or get_default_reporter()
self.__lock = threading.RLock()
self._birth_pid = os.getpid()
def _maybe_reconnect(self):
"""Try (re)connecting to the server if it's been long enough since our
last attempt.
"""
assert self.connected == False
# don't retry too often
now = time.time()
if (now - self.last_connect_time) > self.retry_interval:
try:
self.transport.open()
self.connected = True
except TTransportException:
self.last_connect_time = now
self.report_status(True, 'yelp_clog failed to connect to scribe server')
def _log_line_no_size_limit(self, stream, line):
"""Log a single line without size limit. It should not include any newline characters.
Since this method is called in log_line, the line should be in utf-8 format and
less than MAX_LINE_SIZE_IN_BYTES already. We don't limit traceback size.
"""
with self.__lock:
if os.getpid() != self._birth_pid:
raise ScribeIsNotForkSafeError
if not self.connected:
self._maybe_reconnect()
if self.connected:
log_entry = scribe_thrift.LogEntry(category=scribify(stream), message=line + b'\n')
try:
return self.client.Log(messages=[log_entry])
except Exception as e:
try:
self.report_status(
True,
'yelp_clog failed to log to scribe server with '
' exception: %s(%s)' % (type(e), six.text_type(e))
)
finally:
self.close()
self.last_connect_time = time.time()
# Don't reconnect if report_status raises an exception
self._maybe_reconnect()
def log_line(self, stream, line):
"""Log a single line. It should not include any newline characters.
If the line size is over 50 MB, an exception raises and the line will be dropped.
If the line size is over 5 MB, a message consisting origin stream information
will be recorded at WHO_CLOG_LARGE_LINE_STREAM (in json format).
"""
# log unicodes as their utf-8 encoded representation
if isinstance(line, six.text_type):
line = line.encode('UTF-8')
# check log line size
if len(line) <= WARNING_LINE_SIZE_IN_BYTES:
self._log_line_no_size_limit(stream, line)
elif len(line) <= MAX_LINE_SIZE_IN_BYTES:
self._log_line_no_size_limit(stream, line)
# log the origin of the stream with traceback to WHO_CLOG_LARGE_LINE_STREAM category
origin_info = {}
origin_info['stream'] = stream
origin_info['line_size'] = len(line)
origin_info['traceback'] = ''.join(traceback.format_stack())
log_line = json.dumps(origin_info).encode('UTF-8')
self._log_line_no_size_limit(WHO_CLOG_LARGE_LINE_STREAM, log_line)
self.report_status(
False,
'The log line size is larger than %r bytes (monitored in \'%s\')'
% (WARNING_LINE_SIZE_IN_BYTES, WHO_CLOG_LARGE_LINE_STREAM)
)
else:
# raise an exception if too large
self.report_status(
True,
'The log line is dropped (line size larger than %r bytes)'
% MAX_LINE_SIZE_IN_BYTES
)
raise LogLineIsTooLongError('The max log line size allowed is %r bytes'
% MAX_LINE_SIZE_IN_BYTES)
def close(self):
self.transport.close()
self.connected = False
def mk_client(self):
return make_client(addressbook.AddressBookService,
'127.0.0.1',
self.port,
transport_factory=TFramedTransportFactory())
def serve_forever(runtime, port, client):
try:
api_details = import_module(runtime.entrypoint)
r = client.post("http://okapi-engine:5000/okapi/service/v1/%s/%s/spec" % (runtime.service_name, runtime.service_version), json = api_details)
logging.debug("post service spec status: %s" % r.status_code)
except:
traceback.print_exc()
try:
logging.info("start to serve on port %s" % port)
server = make_server(okapi_thrift.InvokeService, Dispatcher(), '0.0.0.0', port, trans_factory=TFramedTransportFactory())
server.serve()
except Exception as e:
traceback.print_exc()
sys.exit(2) # serve fails (e.g. port already used) or serve exited (e.g. user press CTRL+C)
class ScribeLogger(object):
"""Implementation that logs to a scribe server. If errors are encountered,
drop lines and retry occasionally.
:param host: hostname of the scribe server
:param port: port number of the scribe server
:param retry_interval: number of seconds to wait between retries
:param report_status: a function `report_status(is_error, msg)` which is
called to print out errors and status messages. The first
argument indicates whether what is being printed is an error or not,
and the second argument is the actual message.
:param logging_timeout: milliseconds to time out scribe logging; "0" means
blocking (no timeout)
"""
def __init__(self,
host,
port,
retry_interval,
report_status=None,
logging_timeout=None):
# set up thrift and scribe objects
timeout = logging_timeout if logging_timeout is not None else config.scribe_logging_timeout
self.socket = thriftpy.transport.socket.TSocket(
six.text_type(host), int(port))
if timeout:
self.socket.set_timeout(timeout)
self.transport = TFramedTransportFactory().get_transport(self.socket)
protocol = TBinaryProtocolFactory(strict_read=False).get_protocol(
self.transport)
self.client = TClient(scribe_thrift.scribe, protocol)
# our own bookkeeping for connection
self.connected = False # whether or not we think we're currently connected to the scribe server
self.last_connect_time = 0 # last time we got disconnected or failed to reconnect
self.retry_interval = retry_interval
self.report_status = report_status or get_default_reporter()
self.__lock = threading.RLock()
self._birth_pid = os.getpid()
def _maybe_reconnect(self):
"""Try (re)connecting to the server if it's been long enough since our
last attempt.
"""
assert self.connected == False
# don't retry too often
now = time.time()
if (now - self.last_connect_time) > self.retry_interval:
try:
self.transport.open()
self.connected = True
except TTransportException:
self.last_connect_time = now
self.report_status(
True, 'yelp_clog failed to connect to scribe server')
def _log_line_no_size_limit(self, stream, line):
"""Log a single line without size limit. It should not include any newline characters.
Since this method is called in log_line, the line should be in utf-8 format and
less than MAX_LINE_SIZE_IN_BYTES already. We don't limit traceback size.
"""
with self.__lock:
if os.getpid() != self._birth_pid:
raise ScribeIsNotForkSafeError
if not self.connected:
self._maybe_reconnect()
if self.connected:
log_entry = scribe_thrift.LogEntry(category=scribify(stream),
message=line + b'\n')
try:
return self.client.Log(messages=[log_entry])
except Exception as e:
try:
self.report_status(
True,
'yelp_clog failed to log to scribe server with '
' exception: %s(%s)' % (type(e), six.text_type(e)))
finally:
self.close()
self.last_connect_time = time.time()
# Don't reconnect if report_status raises an exception
self._maybe_reconnect()
def log_line(self, stream, line):
"""Log a single line. It should not include any newline characters.
If the line size is over 50 MB, an exception raises and the line will be dropped.
If the line size is over 5 MB, a message consisting origin stream information
will be recorded at WHO_CLOG_LARGE_LINE_STREAM (in json format).
"""
# log unicodes as their utf-8 encoded representation
if isinstance(line, six.text_type):
line = line.encode('UTF-8')
# check log line size
if len(line) <= WARNING_LINE_SIZE_IN_BYTES:
self._log_line_no_size_limit(stream, line)
elif len(line) <= MAX_LINE_SIZE_IN_BYTES:
self._log_line_no_size_limit(stream, line)
# log the origin of the stream with traceback to WHO_CLOG_LARGE_LINE_STREAM category
origin_info = {}
origin_info['stream'] = stream
origin_info['line_size'] = len(line)
origin_info['traceback'] = ''.join(traceback.format_stack())
log_line = json.dumps(origin_info).encode('UTF-8')
self._log_line_no_size_limit(WHO_CLOG_LARGE_LINE_STREAM, log_line)
self.report_status(
False,
'The log line size is larger than %r bytes (monitored in \'%s\')'
% (WARNING_LINE_SIZE_IN_BYTES, WHO_CLOG_LARGE_LINE_STREAM))
else:
# raise an exception if too large
self.report_status(
True,
'The log line is dropped (line size larger than %r bytes)' %
MAX_LINE_SIZE_IN_BYTES)
raise LogLineIsTooLongError(
'The max log line size allowed is %r bytes' %
MAX_LINE_SIZE_IN_BYTES)
def close(self):
self.transport.close()
self.connected = False
