def run(self):
"""Actually run the daemon."""
# Attach a few things to the handler class
# so methods there have easy access to them.
self.handler.request_log = self.setup_request_log(**self.kw_options)
self.handler.error_log = self.setup_error_log(**self.kw_options)
self.handler.output_to_screen = self.output_to_screen
self.handler.version = self.version
# If any additional options were sent in `__init__`,
# set them to the handler also.
for key, value in self.kw_options.iteritems():
setattr(self.handler, key, value)
# Get my server address as a tuple.
server = socketserver.ThreadingTCPServer(self.address, self.handler)
try:
# Make the server accept commands.
server.serve_forever()
except KeyboardInterrupt:
self.pre_shutdown()
# Get the hell outta dodge...
sys.exit(0)
def __init__(self, index, key, nics):
# disabling eager
# tf.compat.v1.disable_eager_execution()
service_name = SparkTaskService.NAME_FORMAT % index
self._index = index
self._service_name = service_name
self._wire = Wire(key)
self._nics = nics
self._server, _ = find_port(
lambda addr: socketserver.ThreadingTCPServer(
addr, self._make_handler()))
self._port = self._server.socket.getsockname()[1]
self._addresses = self._get_local_addresses()
self._thread = threading.Thread(target=self._server.serve_forever)
self._thread.daemon = True
self._thread.start()
self.local_task_index = 0
self._initial_registration_complete = False
self._workload_complete = False
self._wait_cond = threading.Condition()
self._sub_epoch_thread = None
self._sub_epoch_status = None
self._train_readers = {}
self._val_readers = {}
def __init__(self, service_name, key, nics):
self._service_name = service_name
self._wire = Wire(key)
self._nics = nics
self._server, _ = find_port(
lambda addr: socketserver.ThreadingTCPServer(
addr, self._make_handler()))
self._port = self._server.socket.getsockname()[1]
self._addresses = self._get_local_addresses()
self._thread = in_thread(target=self._server.serve_forever)
def __init__(self, service_name, key):
self._service_name = service_name
self._wire = Wire(key)
self._server, _ = find_port(
lambda addr: socketserver.ThreadingTCPServer(
addr, self._make_handler()))
self._port = self._server.socket.getsockname()[1]
self._thread = threading.Thread(target=self._server.serve_forever)
self._thread.daemon = True
self._thread.start()
def _make_server(self):
min_port = 1024
max_port = 65536
num_ports = max_port - min_port
start_port = random.randrange(0, num_ports)
for port_offset in range(num_ports):
try:
port = min_port + (start_port + port_offset) % num_ports
return socketserver.ThreadingTCPServer(('0.0.0.0', port), self._make_handler())
except:
pass
raise Exception('Unable to find a port to bind to.')
def __init__(self, service_name, key, nics):
self._service_name = service_name
self._wire = Wire(key)
self._nics = nics
self._server, _ = find_port(
lambda addr: socketserver.ThreadingTCPServer(
addr, self._make_handler()),
pedl_provisioned_port=0,
verbose=False,
)
self._server._block_on_close = True
self._port = self._server.socket.getsockname()[1]
self._addresses = self._get_local_addresses()
self._thread = in_thread(target=self._server.serve_forever)
def __init__(self, num_workers, key, nics):
self._service_name = SparkDriverService.NAME
self._wire = Wire(key)
self._nics = nics
self._server, _ = find_port(
lambda addr: socketserver.ThreadingTCPServer(
addr, self._make_handler()))
self._port = self._server.socket.getsockname()[1]
self._addresses = self._get_local_addresses()
self._thread = threading.Thread(target=self._server.serve_forever)
self._thread.daemon = True
self._thread.start()
self._num_workers = num_workers
self._all_task_addresses = {}
self._task_addresses_for_driver = {}
self._task_host_hash_indices = {}
self._wait_cond = threading.Condition()
self._spark_job_failed = False
def _start_thread(self, system):
self._server = socketserver.ThreadingTCPServer(
("", 0), # Let the OS pick a port for us
FMSTCPHandler, # Handler of the requests
False)
self._server.allow_reuse_address = True
self._server.server_bind()
self._server.server_activate()
self._server.terminate = False
self._server.system = system
self._server.sleep_in_seconds = self._config["sleep"]
if "min_event_per_burst" in self._config:
self._server.min_event_per_burst = \
int(self._config["min_event_per_burst"])
else:
self._server.min_event_per_burst = 500
self._server_thread = threading.Thread(target=self._serve_forever)
self._server_thread.start()
port_used = self._server.socket.getsockname()[1]
return port_used
def _configure_server(self):
"""Creates and configures the Server object
The server object is configured according to
the configuration of this source module
"""
self._server = socketserver.ThreadingTCPServer(
(self._config["params"]["host"], self._config["params"]["port"]),
MonanasTCPHandler, False)
self._server.generate = getattr(
self, "_generate_" + self._config["params"]["model"]["name"])
# self._server.validate = getattr(
# source_model, self._config["validate"])
self._server.allow_reuse_address = True
self._server.server_bind()
self._server.server_activate()
self._server.terminate = False
self._server.generate_alerts_per_second =\
self._config["params"]["alerts_per_burst"]
self._server.generate_idle_time_between_bursts =\
self._config["params"]["idle_time_between_bursts"]
self._server_thread = th.Thread(target=self._server.serve_forever)
self._is_server_running = False
def start_server(host, port, config, serving_state, backend_service_fn,
preprocess_fn, translate_fn, postprocess_fn):
"""Start a serving service.
This function will only return on SIGINT or SIGTERM signals.
Args:
host: The hostname of the service.
port: The port used by the service.
serving_state: The framework state to propagate to pre/postprocessing callbacks.
backend_service_fn: A callable to start the framework dependent backend service.
preprocess_fn: A callable taking (serving_state, text, config) and returning tokens.
translation_fn: A callable that forwards the request to the translation backend.
postprocess_fn: A callable taking (serving_state, src_tokens, tgt_tokens, config)
and returning text.
"""
global backend_process
global backend_info
backend_process, backend_info = backend_service_fn()
global_timeout = None
global_max_batch_size = None
serving_config = config.get('serving')
if serving_config is not None and isinstance(serving_config, dict):
global_timeout = config.get('timeout')
global_max_batch_size = config.get('max_batch_size')
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_POST(self):
if self.path == '/translate':
self.translate()
elif self.path == '/unload_model':
self.unload_model()
elif self.path == '/reload_model':
self.reload_model()
else:
self.send_error(404, 'invalid route %s' % self.path)
def translate(self):
global backend_process
if backend_process is not None and not _process_is_running(
backend_process):
self.send_error(503, 'backend service is unavailable')
return
header_fn = (self.headers.getheader if hasattr(
self.headers, "getheader") else self.headers.get)
content_len = int(header_fn('content-length', 0))
if content_len == 0:
self.send_error(400, 'missing request data')
return
post_body = self.rfile.read(content_len)
try:
result = run_request(json.loads(six.ensure_str(post_body)),
functools.partial(preprocess_fn,
serving_state),
functools.partial(translate_fn,
info=backend_info),
functools.partial(postprocess_fn,
serving_state),
config=config,
max_batch_size=global_max_batch_size,
timeout=global_timeout)
except ValueError as e:
self.send_error(400, str(e))
except RuntimeError as e:
self.send_error(504, str(e))
else:
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write(six.ensure_binary(json.dumps(result)))
def unload_model(self):
global backend_process
global backend_info
if backend_process is not None and _process_is_running(
backend_process):
backend_process.terminate()
backend_process = None
backend_info = None
self.send_response(200)
def reload_model(self):
global backend_process
global backend_info
if backend_process is not None and _process_is_running(
backend_process):
backend_process.terminate()
backend_process, backend_info = backend_service_fn()
self.send_response(200)
try:
frontend_server = socketserver.ThreadingTCPServer((host, port),
ServerHandler)
except socket.error as e:
if backend_process is not None:
backend_process.terminate()
raise e
def shutdown(signum, frame):
frontend_server.shutdown()
if backend_process is not None:
backend_process.terminate()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
logger.info('Serving model on port %d', port)
server_thread = threading.Thread(target=frontend_server.serve_forever)
server_thread.start()
while server_thread.is_alive():
time.sleep(1)
frontend_server.server_close()
def start_server(host, port, config, serving_state, backend_service_fn,
preprocess_fn, translate_fn, postprocess_fn):
"""Start a serving service.
This function will only return on SIGINT or SIGTERM signals.
Args:
host: The hostname of the service.
port: The port used by the service.
serving_state: The framework state to propagate to pre/postprocessing callbacks.
backend_service_fn: A callable to start the framework dependent backend service.
preprocess_fn: A callable taking (serving_state, text, extra_config) and returning tokens.
translation_fn: A callable that forwards the request to the translation backend.
postprocess_fn: A callable taking (serving_state, src_tokens, tgt_tokens, extra_config)
and returning text.
"""
global backend_process
global backend_info
backend_process, backend_info = backend_service_fn()
global_timeout = None
global_max_batch_size = None
if config is not None and isinstance(config, dict):
global_timeout = config.get('timeout')
global_max_batch_size = config.get('max_batch_size')
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_POST(self):
if self.path == '/translate':
self.translate()
elif self.path == '/unload_model':
self.unload_model()
elif self.path == '/reload_model':
self.reload_model()
else:
self.send_error(404, 'invalid route %s' % self.path)
def translate(self):
global backend_process
if backend_process is not None and not _process_is_running(
backend_process):
self.send_error(503, 'backend service is unavailable')
return
header_fn = (self.headers.getheader if hasattr(
self.headers, "getheader") else self.headers.get)
content_len = int(header_fn('content-length', 0))
if content_len == 0:
self.send_error(400, 'missing request data')
return
post_body = self.rfile.read(content_len)
try:
body = json.loads(post_body)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Incoming request: %s",
json.dumps(body, ensure_ascii=False))
except ValueError:
self.send_error(400, 'badly formatted JSON data')
return
self.handle_request(body)
def unload_model(self):
global backend_process
global backend_info
if backend_process is not None and _process_is_running(
backend_process):
backend_process.terminate()
backend_process = None
backend_info = None
self.send_response(200)
def reload_model(self):
global backend_process
global backend_info
if backend_process is not None and _process_is_running(
backend_process):
backend_process.terminate()
backend_process, backend_info = backend_service_fn()
self.send_response(200)
def send_result(self, result):
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
data = json.dumps(result)
if not isinstance(data, six.binary_type):
data = data.encode("utf-8")
self.wfile.write(data)
def handle_request(self, request):
if 'src' not in request:
self.send_error(400, 'missing src field')
return
results = {'tgt': []}
if not request['src']:
self.send_result(results)
return
if not isinstance(request['src'], list):
self.send_error(400, 'src field must be a list')
return
timeout = global_timeout
max_batch_size = global_max_batch_size
batch_config = {}
if 'options' in request and isinstance(request['options'], dict):
timeout = request['options'].get('timeout', timeout)
max_batch_size = request['options'].get(
'max_batch_size', max_batch_size)
batch_config = request['options'].get('config', batch_config)
extra_config = []
batch_metadata = []
batch_offsets = []
batch_tokens = []
offset = 0
for src in request['src']:
local_config = batch_config
if 'config' in src:
local_config = merge_dict(copy.deepcopy(local_config),
src['config'])
data = preprocess_fn(serving_state, src['text'], local_config)
# Preprocessing may return additional metadata.
if isinstance(data, tuple):
tokens, metadata = data
else:
tokens, metadata = data, None
# Preprocessing may split input text into multiple parts.
if tokens and isinstance(tokens[0], list):
size = len(tokens)
# Flatten the parts in the batch collection.
batch_tokens.extend(tokens)
batch_metadata.extend(metadata)
else:
size = 1
batch_tokens.append(tokens)
batch_metadata.append(metadata)
extra_config.append(local_config)
batch_offsets.append((offset, offset + size))
offset += size
if max_batch_size is not None and len(
batch_tokens) > max_batch_size:
offset = 0
batch_hypotheses = []
while offset < len(batch_tokens):
lower_bound = offset
upper_bound = min(offset + max_batch_size,
len(batch_tokens))
batch_hypotheses.extend(
translate_fn(batch_tokens[lower_bound:upper_bound],
backend_info,
timeout=timeout))
offset = upper_bound
else:
batch_hypotheses = translate_fn(batch_tokens,
backend_info,
timeout=timeout)
if batch_hypotheses is None:
self.send_error(504, 'translation request timed out')
return
for local_config, offset in zip(extra_config, batch_offsets):
hypotheses = batch_hypotheses[offset[0]:offset[1]]
num_parts = offset[1] - offset[0]
num_hypotheses = len(hypotheses[0])
src_tokens = batch_tokens[offset[0]:offset[1]]
src_metadata = batch_metadata[offset[0]:offset[1]]
result = []
for h in range(num_hypotheses):
if num_parts == 1:
src = src_tokens[0]
if src_metadata[0] is not None:
src = (src, src_metadata[0])
tgt = hypotheses[0][h].output
scores = hypotheses[0][h].score
attention = hypotheses[0][h].attention
else:
# For multi parts inputs, send all result parts to the postprocessing.
src = (src_tokens, src_metadata)
tgt = []
scores = []
attention = None
for j in range(num_parts):
tgt.append(hypotheses[j][h].output)
scores.append(hypotheses[j][h].score)
result.append(
_build_result(lambda src, tgt: postprocess_fn(
serving_state, src, tgt, local_config),
src,
tgt,
scores=scores,
attention=attention,
num_parts=num_parts))
results['tgt'].append(result)
self.send_result(results)
try:
frontend_server = socketserver.ThreadingTCPServer((host, port),
ServerHandler)
except socket.error as e:
if backend_process is not None:
backend_process.terminate()
raise e
def shutdown(signum, frame):
frontend_server.shutdown()
if backend_process is not None:
backend_process.terminate()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
logger.info('Serving model on port %d', port)
server_thread = threading.Thread(target=frontend_server.serve_forever)
server_thread.start()
while server_thread.is_alive():
time.sleep(1)
frontend_server.server_close()
def start_server(
host,
port,
config,
backend_service_fn,
translate_fn,
preprocessor=None,
postprocessor=None,
backend_info_fn=None,
rebatch_request=True,
):
"""Start a serving service.
This function will only return on SIGINT or SIGTERM signals.
Args:
host: The hostname of the service.
port: The port used by the service.
backend_service_fn: A callable to start the framework dependent backend service.
translation_fn: A callable that forwards the request to the translation backend.
backend_info_fn: A callable returning some information about the backend service,
and whether it can accept new requests or not.
preprocessor: A Processor instance for preprocessing.
postprocessor: A Processor instance for postprocessing.
rebatch_request: If True, incoming requests are rebatched according to
max_batch_size. Otherwise, max_batch_size is passed as a translation option
to translate_fn which takes responsibility over batching.
"""
global backend_process
global backend_info
backend_process, backend_info = backend_service_fn()
serving_config = config.get("serving")
if serving_config is None:
serving_config = {}
global_timeout = serving_config.get("timeout")
global_max_batch_size = serving_config.get("max_batch_size")
def _backend_is_reachable():
return (backend_info is not None and backend_process is None
or _process_is_running(backend_process))
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def _send_response(self, data, status=200):
self.send_response(status)
self.send_header("Content-type", "application/json")
self.end_headers()
self.wfile.write(six.ensure_binary(json.dumps(data)))
def _send_error(self, status, message, from_request=None):
if from_request is not None:
logger.exception(
"Exception raised for request:\n%s",
json.dumps(from_request, ensure_ascii=False),
)
data = {"message": message}
self._send_response(data, status=status)
def do_GET(self):
if self.path == "/status":
self.status()
elif self.path == "/health":
self.health()
else:
self._send_error(404, "invalid route %s" % self.path)
def do_POST(self):
if self.path == "/translate":
self.translate()
elif self.path == "/unload_model":
self.unload_model()
elif self.path == "/reload_model":
self.reload_model()
else:
self._send_error(404, "invalid route %s" % self.path)
def translate(self):
if not _backend_is_reachable():
self._send_error(503, "backend service is unavailable")
return
header_fn = (self.headers.getheader if hasattr(
self.headers, "getheader") else self.headers.get)
content_len = int(header_fn("content-length", 0))
if content_len == 0:
self._send_error(400, "missing request data")
return
post_body = self.rfile.read(content_len)
request = None
try:
request = json.loads(six.ensure_str(post_body))
result = run_request(
request,
functools.partial(translate_fn, backend_info),
preprocessor=preprocessor,
postprocessor=postprocessor,
config=config,
rebatch_request=rebatch_request,
max_batch_size=global_max_batch_size,
timeout=global_timeout,
)
except InvalidRequest as e:
self._send_error(400, str(e), from_request=request)
except TranslationTimeout as e:
self._send_error(504, str(e), from_request=request)
except Exception as e:
self._send_error(500, str(e), from_request=request)
else:
self._send_response(result)
def health(self):
if not _backend_is_reachable():
self._send_error(503, "backend service is unavailable")
return
if backend_info_fn is None:
self._send_response({})
return
info, available = backend_info_fn(serving_config, backend_info)
self._send_response(info, status=200 if available else 503)
def status(self):
if backend_info is None:
status = "unloaded"
else:
status = "ready"
self._send_response({"status": status})
def unload_model(self):
global backend_process
global backend_info
if backend_process is not None and _process_is_running(
backend_process):
backend_process.terminate()
backend_process = None
backend_info = None
self.status()
def reload_model(self):
global backend_process
global backend_info
if backend_process is not None and _process_is_running(
backend_process):
backend_process.terminate()
backend_process, backend_info = backend_service_fn()
self.status()
try:
frontend_server = socketserver.ThreadingTCPServer((host, port),
ServerHandler)
except socket.error as e:
if backend_process is not None:
backend_process.terminate()
raise e
def shutdown(signum, frame):
frontend_server.shutdown()
if backend_process is not None:
backend_process.terminate()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
logger.info("Serving model on port %d", port)
server_thread = threading.Thread(target=frontend_server.serve_forever)
server_thread.start()
while server_thread.is_alive():
time.sleep(1)
frontend_server.server_close()
def proxy_server():
httpd = socketserver.ThreadingTCPServer(("", 0), Proxy)
proxy_process = multiprocessing.Process(target=httpd.serve_forever)
proxy_process.start()
yield "http://{}:{}".format(*httpd.server_address)
proxy_process.terminate()
