def run(self): """Actually run the daemon.""" # Attach a few things to the handler class # so methods there have easy access to them. self.handler.request_log = self.setup_request_log(**self.kw_options) self.handler.error_log = self.setup_error_log(**self.kw_options) self.handler.output_to_screen = self.output_to_screen self.handler.version = self.version # If any additional options were sent in `__init__`, # set them to the handler also. for key, value in self.kw_options.iteritems(): setattr(self.handler, key, value) # Get my server address as a tuple. server = socketserver.ThreadingTCPServer(self.address, self.handler) try: # Make the server accept commands. server.serve_forever() except KeyboardInterrupt: self.pre_shutdown() # Get the hell outta dodge... sys.exit(0)
def __init__(self, index, key, nics): # disabling eager # tf.compat.v1.disable_eager_execution() service_name = SparkTaskService.NAME_FORMAT % index self._index = index self._service_name = service_name self._wire = Wire(key) self._nics = nics self._server, _ = find_port( lambda addr: socketserver.ThreadingTCPServer( addr, self._make_handler())) self._port = self._server.socket.getsockname()[1] self._addresses = self._get_local_addresses() self._thread = threading.Thread(target=self._server.serve_forever) self._thread.daemon = True self._thread.start() self.local_task_index = 0 self._initial_registration_complete = False self._workload_complete = False self._wait_cond = threading.Condition() self._sub_epoch_thread = None self._sub_epoch_status = None self._train_readers = {} self._val_readers = {}
def __init__(self, service_name, key, nics): self._service_name = service_name self._wire = Wire(key) self._nics = nics self._server, _ = find_port( lambda addr: socketserver.ThreadingTCPServer( addr, self._make_handler())) self._port = self._server.socket.getsockname()[1] self._addresses = self._get_local_addresses() self._thread = in_thread(target=self._server.serve_forever)
def __init__(self, service_name, key): self._service_name = service_name self._wire = Wire(key) self._server, _ = find_port( lambda addr: socketserver.ThreadingTCPServer( addr, self._make_handler())) self._port = self._server.socket.getsockname()[1] self._thread = threading.Thread(target=self._server.serve_forever) self._thread.daemon = True self._thread.start()
def _make_server(self): min_port = 1024 max_port = 65536 num_ports = max_port - min_port start_port = random.randrange(0, num_ports) for port_offset in range(num_ports): try: port = min_port + (start_port + port_offset) % num_ports return socketserver.ThreadingTCPServer(('0.0.0.0', port), self._make_handler()) except: pass raise Exception('Unable to find a port to bind to.')
def __init__(self, service_name, key, nics): self._service_name = service_name self._wire = Wire(key) self._nics = nics self._server, _ = find_port( lambda addr: socketserver.ThreadingTCPServer( addr, self._make_handler()), pedl_provisioned_port=0, verbose=False, ) self._server._block_on_close = True self._port = self._server.socket.getsockname()[1] self._addresses = self._get_local_addresses() self._thread = in_thread(target=self._server.serve_forever)
def __init__(self, num_workers, key, nics): self._service_name = SparkDriverService.NAME self._wire = Wire(key) self._nics = nics self._server, _ = find_port( lambda addr: socketserver.ThreadingTCPServer( addr, self._make_handler())) self._port = self._server.socket.getsockname()[1] self._addresses = self._get_local_addresses() self._thread = threading.Thread(target=self._server.serve_forever) self._thread.daemon = True self._thread.start() self._num_workers = num_workers self._all_task_addresses = {} self._task_addresses_for_driver = {} self._task_host_hash_indices = {} self._wait_cond = threading.Condition() self._spark_job_failed = False
def _start_thread(self, system): self._server = socketserver.ThreadingTCPServer( ("", 0), # Let the OS pick a port for us FMSTCPHandler, # Handler of the requests False) self._server.allow_reuse_address = True self._server.server_bind() self._server.server_activate() self._server.terminate = False self._server.system = system self._server.sleep_in_seconds = self._config["sleep"] if "min_event_per_burst" in self._config: self._server.min_event_per_burst = \ int(self._config["min_event_per_burst"]) else: self._server.min_event_per_burst = 500 self._server_thread = threading.Thread(target=self._serve_forever) self._server_thread.start() port_used = self._server.socket.getsockname()[1] return port_used
def _configure_server(self): """Creates and configures the Server object The server object is configured according to the configuration of this source module """ self._server = socketserver.ThreadingTCPServer( (self._config["params"]["host"], self._config["params"]["port"]), MonanasTCPHandler, False) self._server.generate = getattr( self, "_generate_" + self._config["params"]["model"]["name"]) # self._server.validate = getattr( # source_model, self._config["validate"]) self._server.allow_reuse_address = True self._server.server_bind() self._server.server_activate() self._server.terminate = False self._server.generate_alerts_per_second =\ self._config["params"]["alerts_per_burst"] self._server.generate_idle_time_between_bursts =\ self._config["params"]["idle_time_between_bursts"] self._server_thread = th.Thread(target=self._server.serve_forever) self._is_server_running = False
def start_server(host, port, config, serving_state, backend_service_fn, preprocess_fn, translate_fn, postprocess_fn): """Start a serving service. This function will only return on SIGINT or SIGTERM signals. Args: host: The hostname of the service. port: The port used by the service. serving_state: The framework state to propagate to pre/postprocessing callbacks. backend_service_fn: A callable to start the framework dependent backend service. preprocess_fn: A callable taking (serving_state, text, config) and returning tokens. translation_fn: A callable that forwards the request to the translation backend. postprocess_fn: A callable taking (serving_state, src_tokens, tgt_tokens, config) and returning text. """ global backend_process global backend_info backend_process, backend_info = backend_service_fn() global_timeout = None global_max_batch_size = None serving_config = config.get('serving') if serving_config is not None and isinstance(serving_config, dict): global_timeout = config.get('timeout') global_max_batch_size = config.get('max_batch_size') class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): def do_POST(self): if self.path == '/translate': self.translate() elif self.path == '/unload_model': self.unload_model() elif self.path == '/reload_model': self.reload_model() else: self.send_error(404, 'invalid route %s' % self.path) def translate(self): global backend_process if backend_process is not None and not _process_is_running( backend_process): self.send_error(503, 'backend service is unavailable') return header_fn = (self.headers.getheader if hasattr( self.headers, "getheader") else self.headers.get) content_len = int(header_fn('content-length', 0)) if content_len == 0: self.send_error(400, 'missing request data') return post_body = self.rfile.read(content_len) try: result = run_request(json.loads(six.ensure_str(post_body)), functools.partial(preprocess_fn, serving_state), functools.partial(translate_fn, info=backend_info), functools.partial(postprocess_fn, serving_state), config=config, max_batch_size=global_max_batch_size, timeout=global_timeout) except ValueError as e: self.send_error(400, str(e)) except RuntimeError as e: self.send_error(504, str(e)) else: self.send_response(200) self.send_header('Content-type', 'application/json') self.end_headers() self.wfile.write(six.ensure_binary(json.dumps(result))) def unload_model(self): global backend_process global backend_info if backend_process is not None and _process_is_running( backend_process): backend_process.terminate() backend_process = None backend_info = None self.send_response(200) def reload_model(self): global backend_process global backend_info if backend_process is not None and _process_is_running( backend_process): backend_process.terminate() backend_process, backend_info = backend_service_fn() self.send_response(200) try: frontend_server = socketserver.ThreadingTCPServer((host, port), ServerHandler) except socket.error as e: if backend_process is not None: backend_process.terminate() raise e def shutdown(signum, frame): frontend_server.shutdown() if backend_process is not None: backend_process.terminate() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) logger.info('Serving model on port %d', port) server_thread = threading.Thread(target=frontend_server.serve_forever) server_thread.start() while server_thread.is_alive(): time.sleep(1) frontend_server.server_close()
def start_server(host, port, config, serving_state, backend_service_fn, preprocess_fn, translate_fn, postprocess_fn): """Start a serving service. This function will only return on SIGINT or SIGTERM signals. Args: host: The hostname of the service. port: The port used by the service. serving_state: The framework state to propagate to pre/postprocessing callbacks. backend_service_fn: A callable to start the framework dependent backend service. preprocess_fn: A callable taking (serving_state, text, extra_config) and returning tokens. translation_fn: A callable that forwards the request to the translation backend. postprocess_fn: A callable taking (serving_state, src_tokens, tgt_tokens, extra_config) and returning text. """ global backend_process global backend_info backend_process, backend_info = backend_service_fn() global_timeout = None global_max_batch_size = None if config is not None and isinstance(config, dict): global_timeout = config.get('timeout') global_max_batch_size = config.get('max_batch_size') class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): def do_POST(self): if self.path == '/translate': self.translate() elif self.path == '/unload_model': self.unload_model() elif self.path == '/reload_model': self.reload_model() else: self.send_error(404, 'invalid route %s' % self.path) def translate(self): global backend_process if backend_process is not None and not _process_is_running( backend_process): self.send_error(503, 'backend service is unavailable') return header_fn = (self.headers.getheader if hasattr( self.headers, "getheader") else self.headers.get) content_len = int(header_fn('content-length', 0)) if content_len == 0: self.send_error(400, 'missing request data') return post_body = self.rfile.read(content_len) try: body = json.loads(post_body) if logger.isEnabledFor(logging.DEBUG): logger.debug("Incoming request: %s", json.dumps(body, ensure_ascii=False)) except ValueError: self.send_error(400, 'badly formatted JSON data') return self.handle_request(body) def unload_model(self): global backend_process global backend_info if backend_process is not None and _process_is_running( backend_process): backend_process.terminate() backend_process = None backend_info = None self.send_response(200) def reload_model(self): global backend_process global backend_info if backend_process is not None and _process_is_running( backend_process): backend_process.terminate() backend_process, backend_info = backend_service_fn() self.send_response(200) def send_result(self, result): self.send_response(200) self.send_header('Content-type', 'application/json') self.end_headers() data = json.dumps(result) if not isinstance(data, six.binary_type): data = data.encode("utf-8") self.wfile.write(data) def handle_request(self, request): if 'src' not in request: self.send_error(400, 'missing src field') return results = {'tgt': []} if not request['src']: self.send_result(results) return if not isinstance(request['src'], list): self.send_error(400, 'src field must be a list') return timeout = global_timeout max_batch_size = global_max_batch_size batch_config = {} if 'options' in request and isinstance(request['options'], dict): timeout = request['options'].get('timeout', timeout) max_batch_size = request['options'].get( 'max_batch_size', max_batch_size) batch_config = request['options'].get('config', batch_config) extra_config = [] batch_metadata = [] batch_offsets = [] batch_tokens = [] offset = 0 for src in request['src']: local_config = batch_config if 'config' in src: local_config = merge_dict(copy.deepcopy(local_config), src['config']) data = preprocess_fn(serving_state, src['text'], local_config) # Preprocessing may return additional metadata. if isinstance(data, tuple): tokens, metadata = data else: tokens, metadata = data, None # Preprocessing may split input text into multiple parts. if tokens and isinstance(tokens[0], list): size = len(tokens) # Flatten the parts in the batch collection. batch_tokens.extend(tokens) batch_metadata.extend(metadata) else: size = 1 batch_tokens.append(tokens) batch_metadata.append(metadata) extra_config.append(local_config) batch_offsets.append((offset, offset + size)) offset += size if max_batch_size is not None and len( batch_tokens) > max_batch_size: offset = 0 batch_hypotheses = [] while offset < len(batch_tokens): lower_bound = offset upper_bound = min(offset + max_batch_size, len(batch_tokens)) batch_hypotheses.extend( translate_fn(batch_tokens[lower_bound:upper_bound], backend_info, timeout=timeout)) offset = upper_bound else: batch_hypotheses = translate_fn(batch_tokens, backend_info, timeout=timeout) if batch_hypotheses is None: self.send_error(504, 'translation request timed out') return for local_config, offset in zip(extra_config, batch_offsets): hypotheses = batch_hypotheses[offset[0]:offset[1]] num_parts = offset[1] - offset[0] num_hypotheses = len(hypotheses[0]) src_tokens = batch_tokens[offset[0]:offset[1]] src_metadata = batch_metadata[offset[0]:offset[1]] result = [] for h in range(num_hypotheses): if num_parts == 1: src = src_tokens[0] if src_metadata[0] is not None: src = (src, src_metadata[0]) tgt = hypotheses[0][h].output scores = hypotheses[0][h].score attention = hypotheses[0][h].attention else: # For multi parts inputs, send all result parts to the postprocessing. src = (src_tokens, src_metadata) tgt = [] scores = [] attention = None for j in range(num_parts): tgt.append(hypotheses[j][h].output) scores.append(hypotheses[j][h].score) result.append( _build_result(lambda src, tgt: postprocess_fn( serving_state, src, tgt, local_config), src, tgt, scores=scores, attention=attention, num_parts=num_parts)) results['tgt'].append(result) self.send_result(results) try: frontend_server = socketserver.ThreadingTCPServer((host, port), ServerHandler) except socket.error as e: if backend_process is not None: backend_process.terminate() raise e def shutdown(signum, frame): frontend_server.shutdown() if backend_process is not None: backend_process.terminate() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) logger.info('Serving model on port %d', port) server_thread = threading.Thread(target=frontend_server.serve_forever) server_thread.start() while server_thread.is_alive(): time.sleep(1) frontend_server.server_close()
def start_server( host, port, config, backend_service_fn, translate_fn, preprocessor=None, postprocessor=None, backend_info_fn=None, rebatch_request=True, ): """Start a serving service. This function will only return on SIGINT or SIGTERM signals. Args: host: The hostname of the service. port: The port used by the service. backend_service_fn: A callable to start the framework dependent backend service. translation_fn: A callable that forwards the request to the translation backend. backend_info_fn: A callable returning some information about the backend service, and whether it can accept new requests or not. preprocessor: A Processor instance for preprocessing. postprocessor: A Processor instance for postprocessing. rebatch_request: If True, incoming requests are rebatched according to max_batch_size. Otherwise, max_batch_size is passed as a translation option to translate_fn which takes responsibility over batching. """ global backend_process global backend_info backend_process, backend_info = backend_service_fn() serving_config = config.get("serving") if serving_config is None: serving_config = {} global_timeout = serving_config.get("timeout") global_max_batch_size = serving_config.get("max_batch_size") def _backend_is_reachable(): return (backend_info is not None and backend_process is None or _process_is_running(backend_process)) class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): def _send_response(self, data, status=200): self.send_response(status) self.send_header("Content-type", "application/json") self.end_headers() self.wfile.write(six.ensure_binary(json.dumps(data))) def _send_error(self, status, message, from_request=None): if from_request is not None: logger.exception( "Exception raised for request:\n%s", json.dumps(from_request, ensure_ascii=False), ) data = {"message": message} self._send_response(data, status=status) def do_GET(self): if self.path == "/status": self.status() elif self.path == "/health": self.health() else: self._send_error(404, "invalid route %s" % self.path) def do_POST(self): if self.path == "/translate": self.translate() elif self.path == "/unload_model": self.unload_model() elif self.path == "/reload_model": self.reload_model() else: self._send_error(404, "invalid route %s" % self.path) def translate(self): if not _backend_is_reachable(): self._send_error(503, "backend service is unavailable") return header_fn = (self.headers.getheader if hasattr( self.headers, "getheader") else self.headers.get) content_len = int(header_fn("content-length", 0)) if content_len == 0: self._send_error(400, "missing request data") return post_body = self.rfile.read(content_len) request = None try: request = json.loads(six.ensure_str(post_body)) result = run_request( request, functools.partial(translate_fn, backend_info), preprocessor=preprocessor, postprocessor=postprocessor, config=config, rebatch_request=rebatch_request, max_batch_size=global_max_batch_size, timeout=global_timeout, ) except InvalidRequest as e: self._send_error(400, str(e), from_request=request) except TranslationTimeout as e: self._send_error(504, str(e), from_request=request) except Exception as e: self._send_error(500, str(e), from_request=request) else: self._send_response(result) def health(self): if not _backend_is_reachable(): self._send_error(503, "backend service is unavailable") return if backend_info_fn is None: self._send_response({}) return info, available = backend_info_fn(serving_config, backend_info) self._send_response(info, status=200 if available else 503) def status(self): if backend_info is None: status = "unloaded" else: status = "ready" self._send_response({"status": status}) def unload_model(self): global backend_process global backend_info if backend_process is not None and _process_is_running( backend_process): backend_process.terminate() backend_process = None backend_info = None self.status() def reload_model(self): global backend_process global backend_info if backend_process is not None and _process_is_running( backend_process): backend_process.terminate() backend_process, backend_info = backend_service_fn() self.status() try: frontend_server = socketserver.ThreadingTCPServer((host, port), ServerHandler) except socket.error as e: if backend_process is not None: backend_process.terminate() raise e def shutdown(signum, frame): frontend_server.shutdown() if backend_process is not None: backend_process.terminate() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) logger.info("Serving model on port %d", port) server_thread = threading.Thread(target=frontend_server.serve_forever) server_thread.start() while server_thread.is_alive(): time.sleep(1) frontend_server.server_close()
def proxy_server(): httpd = socketserver.ThreadingTCPServer(("", 0), Proxy) proxy_process = multiprocessing.Process(target=httpd.serve_forever) proxy_process.start() yield "http://{}:{}".format(*httpd.server_address) proxy_process.terminate()