def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as ex: logger.exception( f'ignore_exception {func.__name__} ex {ex.__class__.__name__} error: %s', ex)
async def staging_dh(decrypted_message_json, UUID): if "session_id" not in decrypted_message_json or "pub_key" not in decrypted_message_json: logger.exception( "Failed to get \"session_id\" or \"pub_key\" from message in staging_dh" ) return None, None # generate random AES256 key dh = DiffieHellman() # print(dh.publicKey) dh.genKey(decrypted_message_json['pub_key']) session_key_encoded = base64.b64encode(dh.getKey()).decode('utf-8') # print("created base64 encoded session key: " + session_key_encoded) # Save session_key and SESSIONID into database temp_uuid = str(uuid4()) try: stage_info = await db_objects.create( StagingInfo, session_id=decrypted_message_json['session_id'], session_key=session_key_encoded, payload_uuid=UUID, staging_uuid=temp_uuid) except Exception as e: logger.exception("Issue creating staging info for a new callback: " + str(e)) return None, None # encrypt a nonce and the session_key_encoded with the message['PUB'] public key from the agent response = { "uuid": temp_uuid, "session_key": dh.publicKey, "action": "staging_dh", "session_id": decrypted_message_json['session_id'] } # print("created response: " + js.dumps(response)) return response, stage_info
async def connect_and_consume_pt(): connection = None while connection is None: try: connection = await aio_pika.connect_robust(host="127.0.0.1", login="******", password="******", virtualhost="mythic_vhost") channel = await connection.channel() # declare our exchange await channel.declare_exchange('mythic_traffic', aio_pika.ExchangeType.TOPIC) # get a random queue that only the mythic server will use to listen on to catch all heartbeats queue = await channel.declare_queue('', exclusive=True) # bind the queue to the exchange so we can actually catch messages await queue.bind(exchange='mythic_traffic', routing_key="pt.status.#") await channel.set_qos(prefetch_count=50) logger.info(' [*] Waiting for messages in connect_and_consume_pt.') try: task = queue.consume(rabbit_pt_callback) result = await asyncio.wait_for(task, None) except Exception as r: logger.exception("Exception in connect_and_consume .consume: {}".format(str(r))) # print("Exception in connect_and_consume .consume: {}".format(str(e))) except (ConnectionError, ConnectionRefusedError) as c: logger.error("Connection to rabbitmq failed, trying again...") except Exception as e: logger.exception("Exception in connect_and_consume connect: {}".format(str(e))) # print("Exception in connect_and_consume connect: {}".format(str(e))) await asyncio.sleep(2)
async def connect_and_consume_c2_rpc(): connection = None while connection is None: try: connection = await aio_pika.connect_robust(host="127.0.0.1", login="******", password="******", virtualhost="mythic_vhost") channel = await connection.channel() # get a random queue that only the mythic server will use to listen on to catch all heartbeats queue = await channel.declare_queue('c2rpc_queue') await channel.set_qos(prefetch_count=50) logger.info(' [*] Waiting for messages in connect_and_consume_rpc.') try: task = queue.consume(partial(rabbit_c2_rpc_callback, channel.default_exchange)) result = await asyncio.wait_for(task, None) except Exception as e: logger.exception("Exception in connect_and_consume_c2_rpc .consume: {}".format(str(e))) # print("Exception in connect_and_consume .consume: {}".format(str(e))) except (ConnectionError, ConnectionRefusedError) as c: logger.error("Connection to rabbitmq failed, trying again...") except Exception as e: logger.exception("Exception in connect_and_consume_c2_rpc connect: {}".format(str(e))) # print("Exception in connect_and_consume connect: {}".format(str(e))) await asyncio.sleep(2)
async def rabbit_heartbeat_callback(message: aio_pika.IncomingMessage): with message.process(): pieces = message.routing_key.split(".") #print(" [x] %r:%r" % ( # message.routing_key, # message.body #)) try: if pieces[0] == "c2": query = await db_model.c2profile_query() try: profile = await db_objects.get(query, name=pieces[2], deleted=False) except Exception as e: print("Sending sync message to {}".format(pieces[2])) await send_c2_rabbitmq_message(pieces[2], "sync_classes", "", "") return if profile.last_heartbeat < datetime.datetime.utcnow() + datetime.timedelta(seconds=-30) or not profile.container_running: profile.running = False # container just started, clearly the inner service isn't running #print("setting running to false") profile.container_running = True profile.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(profile) elif pieces[0] == "pt": query = await db_model.payloadtype_query() try: payload_type = await db_objects.get(query, ptype=pieces[2], deleted=False) payload_type.container_running = True payload_type.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(payload_type) except Exception as e: await send_pt_rabbitmq_message(pieces[2], "sync_classes", "", "") except Exception as e: logger.exception("Exception in rabbit_heartbeat_callback: {}, {}".format(pieces, str(e)))
async def rabbit_heartbeat_callback(message: aio_pika.IncomingMessage): with message.process(): pieces = message.routing_key.split(".") #print(" [x] %r:%r" % ( # message.routing_key, # message.body #)) try: if pieces[0] == "c2": query = await db_model.c2profile_query() profile = await db_objects.get(query, name=pieces[2]) if profile.last_heartbeat < datetime.datetime.utcnow() + datetime.timedelta(seconds=-30) or not profile.container_running: profile.running = False # container just started, clearly the inner service isn't running profile.container_running = True profile.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(profile) elif pieces[0] == "pt": if pieces[2] != 'external': query = await db_model.payloadtype_query() payload_type = await db_objects.get(query, ptype=pieces[2]) payload_type.container_running = True payload_type.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(payload_type) # now send updated PT code to everybody transform_code = open("./app/api/transforms/transforms.py", 'rb').read() await send_pt_rabbitmq_message("*", "load_transform_code", base64.b64encode(transform_code).decode('utf-8')) except Exception as e: logger.exception("Exception in rabbit_heartbeat_callback: {}, {}".format(pieces, str(e)))
async def add_command_attack_to_task(task, command): try: query = await db_model.attackcommand_query() attack_mappings = await db_objects.execute(query.where(db_model.ATTACKCommand.command == command)) for attack in attack_mappings: try: query = await db_model.attacktask_query() # try to get the query, if it doens't exist, then create it in the exception await db_objects.get(query, task=task, attack=attack.attack) except Exception as e: await db_objects.create(db_model.ATTACKTask, task=task, attack=attack.attack) # now do the artifact adjustments as well query = await db_model.artifacttemplate_query() artifacts = await db_objects.execute(query.where( (db_model.ArtifactTemplate.command == command) & (db_model.ArtifactTemplate.deleted == False))) for artifact in artifacts: temp_string = artifact.artifact_string if artifact.command_parameter is not None and artifact.command_parameter != 'null': # we need to swap out temp_string's replace_string with task's param's command_parameter.name value parameter_dict = json.loads(task.params) temp_string = temp_string.replace(artifact.replace_string, str(parameter_dict[artifact.command_parameter.name])) else: # we need to swap out temp_string's replace_string with task's params value if artifact.replace_string != "": temp_string = temp_string.replace(artifact.replace_string, str(task.params)) await db_objects.create(db_model.TaskArtifact, task=task, artifact_template=artifact, artifact_instance=temp_string) except Exception as e: logger.exception(str(sys.exc_info()[-1].tb_lineno) + " " + str(e)) # print(str(sys.exc_info()[-1].tb_lineno) + " " + str(e)) raise e
async def start(self, delay: float) -> NoReturn: # TODO: Need to tweak the code a bit logger.debug(f"Supervisor started") while True: for task in self.app.tasks: if task in self.tasks: if task.done(): if task.cancelled(): logger.warning(f"{task.get_name()} cancled.") elif task.exception(): try: task.result() except: logger.exception( f"{task.get_name()} is raise exception.") logger.warning("Purge task") self.app.purge_tasks() # type: ignore registered_task = self.tasks[task] del self.tasks[task] logger.warning(f"Try restart {registered_task.name}.") self.add_task( registered_task.setup_func, registered_task.delay, registered_task.name, ) await sleep(delay)
async def postRequest(self, url, json, max_retry=5): result = None status = self.SERVICE_UNAVAILABLE_STATUS retry_count = 1 while retry_count <= max_retry: try: async with self.client.post(url, json=json) as resp: result, status = await self.getJson(resp) if result is None: logger.error( f'No result received from {url}. Retry {retry_count} of {max_retry}' ) else: return result, status except ConnectionResetError: logger.error( f'Connection reset on post request - {url}. Retry {retry_count} of {max_retry}' ) except Exception: logger.exception( f'Unknown exception occurred on get request {url}. Retry {retry_count} of {max_retry}' ) # Only gets here on failure retry_count += 1 await self.resetConnection() await asyncio.sleep(0.3) # (None, some status) return result, status
async def get_league_data(app, player_cookie, league_id): """Returns the league table for the requested league_id. Args: app (obj): Then sanic app. player_cookie (str): Cookie used for FPL api. league_id (str): The ID of the requested league. current_gameweek (obj): Information about the current gameweek. Returns: obj: The requested league table. """ try: league_data = await get_local_league_data(app.db, league_id) remote_data = False except LocalDataNotFoundException: league_data = await get_remote_league_data(league_id, player_cookie, app) remote_data = True except Exception as e: logger.exception(e) league_data = await get_remote_league_data(league_id, player_cookie, app) remote_data = True return remote_data, league_data
def list_jobs(): job_base_dir = ConfigurationManager.get_confs('mljob').get( 'job', 'dir') try: job_ids = [ file for file in os.listdir(job_base_dir) if os.path.isdir(os.path.join(job_base_dir, file)) ] results = [] for job_id in job_ids: try: logger.debug(f'find one job with id={job_id}') item = {} item['id'] = job_id status = MLJob.get_status_by_id(job_id) item['status'] = status.name meta = MLJob.get_meta(job_id) for key in ['type', 'name']: item[key] = meta[key] results.append(item) except Exception: logger.exception(f'failed to retrieve job id={job_id}') return results except Exception: logger.exception('failed to list job') return []
def response(self, request, exception): """Fetches and executes an exception handler and returns a response object :param request: Request :param exception: Exception to handle :return: Response object """ handler = self.lookup(exception) response = None try: if handler: response = handler(request, exception) if response is None: response = self.default(request, exception) except Exception: self.log(format_exc()) try: url = repr(request.url) except AttributeError: url = "unknown" response_message = ("Exception raised in exception handler " '"%s" for uri: %s') logger.exception(response_message, handler.__name__, url) if self.debug: return text(response_message % (handler.__name__, url), 500) else: return text("An error occurred while handling an error", 500) return response
async def user(request): try: user = UserService.getUsers() return response.json({'message': user.get('result')}, status=200) except Exception: logger.exception('faile to get all users') return response.json({'message': 'Server Internal error'}, status=500)
async def connect_and_consume_heartbeats(): connection = None while connection is None: try: connection = await aio_pika.connect_robust( host="127.0.0.1", login="******", password="******", virtualhost="apfell_vhost") channel = await connection.channel() # declare our exchange await channel.declare_exchange('apfell_traffic', aio_pika.ExchangeType.TOPIC) # get a random queue that only the apfell server will use to listen on to catch all heartbeats queue = await channel.declare_queue('', exclusive=True) # bind the queue to the exchange so we can actually catch messages await queue.bind(exchange='apfell_traffic', routing_key="*.heartbeat.#") await channel.set_qos(prefetch_count=20) logger.info( ' [*] Waiting for messages in connect_and_consume_heartbeats.') try: task = queue.consume(rabbit_heartbeat_callback) result = await asyncio.wait_for(task, None) except Exception as e: logger.exception( "Exception in connect_and_consume .consume: {}".format( str(e))) # print("Exception in connect_and_consume .consume: {}".format(str(e))) except Exception as e: logger.exception( "Exception in connect_and_consume connect: {}".format(str(e))) # print("Exception in connect_and_consume connect: {}".format(str(e))) await asyncio.sleep(2)
def predict(job_id, payload): data = payload['data'] input_type = payload['input_type'] try: model = MLJob.get_model(job_id) if input_type == 'csv': csv_data = BytesIO(base64.b64decode(data)) df = pd.read_csv(csv_data, sep=",") df_prediction = model.predict(df) output_data = df_prediction.to_csv(index=False) result = {} result['data'] = base64.b64encode(output_data.encode('utf-8')) return result elif input_type == 'dataset': dataset = DatasetManager.get_dataset(data) df = dataset.get_df() df_prediction = model.predict(df) payload = {} payload["cols"], payload["rows"] = df_to_cols_rows( df_prediction) return payload else: message = f'input type {input_type} is not supported for prediction' logger.error(message) raise RuntimeError(message) except Exception as e: logger.exception( f'failed to do prediction for data={data} id={job_id} error={e}' ) raise e
async def list_dashboard(request): try: dashboard = get_dashboards() return response.json(dashboard, status=200) except Exception: logger.exception("faile to list dashboard") return response.json({}, status=500)
async def determine_current_gameweek(gameweeks): """Determines the current gameweek. Args: gameweeks (obj): An array containing all the gameweeks data. Returns: obj: The current gameweek info. Raises: FantasyDataException: If there is an error processing the data returned by the FPL API. """ try: for gameweek in gameweeks: if gameweek['is_current']: end_time = None try: end_time = gameweeks[gameweek['id']]['deadline_time'] except IndexError: logger.debug('Last day of season, end_time set to None') relevant_gameweek_info = { "id": gameweek['id'], "start_time": gameweek['deadline_time'], # The deadline time of the following game week # (can use ID as array index starts at 0). # Set to none if no next gameweek (last day of season). "end_time": end_time } return relevant_gameweek_info logger.error('NO GAMEWEEKS CURRENTLY ACTIVE') raise FantasyDataException(ENTRY_DATA_ERROR_MSG) except Exception as e: logger.exception(e) raise FantasyDataException(ENTRY_DATA_ERROR_MSG)
async def _handle_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): while not reader.at_eof(): try: array = await reader.readline() if reader.at_eof(): break assert array[:1] == b'*' count = parse_int(array[1:-2]) items = [await read_bulk_string(reader) for _ in range(count)] command = items[0] params = items[1:] try: with timeout(REDIS_SERVER_TIMEOUT): await self._execute(command, params, writer) except Exception: logger.exception(f"Command failed: {command}") writer.write(encode_error("Command failed")) except Exception: logger.exception(f"Invalid command: {array}") writer.write(encode_error("Invalid command")) writer.close()
async def get_confs(request, domain): try: config = ConfigurationManager.get_confs_values(domain) return response.json(config, status=200) except Exception: logger.exception('failed to get content of configurations') return response.json({}, status=500)
async def list_confs(request): try: config = ConfigurationManager.list_confs() return response.json(config, status=200) except Exception: logger.exception('failed to list configurations') return response.json({}, status=500)
async def get_remote_league_data(league_id, player_cookie, app): """Function responsible for gathering remote league data processing it and storing it locally Args: league_id (int): The ID of the league to fetch. player_cookie (obj): The player cookie used for authentication on the FPL API. app (obj): The sanic app. Returns: obj: The league data in a useful format. """ logger.info('Fetching remote league data') url = app.config.FPL_URL + app.config.LEAGUE_DATA.format( league_id=league_id ) remote_league_data = await process_remote_league_data( await call_fpl_endpoint( url, player_cookie, app.config ) ) try: await put_local_league_data(app.db, remote_league_data) except Exception as e: # Log the error but we don't care if we cant save locally. logger.exception(e) return remote_league_data
async def get_job(request, id): try: job = MLJobManager.get_job(id) return response.json(job, status=200) except Exception: logger.exception('faile to get ml job') return response.json({}, status=500)
def predict(job_id, payload): data = payload['data'] input_type = payload['input_type'] try: model = MLJob.get_model(job_id) if input_type == 'csv': csv_data = StringIO(data) df = pd.read_csv(csv_data, sep=",") df_prediction = model.predict(df) output_data = df_prediction.to_csv() return output_data elif input_type == 'dataset': dataset = DatasetManager.get_dataset(data) df = dataset.get_df() df_prediction = model.predict(df) payload = {} payload["cols"], payload["rows"] = df_to_cols_rows( df_prediction) return payload else: message = f'input type {input_type} is not supported for prediction' logger.error(message) raise RuntimeError(message) except Exception as e: logger.exception( f'failed to do prediction for data={data} id={job_id} error={e}' ) raise e
async def send_pt_rabbitmq_message(payload_type, command, message_body, username): try: connection = await aio_pika.connect(host="127.0.0.1", login="******", password="******", virtualhost="apfell_vhost") channel = await connection.channel() # declare our exchange exchange = await channel.declare_exchange('apfell_traffic', aio_pika.ExchangeType.TOPIC) message = aio_pika.Message( message_body.encode(), delivery_mode=aio_pika.DeliveryMode.PERSISTENT) # Sending the message await exchange.publish(message, routing_key="pt.task.{}.{}.{}".format( payload_type, command, base64.b64encode( username.encode()).decode('utf-8'))) await connection.close() return {"status": "success"} except Exception as e: logger.exception(str(sys.exc_info()[-1].tb_lineno) + " " + str(e)) # print(str(sys.exc_info()[-1].tb_lineno) + " " + str(e)) return { "status": 'error', 'error': "Failed to connect to rabbitmq, refresh" }
async def delete_dataset(request, id): try: DatasetManager.delete_dataset(id) return response.json({}, status=204) except Exception: logger.exception('faile to delete dataset') return response.json({}, status=500)
async def receive(request: Request) -> HTTPResponse: sender_id = await self._extract_sender(request) text = self._extract_message(request) should_use_stream = rasa.utils.endpoints.bool_arg(request, "stream", default=False) input_channel = self._extract_input_channel(request) metadata = self.get_metadata(request) if should_use_stream: return response.stream( self.stream_response(on_new_message, text, sender_id, input_channel, metadata), content_type="text/event-stream", ) else: collector = CollectingOutputChannel() # noinspection PyBroadException try: await on_new_message( UserMessage( text, collector, sender_id, input_channel=input_channel, metadata=metadata, )) except CancelledError: logger.error("Message handling timed out for " "user message '{}'.".format(text)) except Exception: logger.exception("An exception occured while handling " "user message '{}'.".format(text)) return response.json(collector.messages)
async def list_datasets(request): try: datasets = DatasetManager.list_datasets() return response.json(datasets, status=200) except Exception: logger.exception('faile to list dataset') return response.json({}, status=500)
def check_timeouts(self): """ Runs itself periodically to enforce any expired timeouts. """ try: if not self._task: return duration = current_time() - self._time stage = self._http.stage if stage is Stage.IDLE and duration > self.keep_alive_timeout: logger.debug("KeepAlive Timeout. Closing connection.") elif stage is Stage.REQUEST and duration > self.request_timeout: self._http.exception = RequestTimeout("Request Timeout") elif (stage in (Stage.HANDLER, Stage.RESPONSE, Stage.FAILED) and duration > self.response_timeout): self._http.exception = ServiceUnavailable("Response Timeout") else: interval = (min( self.keep_alive_timeout, self.request_timeout, self.response_timeout, ) / 2) self.loop.call_later(max(0.1, interval), self.check_timeouts) return self._task.cancel() except Exception: logger.exception("protocol.check_timeouts")
async def connection_task(self): """ Run a HTTP connection. Timeouts and some additional error handling occur here, while most of everything else happens in class Http or in code called from there. """ try: self._setup_connection() await self._http.http1() except CancelledError: pass except Exception: logger.exception("protocol.connection_task uncaught") finally: if self.app.debug and self._http: ip = self.transport.get_extra_info("peername") logger.error("Connection lost before response written" f" @ {ip} {self._http.request}") self._http = None self._task = None try: self.close() except BaseException: logger.exception("Closing failed")
def default(self, request, exception): """ Provide a default behavior for the objects of :class:`ErrorHandler`. If a developer chooses to extent the :class:`ErrorHandler` they can provide a custom implementation for this method to behave in a way they see fit. :param request: Incoming request :param exception: Exception object :type request: :class:`sanic.request.Request` :type exception: :class:`sanic.exceptions.SanicException` or :class:`Exception` :return: """ quiet = getattr(exception, "quiet", False) if quiet is False: try: url = repr(request.url) except AttributeError: url = "unknown" self.log(format_exc()) logger.exception("Exception occurred while handling uri: %s", url) return exception_response(request, exception, self.debug)
async def _collect_response(sanic, loop): try: response = await self._local_request( method, url, *request_args, **request_kwargs ) results[-1] = response except Exception as e: logger.exception("Exception") exceptions.append(e) self.app.stop()
def response(self, request, exception): """Fetches and executes an exception handler and returns a response object :param request: Instance of :class:`sanic.request.Request` :param exception: Exception to handle :type request: :class:`sanic.request.Request` :type exception: :class:`sanic.exceptions.SanicException` or :class:`Exception` :return: Wrap the return value obtained from :func:`default` or registered handler for that type of exception. """ handler = self.lookup(exception) response = None try: if handler: response = handler(request, exception) if response is None: response = self.default(request, exception) except Exception: self.log(format_exc()) try: url = repr(request.url) except AttributeError: url = "unknown" response_message = ( "Exception raised in exception handler " '"%s" for uri: %s' ) logger.exception(response_message, handler.__name__, url) if self.debug: return text(response_message % (handler.__name__, url), 500) else: return text("An error occurred while handling an error", 500) return response
def default(self, request, exception): """ Provide a default behavior for the objects of :class:`ErrorHandler`. If a developer chooses to extent the :class:`ErrorHandler` they can provide a custom implementation for this method to behave in a way they see fit. :param request: Incoming request :param exception: Exception object :type request: :class:`sanic.request.Request` :type exception: :class:`sanic.exceptions.SanicException` or :class:`Exception` :return: """ self.log(format_exc()) try: url = repr(request.url) except AttributeError: url = "unknown" response_message = "Exception occurred while handling uri: %s" logger.exception(response_message, url) if issubclass(type(exception), SanicException): return text( "Error: {}".format(exception), status=getattr(exception, "status_code", 500), headers=getattr(exception, "headers", dict()), ) elif self.debug: html_output = self._render_traceback_html(exception, request) return html(html_output, status=500) else: return html(INTERNAL_SERVER_ERROR_HTML, status=500)
def serve(host, port, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, request_timeout=60, response_timeout=60, keep_alive_timeout=5, ssl=None, sock=None, request_max_size=None, reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_async=False, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, state=None, graceful_shutdown_timeout=15.0): """Start asynchronous HTTP Server on an individual process. :param host: Address to host on :param port: Port to host on :param request_handler: Sanic request handler with middleware :param error_handler: Sanic error handler with middleware :param before_start: function to be executed before the server starts listening. Takes arguments `app` instance and `loop` :param after_start: function to be executed after the server starts listening. Takes arguments `app` instance and `loop` :param before_stop: function to be executed when a stop signal is received before it is respected. Takes arguments `app` instance and `loop` :param after_stop: function to be executed when a stop signal is received after it is respected. Takes arguments `app` instance and `loop` :param debug: enables debug output (slows server) :param request_timeout: time in seconds :param response_timeout: time in seconds :param keep_alive_timeout: time in seconds :param ssl: SSLContext :param sock: Socket for the server to accept connections from :param request_max_size: size in bytes, `None` for no limit :param reuse_port: `True` for multiple workers :param loop: asyncio compatible event loop :param protocol: subclass of asyncio protocol class :param request_class: Request class to use :param access_log: disable/enable access log :param is_request_stream: disable/enable Request.stream :param router: Router object :return: Nothing """ if not run_async: loop = async_loop.new_event_loop() asyncio.set_event_loop(loop) if debug: loop.set_debug(debug) connections = connections if connections is not None else set() server = partial( protocol, loop=loop, connections=connections, signal=signal, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, state=state, debug=debug, ) server_coroutine = loop.create_server( server, host, port, ssl=ssl, reuse_port=reuse_port, sock=sock, backlog=backlog ) # Instead of pulling time at the end of every request, # pull it once per minute loop.call_soon(partial(update_current_time, loop)) if run_async: return server_coroutine trigger_events(before_start, loop) try: http_server = loop.run_until_complete(server_coroutine) except BaseException: logger.exception("Unable to start server") return trigger_events(after_start, loop) # Register signals for graceful termination if register_sys_signals: for _signal in (SIGINT, SIGTERM): try: loop.add_signal_handler(_signal, loop.stop) except NotImplementedError: logger.warning('Sanic tried to use loop.add_signal_handler ' 'but it is not implemented on this platform.') pid = os.getpid() try: logger.info('Starting worker [%s]', pid) loop.run_forever() finally: logger.info("Stopping worker [%s]", pid) # Run the on_stop function if provided trigger_events(before_stop, loop) # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) # Complete all tasks on the loop signal.stopped = True for connection in connections: connection.close_if_idle() # Gracefully shutdown timeout. # We should provide graceful_shutdown_timeout, # instead of letting connection hangs forever. # Let's roughly calcucate time. start_shutdown = 0 while connections and (start_shutdown < graceful_shutdown_timeout): loop.run_until_complete(asyncio.sleep(0.1)) start_shutdown = start_shutdown + 0.1 # Force close non-idle connection after waiting for # graceful_shutdown_timeout coros = [] for conn in connections: if hasattr(conn, "websocket") and conn.websocket: coros.append( conn.websocket.close_connection(after_handshake=True) ) else: conn.close() _shutdown = asyncio.gather(*coros, loop=loop) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) loop.close()
def serve( host, port, app, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, request_timeout=60, response_timeout=60, keep_alive_timeout=5, ssl=None, sock=None, request_max_size=None, request_buffer_queue_size=100, reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_multiple=False, run_async=False, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, websocket_read_limit=2 ** 16, websocket_write_limit=2 ** 16, state=None, graceful_shutdown_timeout=15.0, asyncio_server_kwargs=None, ): """Start asynchronous HTTP Server on an individual process. :param host: Address to host on :param port: Port to host on :param request_handler: Sanic request handler with middleware :param error_handler: Sanic error handler with middleware :param before_start: function to be executed before the server starts listening. Takes arguments `app` instance and `loop` :param after_start: function to be executed after the server starts listening. Takes arguments `app` instance and `loop` :param before_stop: function to be executed when a stop signal is received before it is respected. Takes arguments `app` instance and `loop` :param after_stop: function to be executed when a stop signal is received after it is respected. Takes arguments `app` instance and `loop` :param debug: enables debug output (slows server) :param request_timeout: time in seconds :param response_timeout: time in seconds :param keep_alive_timeout: time in seconds :param ssl: SSLContext :param sock: Socket for the server to accept connections from :param request_max_size: size in bytes, `None` for no limit :param reuse_port: `True` for multiple workers :param loop: asyncio compatible event loop :param protocol: subclass of asyncio protocol class :param request_class: Request class to use :param access_log: disable/enable access log :param websocket_max_size: enforces the maximum size for incoming messages in bytes. :param websocket_max_queue: sets the maximum length of the queue that holds incoming messages. :param websocket_read_limit: sets the high-water limit of the buffer for incoming bytes, the low-water limit is half the high-water limit. :param websocket_write_limit: sets the high-water limit of the buffer for outgoing bytes, the low-water limit is a quarter of the high-water limit. :param is_request_stream: disable/enable Request.stream :param request_buffer_queue_size: streaming request buffer queue size :param router: Router object :param graceful_shutdown_timeout: How long take to Force close non-idle connection :param asyncio_server_kwargs: key-value args for asyncio/uvloop create_server method :return: Nothing """ if not run_async: # create new event_loop after fork loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) if debug: loop.set_debug(debug) connections = connections if connections is not None else set() server = partial( protocol, loop=loop, connections=connections, signal=signal, app=app, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, websocket_read_limit=websocket_read_limit, websocket_write_limit=websocket_write_limit, state=state, debug=debug, ) asyncio_server_kwargs = ( asyncio_server_kwargs if asyncio_server_kwargs else {} ) server_coroutine = loop.create_server( server, host, port, ssl=ssl, reuse_port=reuse_port, sock=sock, backlog=backlog, **asyncio_server_kwargs ) if run_async: return server_coroutine trigger_events(before_start, loop) try: http_server = loop.run_until_complete(server_coroutine) except BaseException: logger.exception("Unable to start server") return trigger_events(after_start, loop) # Ignore SIGINT when run_multiple if run_multiple: signal_func(SIGINT, SIG_IGN) # Register signals for graceful termination if register_sys_signals: _singals = (SIGTERM,) if run_multiple else (SIGINT, SIGTERM) for _signal in _singals: try: loop.add_signal_handler(_signal, loop.stop) except NotImplementedError: logger.warning( "Sanic tried to use loop.add_signal_handler " "but it is not implemented on this platform." ) pid = os.getpid() try: logger.info("Starting worker [%s]", pid) loop.run_forever() finally: logger.info("Stopping worker [%s]", pid) # Run the on_stop function if provided trigger_events(before_stop, loop) # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) # Complete all tasks on the loop signal.stopped = True for connection in connections: connection.close_if_idle() # Gracefully shutdown timeout. # We should provide graceful_shutdown_timeout, # instead of letting connection hangs forever. # Let's roughly calcucate time. start_shutdown = 0 while connections and (start_shutdown < graceful_shutdown_timeout): loop.run_until_complete(asyncio.sleep(0.1)) start_shutdown = start_shutdown + 0.1 # Force close non-idle connection after waiting for # graceful_shutdown_timeout coros = [] for conn in connections: if hasattr(conn, "websocket") and conn.websocket: coros.append(conn.websocket.close_connection()) else: conn.close() _shutdown = asyncio.gather(*coros, loop=loop) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) loop.close()
module = import_module(module_name) app = getattr(module, app_name, None) if not isinstance(app, Sanic): raise ValueError( "Module is not a Sanic app, it is a {}. " "Perhaps you meant {}.app?".format( type(app).__name__, args.module ) ) if args.cert is not None or args.key is not None: ssl = {"cert": args.cert, "key": args.key} else: ssl = None app.run( host=args.host, port=args.port, workers=args.workers, debug=args.debug, ssl=ssl, ) except ImportError as e: logger.error( "No module named {} found.\n" " Example File: project/sanic_server.py -> app\n" " Example Module: project.sanic_server.app".format(e.name) ) except ValueError: logger.exception("Failed to run app")