async def handle(self, websocket: websockets.WebSocketCommonProtocol): """ websocket listens to incoming traffic and then pipes in the processed data to a "consumer" co-routine, in this example a call to Redis's PUBLISH function. msg format: { "action_type": <action_type>, "data": <data> } :param websocket: :param path: :return: """ try: while True: msg = await websocket.recv() # TODO Validate message is of the correct format loaded_msg = ujson.loads(msg) action_type = loaded_msg.get("action_type") try: func = getattr(self, f"_{action_type}") if not asyncio.iscoroutinefunction(func): raise AttributeError await func(loaded_msg) except AttributeError: logger.warn( f"Invalid action requested: {action_type} full msg: {msg}" ) except websockets.ConnectionClosed as e: print(f"<ConsumerHandler:handle>[error] {e}")
async def send_email(self, to: str, subject: str, from_: str = None, text: str = None, html: str = None, attachments: Any = None) -> bool: if from_ is None: from_ = self.config["MAIL_SENDER"] if self.config.get("DEBUG_NOTIFICATIONS", False): logger.info(to) logger.info(from_) logger.info(subject) logger.info(text) logger.info(html) logger.info(attachments) else: if text is None and html is None: raise ValidationError( "Neither text nor html has been provided") elif text is not None and html is not None: message = MIMEMultipart("alternative") message.attach(MIMEText(text)) message.attach(MIMEText(html, "html")) elif html is not None and attachments: message = MIMEMultipart() message.attach(MIMEText(html, "html")) elif text is None: message = MIMEText(html, "html") else: message = MIMEText(text) if attachments: for idx, attachment in enumerate(attachments): if isfile(attachment): m_type = guess_type(attachment)[0] main_type, subtype = m_type.split('/', 1) attach = MIMEBase(main_type, subtype) with open(attachment, 'rb') as f: attach.set_payload(f.read()) encode_base64(attach) attach.add_header("Content-Disposition", "attachment", filename=attachment) attach.add_header("X-Attachment-Id", str(idx)) attach.add_header("Content-ID", f"<{idx}>") message.attach(attach) else: logger.warn(f"{attachment} is not a file") message["From"] = from_ message["To"] = to message["Subject"] = subject return await send(message, hostname=self.config["MAIL_SERVER"], port=self.config["MAIL_PORT"], **self.config.get("MAIL_ARGS", {}))
async def get_response(self, bot_id, user_id, question): if bot_id in self.bots: bot = self.bots[bot_id] # type:Bot intent, response = await bot.handle_text(user_id, question) return bot_id, intent, response else: resp = f"{bot_id} not published" _logger.warn(resp) return bot_id, None, resp
def copy_pkey(private_key): pkey = Path(Path.home(), '.ssh/id_rsa') if not pkey.exists(): pkey.parent.mkdir(mode=0o700, parents=True, exist_ok=True) shutil.copy(private_key, pkey) pkey.chmod(0o600) return pkey else: logger.warn('Find existing %s, skipping copying key...', pkey.as_posix()) return private_key
async def handle(self, websocket: websockets.WebSocketCommonProtocol): try: while not websocket.closed: msg = await websocket.recv() loaded_msg = json.loads(msg) print('loaded_msg', loaded_msg, websocket) try: await self._chat(loaded_msg) except AttributeError: logger.warn(f"Invalid action requested full msg: {msg}") except websockets.ConnectionClosed as e: logger.error(f"<ConsumerHandler:handle>[error] {e}")
def database_credential(self) -> Dict[str, str]: try: if not self._database_credential: logger.warn("A new database credential acquired") data = self.client.read( f"database/creds/{SERVICE_NAME}-{RUN_ENV}")["data"] self._database_credential = { "username": data["username"], "password": data["password"], } except Exception as e: raise e return self._database_credential
async def wrapper(*args, **kwargs): try: return await fn(*args, **kwargs) except OperationalError as e: if e.args[0] == 2003: # check connection is expired logger.warn( "MySQL credential expired. trying to get new credentials from vault" ) MySQLConnection._read_pool = None MySQLConnection._write_pool = None settings.vault_client._database_credential = None MySQLConnection._connection_info = settings.database_connection_info return await fn(*args, **kwargs) else: raise e
async def _fetch_base_html(retry_client, query): # TODO needs further investigation as it happens quite often base_html, retries = '', RETRIES while retries > -1: base_html = await _get_html(retry_client, BASE_URL.format(query)) def is_empty(): soup = BeautifulSoup(base_html, 'html.parser') html_text = soup.get_text() return '抱歉,暂时没有找到' in html_text if is_empty(): retries -= 1 logger.warn('Baidu returns no result. Retry.') continue break return base_html
async def sample(request, template_key): if settings.DEBUG: template = models.Template.objects.get_or_create(template_key) else: template = models.Template.objects.get_or_none(template_key) if template and template.valid: url = template.build_sample_url(request.app, "shortcuts.custom", external=False) return response.redirect(url) if settings.DEBUG: message = f"Template not fully implemented: {template}" logger.warn(message) template.datafile.save() abort(501, message) abort(404, f"Template not found: {template_key}")
async def create_tunnel(tunnel: TunnelInfo, private_key, destination: str, port: int = 22): cmd = f'ssh -o StrictHostKeyChecking=no' \ f' -i {private_key}' \ f' -D 0.0.0.0:1080 -C -N {destination} -p {port}' r = None while r != 0: p = await _spawn_process(cmd) logger.info('Tunnel started with pid: %s', p.pid) tunnel.process = p tunnel.status.restarts += 1 r = await p.wait() msg = await _proc_status(p) tunnel.status.last_msg = msg if r != 0: logger.warn('Tunnel daemon stopped: code: %s, msg: %s', r, msg) await asyncio.sleep(4) else: logger.info('Tunnel daemon stopped: %s', msg)
async def healthcheck_tunnel(tunnel: TunnelInfo, period: int, port: int): await asyncio.sleep(4) logger.info('Starting healthcheck') while tunnel.healthcheck: if tunnel.process and tunnel.process.returncode is None: try: async def _work(): proxy = Proxy.from_url('socks5://127.0.0.1:1080') sock = await proxy.connect(dest_host='127.0.0.1', dest_port=port) reader, writer = await asyncio.open_connection(host=None, port=None, sock=sock) writer.write(b'Ping\r\n') await reader.read(-1) await asyncio.wait_for(_work(), timeout=5) logger.debug('Healthcheck: ok') except Exception as e: logger.warn('Healthcheck: error (%s)', e) tunnel.process.send_signal(signal.SIGKILL) await asyncio.sleep(period)
def read_topic_words_from_file(topic_words_file_name='topic_words.lda.txt'): logger.info(f"reading topic_words from file: {topic_words_file_name}") topic_words = defaultdict(list) file_path = os.path.join(model_dir, topic_words_file_name) if not os.path.exists(file_path): logger.warn(f"topic_words file not found: {file_path}") return topic_words with open(file_path, 'r') as f: line = f.readline() while line: pos = line.find('=') line = line[pos + 2:] topic_id, num = line.strip().split('\t') topic_id, num = int(topic_id), int(num) f.readline() items = list() for i in range(num): data = f.readline() word, score = data.strip().split('\t') items.append([word, float(score)]) topic_words[topic_id] = items line = f.readline() return topic_words
def serve(cls, primary: Optional[Sanic] = None) -> None: apps = list(cls._app_registry.values()) if not primary: try: primary = apps[0] except IndexError: raise RuntimeError("Did not find any applications.") reloader_start = primary.listeners.get("reload_process_start") reloader_stop = primary.listeners.get("reload_process_stop") # We want to run auto_reload if ANY of the applications have it enabled if ( cls.should_auto_reload() and os.environ.get("SANIC_SERVER_RUNNING") != "true" ): # no cov loop = new_event_loop() trigger_events(reloader_start, loop, primary) reload_dirs: Set[Path] = primary.state.reload_dirs.union( *(app.state.reload_dirs for app in apps) ) reloader_helpers.watchdog(1.0, reload_dirs) trigger_events(reloader_stop, loop, primary) return # This exists primarily for unit testing if not primary.state.server_info: # no cov for app in apps: app.state.server_info.clear() return primary_server_info = primary.state.server_info[0] primary.before_server_start(partial(primary._start_servers, apps=apps)) try: primary_server_info.stage = ServerStage.SERVING if primary.state.workers > 1 and os.name != "posix": # no cov logger.warn( f"Multiprocessing is currently not supported on {os.name}," " using workers=1 instead" ) primary.state.workers = 1 if primary.state.workers == 1: serve_single(primary_server_info.settings) elif primary.state.workers == 0: raise RuntimeError("Cannot serve with no workers") else: serve_multiple( primary_server_info.settings, primary.state.workers ) except BaseException: error_logger.exception( "Experienced exception while trying to serve" ) raise finally: primary_server_info.stage = ServerStage.STOPPED logger.info("Server Stopped") for app in apps: app.state.server_info.clear() app.router.reset() app.signal_router.reset()
async def check_avatar_item_directory(app, loop): if not os.path.exists(avatar_item_directory): logger.warn(( f'Avatar directory \'{avatar_item_directory}\' is missing! ' 'Either download from https://icer.ink/media1.clubpenguin.com/avatar/paper/ ' 'or let wand mount the directory for you!'))
from os import environ, path from sanic.log import logger ENV_FILE = ".env" SECRET_KEY = environ.get("SECRET_KEY") if not SECRET_KEY: from secrets import choice import string alphabet = string.ascii_letters + string.digits + string.punctuation SECRET_KEY = "".join(choice(alphabet) for i in range(64)) if path.exists(ENV_FILE): logger.warn( 'Unable to persist SECRET_KEY, `.env` file already exists. Are you running `pipenv run`?' f' Temporal value: "{SECRET_KEY}".', ) else: with open(ENV_FILE, "w") as f: f.write(f'SECRET_KEY="{SECRET_KEY}"\n') DATABASE_URL = environ.get("DATABASE_URL", "sqlite:///secrets.db") STATIC_PATH = environ.get("STATIC_PATH", path.join(path.dirname(__file__), "static")) MAX_DATA_LENGTH = int(environ.get("MAX_DATA_LENGTH", 2**20)) MAX_EXPIRATION = int(environ.get("MAX_EXPIRATION", 7 * 24)) # Hours MAX_READS = int(environ.get("MAX_READS", 7)) SANIC_CONFIG = { "debug": environ.get("DEBUG"),
async def test(request): request_data = request.json['data'] logger.warn("logger %r", request_data) resp = iesha_chatbot.respond(request_data) return json({"hello": resp})
async def handle_404(request, exception): logger.warn(f"URL {request.path} not found, sending current UTC Time...") current_time = int( round(datetime.now(tz=pytz.timezone("UTC")).timestamp()) ) return json({"time": current_time}, status=404)
def run( self, host: Optional[str] = None, port: Optional[int] = None, *, debug: bool = False, auto_reload: Optional[bool] = None, ssl: Union[dict, SSLContext, None] = None, sock: Optional[socket] = None, workers: int = 1, protocol: Optional[Type[Protocol]] = None, backlog: int = 100, register_sys_signals: bool = True, access_log: Optional[bool] = None, unix: Optional[str] = None, loop: None = None, ) -> None: """Run the HTTP Server and listen until keyboard interrupt or term signal. On termination, drain connections before closing. :param host: Address to host on :type host: str :param port: Port to host on :type port: int :param debug: Enables debug output (slows server) :type debug: bool :param auto_reload: Reload app whenever its source code is changed. Enabled by default in debug mode. :type auto_relaod: bool :param ssl: SSLContext, or location of certificate and key for SSL encryption of worker(s) :type ssl: SSLContext or dict :param sock: Socket for the server to accept connections from :type sock: socket :param workers: Number of processes received before it is respected :type workers: int :param protocol: Subclass of asyncio Protocol class :type protocol: type[Protocol] :param backlog: a number of unaccepted connections that the system will allow before refusing new connections :type backlog: int :param register_sys_signals: Register SIG* events :type register_sys_signals: bool :param access_log: Enables writing access logs (slows server) :type access_log: bool :param unix: Unix socket to listen on instead of TCP port :type unix: str :return: Nothing """ if loop is not None: raise TypeError( "loop is not a valid argument. To use an existing loop, " "change to create_server().\nSee more: " "https://sanic.readthedocs.io/en/latest/sanic/deploying.html" "#asynchronous-support") if auto_reload or auto_reload is None and debug: if os.environ.get("SANIC_SERVER_RUNNING") != "true": return reloader_helpers.watchdog(1.0) if sock is None: host, port = host or "127.0.0.1", port or 8000 if protocol is None: protocol = (WebSocketProtocol if self.websocket_enabled else HttpProtocol) # if access_log is passed explicitly change config.ACCESS_LOG if access_log is not None: self.config.ACCESS_LOG = access_log server_settings = self._helper( host=host, port=port, debug=debug, ssl=ssl, sock=sock, unix=unix, workers=workers, protocol=protocol, backlog=backlog, register_sys_signals=register_sys_signals, auto_reload=auto_reload, ) try: self.is_running = True self.is_stopping = False if workers > 1 and os.name != "posix": logger.warn( f"Multiprocessing is currently not supported on {os.name}," " using workers=1 instead") workers = 1 if workers == 1: serve(**server_settings) else: serve_multiple(server_settings, workers) except BaseException: error_logger.exception( "Experienced exception while trying to serve") raise finally: self.is_running = False logger.info("Server Stopped")
def serve(host, port, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, request_timeout=60, response_timeout=60, keep_alive_timeout=60, ssl=None, sock=None, request_max_size=None, reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_async=False, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, state=None, graceful_shutdown_timeout=15.0): """Start asynchronous HTTP Server on an individual process. :param host: Address to host on :param port: Port to host on :param request_handler: Sanic request handler with middleware :param error_handler: Sanic error handler with middleware :param before_start: function to be executed before the server starts listening. Takes arguments `app` instance and `loop` :param after_start: function to be executed after the server starts listening. Takes arguments `app` instance and `loop` :param before_stop: function to be executed when a stop signal is received before it is respected. Takes arguments `app` instance and `loop` :param after_stop: function to be executed when a stop signal is received after it is respected. Takes arguments `app` instance and `loop` :param debug: enables debug output (slows server) :param request_timeout: time in seconds :param ssl: SSLContext :param sock: Socket for the server to accept connections from :param request_max_size: size in bytes, `None` for no limit :param reuse_port: `True` for multiple workers :param loop: asyncio compatible event loop :param protocol: subclass of asyncio protocol class :param request_class: Request class to use :param access_log: disable/enable access log :param is_request_stream: disable/enable Request.stream :param router: Router object :return: Nothing """ if not run_async: loop = async_loop.new_event_loop() asyncio.set_event_loop(loop) if debug: loop.set_debug(debug) connections = connections if connections is not None else set() server = partial( protocol, loop=loop, connections=connections, signal=signal, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, state=state, debug=debug, ) server_coroutine = loop.create_server(server, host, port, ssl=ssl, reuse_port=reuse_port, sock=sock, backlog=backlog) # Instead of pulling time at the end of every request, # pull it once per minute loop.call_soon(partial(update_current_time, loop)) if run_async: return server_coroutine trigger_events(before_start, loop) try: http_server = loop.run_until_complete(server_coroutine) except: logger.exception("Unable to start server") return trigger_events(after_start, loop) # Register signals for graceful termination if register_sys_signals: for _signal in (SIGINT, SIGTERM): try: loop.add_signal_handler(_signal, loop.stop) except NotImplementedError: logger.warn( 'Sanic tried to use loop.add_signal_handler but it is' ' not implemented on this platform.') pid = os.getpid() try: logger.info('Starting worker [{}]'.format(pid)) loop.run_forever() finally: logger.info("Stopping worker [{}]".format(pid)) # Run the on_stop function if provided trigger_events(before_stop, loop) # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) # Complete all tasks on the loop signal.stopped = True for connection in connections: connection.close_if_idle() # Gracefully shutdown timeout. # We should provide graceful_shutdown_timeout, # instead of letting connection hangs forever. # Let's roughly calcucate time. start_shutdown = 0 while connections and (start_shutdown < graceful_shutdown_timeout): loop.run_until_complete(asyncio.sleep(0.1)) start_shutdown = start_shutdown + 0.1 # Force close non-idle connection after waiting for # graceful_shutdown_timeout coros = [] for conn in connections: if hasattr(conn, "websocket") and conn.websocket: coros.append(conn.websocket.close_connection(force=True)) else: conn.close() _shutdown = asyncio.gather(*coros, loop=loop) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) loop.close()