async def handle(self, websocket: websockets.WebSocketCommonProtocol):
"""
websocket listens to incoming traffic and then pipes in the processed data to a "consumer" co-routine,
in this example a call to Redis's PUBLISH function.
msg format: {
"action_type": <action_type>,
"data": <data>
}
:param websocket:
:param path:
:return:
"""
try:
while True:
msg = await websocket.recv()
# TODO Validate message is of the correct format
loaded_msg = ujson.loads(msg)
action_type = loaded_msg.get("action_type")
try:
func = getattr(self, f"_{action_type}")
if not asyncio.iscoroutinefunction(func):
raise AttributeError
await func(loaded_msg)
except AttributeError:
logger.warn(
f"Invalid action requested: {action_type} full msg: {msg}"
)
except websockets.ConnectionClosed as e:
print(f"<ConsumerHandler:handle>[error] {e}")
async def send_email(self,
to: str,
subject: str,
from_: str = None,
text: str = None,
html: str = None,
attachments: Any = None) -> bool:
if from_ is None:
from_ = self.config["MAIL_SENDER"]
if self.config.get("DEBUG_NOTIFICATIONS", False):
logger.info(to)
logger.info(from_)
logger.info(subject)
logger.info(text)
logger.info(html)
logger.info(attachments)
else:
if text is None and html is None:
raise ValidationError(
"Neither text nor html has been provided")
elif text is not None and html is not None:
message = MIMEMultipart("alternative")
message.attach(MIMEText(text))
message.attach(MIMEText(html, "html"))
elif html is not None and attachments:
message = MIMEMultipart()
message.attach(MIMEText(html, "html"))
elif text is None:
message = MIMEText(html, "html")
else:
message = MIMEText(text)
if attachments:
for idx, attachment in enumerate(attachments):
if isfile(attachment):
m_type = guess_type(attachment)[0]
main_type, subtype = m_type.split('/', 1)
attach = MIMEBase(main_type, subtype)
with open(attachment, 'rb') as f:
attach.set_payload(f.read())
encode_base64(attach)
attach.add_header("Content-Disposition",
"attachment",
filename=attachment)
attach.add_header("X-Attachment-Id", str(idx))
attach.add_header("Content-ID", f"<{idx}>")
message.attach(attach)
else:
logger.warn(f"{attachment} is not a file")
message["From"] = from_
message["To"] = to
message["Subject"] = subject
return await send(message,
hostname=self.config["MAIL_SERVER"],
port=self.config["MAIL_PORT"],
**self.config.get("MAIL_ARGS", {}))
async def get_response(self, bot_id, user_id, question):
if bot_id in self.bots:
bot = self.bots[bot_id] # type:Bot
intent, response = await bot.handle_text(user_id, question)
return bot_id, intent, response
else:
resp = f"{bot_id} not published"
_logger.warn(resp)
return bot_id, None, resp
def copy_pkey(private_key):
pkey = Path(Path.home(), '.ssh/id_rsa')
if not pkey.exists():
pkey.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
shutil.copy(private_key, pkey)
pkey.chmod(0o600)
return pkey
else:
logger.warn('Find existing %s, skipping copying key...',
pkey.as_posix())
return private_key
async def handle(self, websocket: websockets.WebSocketCommonProtocol):
try:
while not websocket.closed:
msg = await websocket.recv()
loaded_msg = json.loads(msg)
print('loaded_msg', loaded_msg, websocket)
try:
await self._chat(loaded_msg)
except AttributeError:
logger.warn(f"Invalid action requested full msg: {msg}")
except websockets.ConnectionClosed as e:
logger.error(f"<ConsumerHandler:handle>[error] {e}")
def database_credential(self) -> Dict[str, str]:
try:
if not self._database_credential:
logger.warn("A new database credential acquired")
data = self.client.read(
f"database/creds/{SERVICE_NAME}-{RUN_ENV}")["data"]
self._database_credential = {
"username": data["username"],
"password": data["password"],
}
except Exception as e:
raise e
return self._database_credential
async def wrapper(*args, **kwargs):
try:
return await fn(*args, **kwargs)
except OperationalError as e:
if e.args[0] == 2003: # check connection is expired
logger.warn(
"MySQL credential expired. trying to get new credentials from vault"
)
MySQLConnection._read_pool = None
MySQLConnection._write_pool = None
settings.vault_client._database_credential = None
MySQLConnection._connection_info = settings.database_connection_info
return await fn(*args, **kwargs)
else:
raise e
async def _fetch_base_html(retry_client, query):
# TODO needs further investigation as it happens quite often
base_html, retries = '', RETRIES
while retries > -1:
base_html = await _get_html(retry_client, BASE_URL.format(query))
def is_empty():
soup = BeautifulSoup(base_html, 'html.parser')
html_text = soup.get_text()
return '抱歉，暂时没有找到' in html_text
if is_empty():
retries -= 1
logger.warn('Baidu returns no result. Retry.')
continue
break
return base_html
async def sample(request, template_key):
if settings.DEBUG:
template = models.Template.objects.get_or_create(template_key)
else:
template = models.Template.objects.get_or_none(template_key)
if template and template.valid:
url = template.build_sample_url(request.app,
"shortcuts.custom",
external=False)
return response.redirect(url)
if settings.DEBUG:
message = f"Template not fully implemented: {template}"
logger.warn(message)
template.datafile.save()
abort(501, message)
abort(404, f"Template not found: {template_key}")
async def create_tunnel(tunnel: TunnelInfo,
private_key,
destination: str,
port: int = 22):
cmd = f'ssh -o StrictHostKeyChecking=no' \
f' -i {private_key}' \
f' -D 0.0.0.0:1080 -C -N {destination} -p {port}'
r = None
while r != 0:
p = await _spawn_process(cmd)
logger.info('Tunnel started with pid: %s', p.pid)
tunnel.process = p
tunnel.status.restarts += 1
r = await p.wait()
msg = await _proc_status(p)
tunnel.status.last_msg = msg
if r != 0:
logger.warn('Tunnel daemon stopped: code: %s, msg: %s', r, msg)
await asyncio.sleep(4)
else:
logger.info('Tunnel daemon stopped: %s', msg)
async def healthcheck_tunnel(tunnel: TunnelInfo, period: int, port: int):
await asyncio.sleep(4)
logger.info('Starting healthcheck')
while tunnel.healthcheck:
if tunnel.process and tunnel.process.returncode is None:
try:
async def _work():
proxy = Proxy.from_url('socks5://127.0.0.1:1080')
sock = await proxy.connect(dest_host='127.0.0.1',
dest_port=port)
reader, writer = await asyncio.open_connection(host=None,
port=None,
sock=sock)
writer.write(b'Ping\r\n')
await reader.read(-1)
await asyncio.wait_for(_work(), timeout=5)
logger.debug('Healthcheck: ok')
except Exception as e:
logger.warn('Healthcheck: error (%s)', e)
tunnel.process.send_signal(signal.SIGKILL)
await asyncio.sleep(period)
def read_topic_words_from_file(topic_words_file_name='topic_words.lda.txt'):
logger.info(f"reading topic_words from file: {topic_words_file_name}")
topic_words = defaultdict(list)
file_path = os.path.join(model_dir, topic_words_file_name)
if not os.path.exists(file_path):
logger.warn(f"topic_words file not found: {file_path}")
return topic_words
with open(file_path, 'r') as f:
line = f.readline()
while line:
pos = line.find('=')
line = line[pos + 2:]
topic_id, num = line.strip().split('\t')
topic_id, num = int(topic_id), int(num)
f.readline()
items = list()
for i in range(num):
data = f.readline()
word, score = data.strip().split('\t')
items.append([word, float(score)])
topic_words[topic_id] = items
line = f.readline()
return topic_words
def serve(cls, primary: Optional[Sanic] = None) -> None:
apps = list(cls._app_registry.values())
if not primary:
try:
primary = apps[0]
except IndexError:
raise RuntimeError("Did not find any applications.")
reloader_start = primary.listeners.get("reload_process_start")
reloader_stop = primary.listeners.get("reload_process_stop")
# We want to run auto_reload if ANY of the applications have it enabled
if (
cls.should_auto_reload()
and os.environ.get("SANIC_SERVER_RUNNING") != "true"
): # no cov
loop = new_event_loop()
trigger_events(reloader_start, loop, primary)
reload_dirs: Set[Path] = primary.state.reload_dirs.union(
*(app.state.reload_dirs for app in apps)
)
reloader_helpers.watchdog(1.0, reload_dirs)
trigger_events(reloader_stop, loop, primary)
return
# This exists primarily for unit testing
if not primary.state.server_info: # no cov
for app in apps:
app.state.server_info.clear()
return
primary_server_info = primary.state.server_info[0]
primary.before_server_start(partial(primary._start_servers, apps=apps))
try:
primary_server_info.stage = ServerStage.SERVING
if primary.state.workers > 1 and os.name != "posix": # no cov
logger.warn(
f"Multiprocessing is currently not supported on {os.name},"
" using workers=1 instead"
)
primary.state.workers = 1
if primary.state.workers == 1:
serve_single(primary_server_info.settings)
elif primary.state.workers == 0:
raise RuntimeError("Cannot serve with no workers")
else:
serve_multiple(
primary_server_info.settings, primary.state.workers
)
except BaseException:
error_logger.exception(
"Experienced exception while trying to serve"
)
raise
finally:
primary_server_info.stage = ServerStage.STOPPED
logger.info("Server Stopped")
for app in apps:
app.state.server_info.clear()
app.router.reset()
app.signal_router.reset()
async def check_avatar_item_directory(app, loop):
if not os.path.exists(avatar_item_directory):
logger.warn((
f'Avatar directory \'{avatar_item_directory}\' is missing! '
'Either download from https://icer.ink/media1.clubpenguin.com/avatar/paper/ '
'or let wand mount the directory for you!'))
from os import environ, path
from sanic.log import logger
ENV_FILE = ".env"
SECRET_KEY = environ.get("SECRET_KEY")
if not SECRET_KEY:
from secrets import choice
import string
alphabet = string.ascii_letters + string.digits + string.punctuation
SECRET_KEY = "".join(choice(alphabet) for i in range(64))
if path.exists(ENV_FILE):
logger.warn(
'Unable to persist SECRET_KEY, `.env` file already exists. Are you running `pipenv run`?'
f' Temporal value: "{SECRET_KEY}".', )
else:
with open(ENV_FILE, "w") as f:
f.write(f'SECRET_KEY="{SECRET_KEY}"\n')
DATABASE_URL = environ.get("DATABASE_URL", "sqlite:///secrets.db")
STATIC_PATH = environ.get("STATIC_PATH",
path.join(path.dirname(__file__), "static"))
MAX_DATA_LENGTH = int(environ.get("MAX_DATA_LENGTH", 2**20))
MAX_EXPIRATION = int(environ.get("MAX_EXPIRATION", 7 * 24)) # Hours
MAX_READS = int(environ.get("MAX_READS", 7))
SANIC_CONFIG = {
"debug": environ.get("DEBUG"),
async def test(request):
request_data = request.json['data']
logger.warn("logger %r", request_data)
resp = iesha_chatbot.respond(request_data)
return json({"hello": resp})
async def handle_404(request, exception):
logger.warn(f"URL {request.path} not found, sending current UTC Time...")
current_time = int(
round(datetime.now(tz=pytz.timezone("UTC")).timestamp())
)
return json({"time": current_time}, status=404)
def run(
self,
host: Optional[str] = None,
port: Optional[int] = None,
*,
debug: bool = False,
auto_reload: Optional[bool] = None,
ssl: Union[dict, SSLContext, None] = None,
sock: Optional[socket] = None,
workers: int = 1,
protocol: Optional[Type[Protocol]] = None,
backlog: int = 100,
register_sys_signals: bool = True,
access_log: Optional[bool] = None,
unix: Optional[str] = None,
loop: None = None,
) -> None:
"""Run the HTTP Server and listen until keyboard interrupt or term
signal. On termination, drain connections before closing.
:param host: Address to host on
:type host: str
:param port: Port to host on
:type port: int
:param debug: Enables debug output (slows server)
:type debug: bool
:param auto_reload: Reload app whenever its source code is changed.
Enabled by default in debug mode.
:type auto_relaod: bool
:param ssl: SSLContext, or location of certificate and key
for SSL encryption of worker(s)
:type ssl: SSLContext or dict
:param sock: Socket for the server to accept connections from
:type sock: socket
:param workers: Number of processes received before it is respected
:type workers: int
:param protocol: Subclass of asyncio Protocol class
:type protocol: type[Protocol]
:param backlog: a number of unaccepted connections that the system
will allow before refusing new connections
:type backlog: int
:param register_sys_signals: Register SIG* events
:type register_sys_signals: bool
:param access_log: Enables writing access logs (slows server)
:type access_log: bool
:param unix: Unix socket to listen on instead of TCP port
:type unix: str
:return: Nothing
"""
if loop is not None:
raise TypeError(
"loop is not a valid argument. To use an existing loop, "
"change to create_server().\nSee more: "
"https://sanic.readthedocs.io/en/latest/sanic/deploying.html"
"#asynchronous-support")
if auto_reload or auto_reload is None and debug:
if os.environ.get("SANIC_SERVER_RUNNING") != "true":
return reloader_helpers.watchdog(1.0)
if sock is None:
host, port = host or "127.0.0.1", port or 8000
if protocol is None:
protocol = (WebSocketProtocol
if self.websocket_enabled else HttpProtocol)
# if access_log is passed explicitly change config.ACCESS_LOG
if access_log is not None:
self.config.ACCESS_LOG = access_log
server_settings = self._helper(
host=host,
port=port,
debug=debug,
ssl=ssl,
sock=sock,
unix=unix,
workers=workers,
protocol=protocol,
backlog=backlog,
register_sys_signals=register_sys_signals,
auto_reload=auto_reload,
)
try:
self.is_running = True
self.is_stopping = False
if workers > 1 and os.name != "posix":
logger.warn(
f"Multiprocessing is currently not supported on {os.name},"
" using workers=1 instead")
workers = 1
if workers == 1:
serve(**server_settings)
else:
serve_multiple(server_settings, workers)
except BaseException:
error_logger.exception(
"Experienced exception while trying to serve")
raise
finally:
self.is_running = False
logger.info("Server Stopped")
def serve(host,
port,
request_handler,
error_handler,
before_start=None,
after_start=None,
before_stop=None,
after_stop=None,
debug=False,
request_timeout=60,
response_timeout=60,
keep_alive_timeout=60,
ssl=None,
sock=None,
request_max_size=None,
reuse_port=False,
loop=None,
protocol=HttpProtocol,
backlog=100,
register_sys_signals=True,
run_async=False,
connections=None,
signal=Signal(),
request_class=None,
access_log=True,
keep_alive=True,
is_request_stream=False,
router=None,
websocket_max_size=None,
websocket_max_queue=None,
state=None,
graceful_shutdown_timeout=15.0):
"""Start asynchronous HTTP Server on an individual process.
:param host: Address to host on
:param port: Port to host on
:param request_handler: Sanic request handler with middleware
:param error_handler: Sanic error handler with middleware
:param before_start: function to be executed before the server starts
listening. Takes arguments `app` instance and `loop`
:param after_start: function to be executed after the server starts
listening. Takes arguments `app` instance and `loop`
:param before_stop: function to be executed when a stop signal is
received before it is respected. Takes arguments
`app` instance and `loop`
:param after_stop: function to be executed when a stop signal is
received after it is respected. Takes arguments
`app` instance and `loop`
:param debug: enables debug output (slows server)
:param request_timeout: time in seconds
:param ssl: SSLContext
:param sock: Socket for the server to accept connections from
:param request_max_size: size in bytes, `None` for no limit
:param reuse_port: `True` for multiple workers
:param loop: asyncio compatible event loop
:param protocol: subclass of asyncio protocol class
:param request_class: Request class to use
:param access_log: disable/enable access log
:param is_request_stream: disable/enable Request.stream
:param router: Router object
:return: Nothing
"""
if not run_async:
loop = async_loop.new_event_loop()
asyncio.set_event_loop(loop)
if debug:
loop.set_debug(debug)
connections = connections if connections is not None else set()
server = partial(
protocol,
loop=loop,
connections=connections,
signal=signal,
request_handler=request_handler,
error_handler=error_handler,
request_timeout=request_timeout,
response_timeout=response_timeout,
keep_alive_timeout=keep_alive_timeout,
request_max_size=request_max_size,
request_class=request_class,
access_log=access_log,
keep_alive=keep_alive,
is_request_stream=is_request_stream,
router=router,
websocket_max_size=websocket_max_size,
websocket_max_queue=websocket_max_queue,
state=state,
debug=debug,
)
server_coroutine = loop.create_server(server,
host,
port,
ssl=ssl,
reuse_port=reuse_port,
sock=sock,
backlog=backlog)
# Instead of pulling time at the end of every request,
# pull it once per minute
loop.call_soon(partial(update_current_time, loop))
if run_async:
return server_coroutine
trigger_events(before_start, loop)
try:
http_server = loop.run_until_complete(server_coroutine)
except:
logger.exception("Unable to start server")
return
trigger_events(after_start, loop)
# Register signals for graceful termination
if register_sys_signals:
for _signal in (SIGINT, SIGTERM):
try:
loop.add_signal_handler(_signal, loop.stop)
except NotImplementedError:
logger.warn(
'Sanic tried to use loop.add_signal_handler but it is'
' not implemented on this platform.')
pid = os.getpid()
try:
logger.info('Starting worker [{}]'.format(pid))
loop.run_forever()
finally:
logger.info("Stopping worker [{}]".format(pid))
# Run the on_stop function if provided
trigger_events(before_stop, loop)
# Wait for event loop to finish and all connections to drain
http_server.close()
loop.run_until_complete(http_server.wait_closed())
# Complete all tasks on the loop
signal.stopped = True
for connection in connections:
connection.close_if_idle()
# Gracefully shutdown timeout.
# We should provide graceful_shutdown_timeout,
# instead of letting connection hangs forever.
# Let's roughly calcucate time.
start_shutdown = 0
while connections and (start_shutdown < graceful_shutdown_timeout):
loop.run_until_complete(asyncio.sleep(0.1))
start_shutdown = start_shutdown + 0.1
# Force close non-idle connection after waiting for
# graceful_shutdown_timeout
coros = []
for conn in connections:
if hasattr(conn, "websocket") and conn.websocket:
coros.append(conn.websocket.close_connection(force=True))
else:
conn.close()
_shutdown = asyncio.gather(*coros, loop=loop)
loop.run_until_complete(_shutdown)
trigger_events(after_stop, loop)
loop.close()
