import coloredlogs
import logging
import os
import shutil
import signal
import socket
import sys
import urllib.parse
from aiohttp import web
from coloredlogs import syslog
from vj4 import app
from vj4.util import options
options.define('listen',
default='http://127.0.0.1:8888',
help='Server listening address.')
options.define('prefork', default=1, help='Number of prefork workers.')
options.define('syslog',
default=False,
help='Use syslog instead of stderr for logging.')
options.define('listen_owner',
default='',
help='Owner of the unix socket which is server listening to.')
options.define('listen_group',
default='',
help='Group of the unix socket which is server listening to.')
options.define(
'listen_mode',
default='',
help='File mode of the unix socket which is server listening to.')
import functools
import logging
from os import path
import sockjs
from aiohttp import web
from vj4 import error
from vj4.service import bus
from vj4.service import smallcache
from vj4.util import json
from vj4.util import locale
from vj4.util import options
from vj4.util import tools
options.define('debug', default=False, help='Enable debug mode.')
options.define('static', default=True, help='Serve static files.')
options.define('ip_header',
default='X-Forwarded-For',
help='Header name for remote IP.')
options.define('unsaved_session_expire_seconds',
default=43200,
help='Expire time for unsaved session, in seconds.')
options.define('saved_session_expire_seconds',
default=2592000,
help='Expire time for saved session, in seconds.')
options.define('cookie_domain', default=None, help='Cookie domain.')
options.define('cookie_secure',
default=False,
help='Enable secure cookie flag.')
options.define('registration_token_expire_seconds',
import bson
from vj4 import mq
from vj4.util import options
options.define('queue_prefetch', default=1, help='Queue prefetch count.')
async def publish(key, **kwargs):
channel = await mq.channel('queue')
await channel.queue_declare(key)
await channel.basic_publish(bson.BSON.encode(kwargs), '', key)
async def consume(key, on_message):
channel = await mq.channel()
await channel.queue_declare(key)
await channel.basic_qos(prefetch_count=options.queue_prefetch)
await channel.basic_consume(
(lambda channel, body, envelope, properties: on_message(
envelope.delivery_tag, **bson.BSON.decode(body))), key)
return channel
from motor import motor_asyncio
from vj4.util import options
options.define('db_host', default='localhost', help='Database hostname or IP address.')
options.define('db_name', default='test', help='Database name.')
class Database(object):
_instance = None
def __new__(cls):
if not cls._instance:
client = motor_asyncio.AsyncIOMotorClient(options.options.db_host)
cls._instance = motor_asyncio.AsyncIOMotorDatabase(client, options.options.db_name)
return cls._instance
class Collection(object):
_instances = {}
def __new__(cls, name):
if name not in cls._instances:
cls._instances[name] = motor_asyncio.AsyncIOMotorCollection(Database(), name)
return cls._instances[name]
class GridFS(object):
_instances = {}
def __new__(cls, name):
import atexit
import logging
import logging.config
import os
import signal
import socket
import sys
import urllib.parse
from aiohttp import web
from vj4 import app
from vj4.util import options
options.define('listen',
default='http://127.0.0.1:8888',
help='Server listening address.')
options.define('prefork', default=1, help='Number of prefork workers.')
options.define(
'log_format',
default=('%(log_color)s[%(levelname).1s '
'%(asctime)s %(module)s:%(lineno)d]%(reset)s %(message)s'),
help='Log format.')
_logger = logging.getLogger(__name__)
def main():
logging.config.dictConfig({
'version': 1,
'handlers': {
'console': {
"""Experimental helper module to wrap methods for command line invoke.
Usage example:
python3.5 -m vj4.model.user --help
python3.5 -m vj4.model.user -- --help
python3.5 -m vj4.model.user get -1
python3.5 -m vj4.model.user --db-name=prod get -1
"""
import collections
from vj4 import db
from vj4.util import options
options.define('pretty', default=False, help='Pretty print the result.')
_methods = collections.OrderedDict()
def wrap(method):
if method.__module__ == '__main__':
_methods[method.__name__] = method
return method
def invoke_by_args():
import argparse
import asyncio
import coloredlogs
import inspect
import pprint
"""Experimental helper module to wrap methods for command line invoke.
Usage example:
python3.5 -m vj4.model.user --help
python3.5 -m vj4.model.user -- --help
python3.5 -m vj4.model.user get -1
python3.5 -m vj4.model.user --db-name=prod get -1
"""
import collections
from vj4 import db
from vj4.util import options
options.define('pretty', default=False, help='Pretty print the result.')
_methods = collections.OrderedDict()
def wrap(method):
if method.__module__ == '__main__':
_methods[method.__name__] = method
return method
def invoke_by_args():
import argparse
import asyncio
import coloredlogs
import inspect
import pprint
import aiomongo
import functools
from vj4.util import options
options.define('db_host',
default='172.111.0.3',
help='Database hostname or IP address.')
options.define('db_name', default='mongo', help='Database name.')
async def init():
global _client, _db
_client = await aiomongo.create_client('mongodb://' + options.db_host)
_db = _client.get_database(options.db_name)
@functools.lru_cache()
def coll(name):
return aiomongo.Collection(_db, name)
@functools.lru_cache()
def fs(name):
return aiomongo.GridFS(_db, name)
import asyncio
import aioamqp
from vj4.util import options
options.define('mq_host', default='localhost', help='Message queue hostname or IP address.')
options.define('mq_vhost', default='/', help='Message queue virtual host.')
_protocol_future = None
_channel_futures = {}
async def _connect():
global _protocol_future
if _protocol_future:
return await _protocol_future
_protocol_future = future = asyncio.Future()
try:
_, protocol = await aioamqp.connect(host=options.options.mq_host,
virtualhost=options.options.mq_vhost)
future.set_result(protocol)
asyncio.get_event_loop().create_task(_wait_protocol(protocol))
return protocol
except Exception as e:
future.set_exception(e)
_protocol_future = None
raise
async def _wait_protocol(protocol):
import asyncio
import atexit
import logging
import os
import signal
import socket
import sys
import urllib.parse
from vj4 import app
from vj4.util import options
options.define('listen', default='http://127.0.0.1:8888', help='Server listening address.')
options.define('prefork', default=1, help='Number of prefork workers.')
_logger = logging.getLogger(__name__)
def main():
options.parse_command_line()
_logger.info('Server listening on %s', options.options.listen)
url = urllib.parse.urlparse(options.options.listen)
if url.scheme == 'http':
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
host, port_str = url.netloc.rsplit(':', 1)
sock.bind((host, int(port_str)))
elif url.scheme == 'unix':
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
os.remove(url.path)
"""A locale module which mimics tornado's interface."""
import csv
import functools
import os
import os.path
from vj4.util import options
options.define('default_locale', default='zh_CN', help='Default locale.')
_locales = {}
def load_translations(translation_path):
for path in os.listdir(translation_path):
if not path.endswith(".csv"):
continue
with open(os.path.join(translation_path, path)) as csv_file:
_locales[path[:-4]] = dict(csv.reader(csv_file))
@functools.lru_cache()
def get_translate(locale_code):
if locale_code not in _locales:
locale_code = options.options.default_locale
locale = _locales[locale_code]
return lambda text: locale[text] if text in locale else text
import collections
import copy
from vj4.service import bus
from vj4.util import options
PREFIX_DISCUSSION_NODES = 'discussion-nodes-'
options.define('smallcache_max_entries', default=64,
help='Maximum number of entries in smallcache.')
_cache = collections.OrderedDict()
async def _on_unset(e):
if e['value'] in _cache:
del _cache[e['value']]
def init():
bus.subscribe(_on_unset, ['smallcache-unset'])
def get_direct(key, default=None):
if key not in _cache:
return default
_cache.move_to_end(key)
return _cache[key]
def get(key, default=None):
import collections
import copy
from vj4.service import bus
from vj4.util import options
PREFIX_DISCUSSION_NODES = 'discussion-nodes-'
options.define('smallcache_max_entries',
default=64,
help='Maximum number of entries in smallcache.')
_cache = collections.OrderedDict()
async def _on_unset(e):
if e['value'] in _cache:
del _cache[e['value']]
def init():
bus.subscribe(_on_unset, ['smallcache-unset'])
def get_direct(key, default=None):
if key not in _cache:
return default
_cache.move_to_end(key)
return _cache[key]
import aiohttp_sentry
import sockjs
from aiohttp import web
from vj4 import db
from vj4 import error
from vj4.model import system
from vj4.service import bus
from vj4.service import smallcache
from vj4.service import staticmanifest
from vj4.util import json
from vj4.util import options
from vj4.util import tools
options.define('debug', default=False, help='Enable debug mode.')
options.define('static', default=True, help='Serve static files.')
options.define('ip_header', default='', help='Header name for remote IP.')
options.define('unsaved_session_expire_seconds', default=43200,
help='Expire time for unsaved session, in seconds.')
options.define('saved_session_expire_seconds', default=2592000,
help='Expire time for saved session, in seconds.')
options.define('cookie_domain', default='', help='Cookie domain.')
options.define('cookie_secure', default=False, help='Enable secure cookie flag.')
options.define('registration_token_expire_seconds', default=86400,
help='Expire time for registration token, in seconds.')
options.define('lostpass_token_expire_seconds', default=3600,
help='Expire time for lostpass token, in seconds.')
options.define('changemail_token_expire_seconds', default=3600,
help='Expire time for changemail token, in seconds.')
options.define('url_prefix', default='https://vijos.org', help='URL prefix.')
import logging
from vj4 import db
from vj4.model import blacklist
from vj4.model import document
from vj4.model import user
from vj4.model.adaptor import discussion
from vj4.util import argmethod
from vj4.util import options
options.define('dryrun', default=True, help='Dry run.')
_logger = logging.getLogger(__name__)
@argmethod.wrap
def address(ip: str):
return _address(ip, set(), set(), set())
@argmethod.wrap
def discuss(domain_id: str, did: document.convert_doc_id):
return _discussion(domain_id, did, set(), set(), set())
@argmethod.wrap
def usr(uid: int):
return _user(uid, set(), set(), set())
async def _address(ip, bset, uset, dset):
if ip in bset:
import atexit
import coloredlogs
import logging
import os
import signal
import socket
import sys
import urllib.parse
from aiohttp import web
from coloredlogs import syslog
from vj4 import app
from vj4.util import options
options.define('listen',
default='http://127.0.0.1:34765',
help='Server listening address.')
options.define('prefork', default=1, help='Number of prefork workers.')
options.define('syslog',
default=False,
help='Use syslog instead of stderr for logging.')
_logger = logging.getLogger(__name__)
def main():
if not options.syslog:
coloredlogs.install(
level=logging.DEBUG if options.debug else logging.INFO,
fmt=
'[%(levelname).1s %(asctime)s %(module)s:%(lineno)d] %(message)s',
import logging
from vj4 import db
from vj4.model import blacklist
from vj4.model import document
from vj4.model import user
from vj4.model.adaptor import discussion
from vj4.util import argmethod
from vj4.util import options
options.define('dryrun', default=True, help='Dry run.')
_logger = logging.getLogger(__name__)
@argmethod.wrap
def address(ip: str):
return _address(ip, set(), set(), set())
@argmethod.wrap
def discuss(domain_id: str, did: document.convert_doc_id):
return _discussion(domain_id, did, set(), set(), set())
@argmethod.wrap
def usr(uid: int):
return _user(uid, set(), set(), set())
async def _address(ip, bset, uset, dset):
if ip in bset:
import sockjs
from aiohttp import web
from vj4 import db
from vj4 import error
from vj4.model import system
from vj4.service import bus
from vj4.service import smallcache
from vj4.service import staticmanifest
from vj4.util import json
from vj4.util import locale
from vj4.util import options
from vj4.util import tools
options.define('debug', default=False, help='Enable debug mode.')
options.define('static', default=True, help='Serve static files.')
options.define('ip_header', default='', help='Header name for remote IP.')
options.define('unsaved_session_expire_seconds', default=43200,
help='Expire time for unsaved session, in seconds.')
options.define('saved_session_expire_seconds', default=2592000,
help='Expire time for saved session, in seconds.')
options.define('cookie_domain', default='', help='Cookie domain.')
options.define('cookie_secure', default=False, help='Enable secure cookie flag.')
options.define('registration_token_expire_seconds', default=86400,
help='Expire time for registration token, in seconds.')
options.define('lostpass_token_expire_seconds', default=3600,
help='Expire time for lostpass token, in seconds.')
options.define('changemail_token_expire_seconds', default=3600,
help='Expire time for changemail token, in seconds.')
options.define('url_prefix', default='https://vijos.org', help='URL prefix.')
import asyncio
import os
from vj4 import app
from vj4.util import options
options.define('path', default='/tmp/vijos.sock', help='UNIX socket path.')
if __name__ == '__main__':
options.parse_command_line()
try:
os.remove(options.options.path)
except OSError:
pass
loop = asyncio.get_event_loop()
loop.run_until_complete(
loop.create_unix_server(app.Application().make_handler(), options.options.path))
loop.run_forever()
import asyncio
import atexit
import logging
import os
import signal
import socket
import sys
import urllib.parse
from vj4 import app
from vj4.util import options
options.define('listen',
default='http://127.0.0.1:8888',
help='Server listening address.')
options.define('prefork', default=1, help='Number of prefork workers.')
_logger = logging.getLogger(__name__)
def main():
options.parse_command_line()
_logger.info('Server listening on %s', options.options.listen)
url = urllib.parse.urlparse(options.options.listen)
if url.scheme == 'http':
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
host, port_str = url.netloc.rsplit(':', 1)
sock.bind((host, int(port_str)))
elif url.scheme == 'unix':
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
import aiomongo
import functools
import yarl
from vj4.util import options
options.define('db_host',
default='localhost',
help='Database hostname or IP address.')
options.define('db_port', default=27017, help='Database port.')
options.define('db_name', default='test', help='Database name.')
options.define('db_username', default='', help='Database username.')
options.define('db_password', default='', help='Database password.')
options.define('db_auth_source',
default='',
help='Database name associated with the user\'s credential.')
async def init():
global _client, _db
query = dict()
if options.db_auth_source:
query['authSource'] = options.db_auth_source
url = yarl.URL.build(
scheme='mongodb',
host=options.db_host,
path='/' + options.db_name,
port=options.db_port,
user=options.db_username if options.db_username else None,
password=options.db_password if options.db_password else None,
import atexit
import coloredlogs
import logging
import os
import shutil
import signal
import socket
import sys
import urllib.parse
from aiohttp import web
from coloredlogs import syslog
from vj4 import app
from vj4.util import options
options.define('listen', default='http://127.0.0.1:8888', help='Server listening address.')
options.define('prefork', default=1, help='Number of prefork workers.')
options.define('syslog', default=False, help='Use syslog instead of stderr for logging.')
options.define('listen_owner', default='', help='Owner of the unix socket which is server listening to.')
options.define('listen_group', default='', help='Group of the unix socket which is server listening to.')
options.define('listen_mode', default='', help='File mode of the unix socket which is server listening to.')
_logger = logging.getLogger(__name__)
def main():
if not options.syslog:
coloredlogs.install(level=logging.DEBUG if options.debug else logging.INFO,
fmt='[%(levelname).1s %(asctime)s %(module)s:%(lineno)d] %(message)s',
datefmt='%y%m%d %H:%M:%S')
else:
"""A locale module which mimics tornado's interface."""
import collections
import os
import os.path
import yaml
from vj4.util import options
options.define('default_locale', default='zh_CN', help='Default locale.')
_locales = {}
def _init():
translation_path = os.path.join(os.path.dirname(__file__), '..', 'locale')
langs = []
for filename in os.listdir(translation_path):
if not filename.endswith(".yaml"):
continue
with open(os.path.join(translation_path, filename),
encoding='utf-8') as yaml_file:
code = filename[:-5]
name = yaml_file.readline()[1:].strip()
locale = yaml.load(yaml_file)
_locales[code] = locale
if code == options.default_locale:
global _default_locale
_default_locale = locale
langs.append((code, name))
global VIEW_LANGS
VIEW_LANGS = collections.OrderedDict(langs)
from email.mime import text
import smtplibaio
from vj4.util import argmethod
from vj4.util import options
options.define('smtp_host', default='', help='SMTP server')
options.define('smtp_port', default=465, help='SMTP server')
options.define('smtp_user', default='', help='SMTP username')
options.define('smtp_password', default='', help='SMTP password')
options.define('mail_from', default='', help='Mail from')
async def send_mail(to: str, subject: str, content: str):
msg = text.MIMEText(content, _subtype='html', _charset='UTF-8')
msg['Subject'] = subject
msg['From'] = options.options.mail_from
msg['To'] = to
server = smtplibaio.SMTP_SSL()
await server.connect(host=options.options.smtp_host, port=options.options.smtp_port)
await server.ehlo()
await server.login(options.options.smtp_user, options.options.smtp_password)
await server.sendmail(options.options.mail_from, to, msg.as_string())
await server.quit()
if __name__ == '__main__':
argmethod.invoke_by_args()
import asyncio
import aioamqp
from vj4.util import options
options.define('mq_host',
default='172.111.0.2',
help='Message queue hostname or IP address.')
options.define('mq_vhost', default='/', help='Message queue virtual host.')
_protocol_future = None
_channel_futures = {}
async def _connect():
global _protocol_future
if _protocol_future:
return await _protocol_future
_protocol_future = future = asyncio.Future()
try:
_, protocol = await aioamqp.connect(host=options.mq_host,
virtualhost=options.mq_vhost)
future.set_result(protocol)
asyncio.get_event_loop().create_task(_wait_protocol(protocol))
return protocol
except Exception as e:
future.set_exception(e)
_protocol_future = None
raise
from motor import motor_asyncio
from vj4.util import options
options.define('db_host',
default='localhost',
help='Database hostname or IP address.')
options.define('db_name', default='test', help='Database name.')
class Database(object):
_instance = None
def __new__(cls):
if not cls._instance:
client = motor_asyncio.AsyncIOMotorClient(options.options.db_host)
cls._instance = motor_asyncio.AsyncIOMotorDatabase(
client, options.options.db_name)
return cls._instance
class Collection(object):
_instances = {}
def __new__(cls, name):
if name not in cls._instances:
cls._instances[name] = motor_asyncio.AsyncIOMotorCollection(
Database(), name)
return cls._instances[name]
import asyncio
import functools
import logging
import sockjs
from aiohttp import web
from os import path
from vj4 import error
from vj4.controller import smallcache
from vj4.model import bus
from vj4.util import json
from vj4.util import locale
from vj4.util import options
from vj4.util import tools
options.define('static', default=True, help='Serve static files.')
options.define('ip_header', default='X-Real-IP', help='Header name for remote IP.')
options.define('unsaved_session_expire_seconds', default=43200,
help='Expire time for unsaved session, in seconds.')
options.define('saved_session_expire_seconds', default=2592000,
help='Expire time for saved session, in seconds.')
options.define('cookie_domain', default=None, help='Cookie domain.')
options.define('cookie_secure', default=False, help='Enable secure cookie flag.')
options.define('registration_token_expire_seconds', default=86400,
help='Expire time for registration token, in seconds.')
options.define('url_prefix', default='https://vijos.org', help='URL prefix.')
options.define('cdn_prefix', default='/', help='CDN prefix.')
_logger = logging.getLogger(__name__)
class Application(web.Application):
def __init__(self):
import logging
from aiohttp import web
from vj4 import app
from vj4.util import options
options.define('port', default=8888, help='HTTP server port.')
_logger = logging.getLogger(__name__)
if __name__ == '__main__':
options.parse_command_line()
web.run_app(app.Application(),
port=options.options.port,
print=lambda s: [logging.info(l) for l in s.splitlines()])