def parse_config():
options.parse_command_line()
if options.config != "":
logger.info("parse config from config file: {config}".format(
config=options.config))
options.parse_config_file(options.config)
if options.slack_token == "":
logger.error("slack token is required!!")
sys.exit(1)
if options.models_path == "":
logger.error("models path is required!!")
sys.exit(1)
if options.db_path == "":
logger.info(
"path of database is not set, use the default db path: {path}".
format(path=DEFAULT_DB_PATH))
options.db_path = DEFAULT_DB_PATH
mg.MODELS_PATH = options.models_path
alert.SLACK_TOKEN = options.slack_token
job.REPORT_ADDRESS = options.address
logger.info("config: {config}".format(config=options.items()))
def _log_options():
if tornado.version_info < (3, 0, 0, 0):
for key, option in options.iteritems():
gen_logger.info('Options: (%s, %s)', key, option.value())
else:
for key, option in options.items():
gen_logger.info('Options: (%s, %s)', key, option)
def main():
try:
tornado.options.parse_command_line()
if (options.docker):
# Translate the environment variables Docker creates
# when containers are linked to a Python dictionary.
from docker_links import parse_links
links = parse_links(os.environ)
logger.info("Docker mode (links):")
logger.info("\n%s" % json.dumps(links, indent=4))
# The user can specify another alias to look for.
if options.docker_db_alias not in links:
raise Exception('No Docker alias named "%s" found. Available:\n%s' \
% (options.docker_db_alias, "\n".join(links.keys())))
db = links[options.docker_db_alias]
options.rethinkdb_host = db["hostname"]
logger.info("Options:\n%s" % "\n".join(map(lambda x: \
"%25s: %s" % (x[0], x[1]), options.items())))
http_server = tornado.httpserver.HTTPServer(Application())
http_server.listen(options.http_port)
tornado.ioloop.IOLoop.instance().start()
except Exception as ex:
logger.error("Error: %s" % ex)
exit(-1)
def connect(self, url=None, options=None, callback=None):
if url is not None:
self.url = url
purl = urlparse(self.url)
credentials = pika.PlainCredentials(purl.username, purl.password)
virtual_host = purl.path[1:]
host = purl.hostname
port = purl.port
options = options or {}
options = dict([(k.lstrip('DEFAULT_').lower(), v) for k, v in options.items()])
options.update(host=host, port=port, virtual_host=virtual_host,
credentials=credentials)
params = pika.ConnectionParameters(**options)
try:
TornadoConnection(params, stop_ioloop_on_close=False,
on_open_callback=partial(self.on_connect, callback),
custom_ioloop=self.io_loop)
except AMQPConnectionError:
logging.info('Retrying to connect in 2 seconds')
self.io_loop.add_timeout(
timedelta(seconds=2),
partial(self.connect, url=url,
options=options, callback=callback))
def _log_options():
if tornado.version_info < (3, 0, 0, 0):
for key, option in options.iteritems():
gen_logger.info('Options: (%s, %s)', key, option.value())
else:
for key, option in options.items():
gen_logger.info('Options: (%s, %s)', key, option)
def connect(self, url=None, options=None, callback=None):
if url is not None:
self.url = url
purl = urlparse(self.url)
credentials = pika.PlainCredentials(purl.username, purl.password)
virtual_host = purl.path[1:]
host = purl.hostname
port = purl.port
options = options or {}
options = dict([(k.lstrip('DEFAULT_').lower(), v)
for k, v in options.items()])
options.update(host=host,
port=port,
virtual_host=virtual_host,
credentials=credentials)
params = pika.ConnectionParameters(**options)
try:
TornadoConnection(params,
stop_ioloop_on_close=False,
on_open_callback=partial(self.on_connect,
callback),
custom_ioloop=self.io_loop)
except AMQPConnectionError:
logging.info('Retrying to connect in 2 seconds')
self.io_loop.add_timeout(
timedelta(seconds=2),
partial(self.connect,
url=url,
options=options,
callback=callback))
def start(self):
def kill_server(sig, frame):
LOG.warning( 'Catch SIG: %d' % sig )
tornado.ioloop.IOLoop.instance().stop()
# ignore Broken Pipe signal
signal.signal(signal.SIGPIPE, signal.SIG_IGN);
# catch kill signal
signal.signal(signal.SIGINT, kill_server)
signal.signal(signal.SIGQUIT, kill_server)
signal.signal(signal.SIGTERM, kill_server)
signal.signal(signal.SIGHUP, kill_server)
for log_name in self.log_list:
mownfish.util.log.setup(log_name)
LOG.info('START TORNADO WEB SERVER ...')
for key, value in sorted(options.items(), key=lambda d:d[0]):
if key not in ('help', 'log_file_prefix', 'log_to_stderr') \
and value.value() is None:
sys.stderr.write('must specify %s\n' % key)
options.print_help()
sys.exit(0)
LOG.info('Options: (%s, %s)', key, value.value())
try:
sockets = tornado.netutil.bind_sockets(options.port,
address=options.bind_ip,
backlog=128)
if not options.multiports:
task_id = tornado.process.fork_processes(options.num_process)
http_server = \
tornado.httpserver.HTTPServer(xheaders=True,
request_callback=self.application)
http_server.add_sockets(sockets)
self.prepare()
tornado.ioloop.IOLoop.instance().start()
http_server.stop()
tornado.ioloop.IOLoop.instance().stop()
LOG.info('STOP TORNADO WEB SERVER ...')
except socket.error as e:
LOG.warning('Socket Error: %s' % str(e))
except KeyboardInterrupt as e:
LOG.warning('Gently Quit')
except Exception as e:
LOG.error('UnCaught Exception: %s' % e, exc_info=True)
def make_app(environ, argv1=None):
parser = OptionParser()
parser.add_option(
"-c", "--config", dest="config_file",
help="Configuration file", metavar="FILE")
parser.add_option(
"-d", "--data", dest="data",
help="Data directory", metavar="FILE")
parser.add_option(
"-l", "--logfile", dest="log_file_prefix",
help="Log file", metavar="FILE")
parser.add_option(
"-L", "--log-level", dest="logging",
help="Log level", metavar="info|warning|error|none")
parser.add_option(
"-p", "--port", dest="port",
help="Port", default="0")
opts, args = parser.parse_args(argv1)
name = args[0]
if opts.data.startswith('/'):
instance_path = os.path.join(opts.data, name)
else:
instance_path = os.path.join(os.getcwd(), opts.data, name)
config_file = opts.config_file or os.path.join(os.path.abspath(instance_path), "etc", "vaytrou.conf")
if config_file is not None:
parse_config_file(config_file)
parse_command_line([])
# Load command line options over Tornado options
for k, v in options.items():
optv = getattr(opts, k, None)
if optv is not None:
try:
options[k].set(v.type(optv))
except:
import pdb; pdb.set_trace()
raise
finder = PersistentApplicationFinder(
'file://%s/%s/var/Data.fs' % (options.data, name), appmaker)
index = finder(environ)
index.fwd = Rtree('%s/%s/var/vrt1' % (options.data, name))
setattr(index, 'name', name)
setattr(index, 'data', options.data)
return Application(index, [
(r'/', MainHandler),
(r'/intersection/?', IntersectionHandler),
(r'/nearest/?', NearestHandler),
(r'/distance/?', DistanceHandler),
(r'/within/?', WithinHandler),
(r'/item/([-\w]+);([-0-9\.]+),([-0-9\.]+),([-0-9\.]+),([-0-9\.]+)',
SingleItemHandler),
(r'/items/([-\w]+)', MultipleItemsHandler)
])
def options_to_settings(options):
"""
Converts the given Tornado-style *options* to new-style settings. Returns
an :class:`RUDict` containing all the settings.
"""
settings = RUDict({'*': {'gateone': {}, 'terminal': {}}})
# In the new settings format some options have moved to the terminal app.
# These settings are below and will be placed in the 'terminal' sub-dict.
terminal_options = [
'command', 'dtach', 'session_logging', 'session_logs_max_age',
'syslog_session_logging'
]
non_options = [
# These are things that don't really belong in settings
'new_api_key',
'help',
'kill',
'config',
'version',
'configure'
]
for key, value in options.items():
if key in terminal_options:
settings['*']['terminal'].update({key: value})
elif key in non_options:
continue
else:
if key == 'origins':
#if value == '*':
#continue
# Convert to the new format (a list with no http://)
origins = value.split(';')
converted_origins = []
for origin in origins:
if '://' in origin:
# The new format doesn't bother with http:// or https://
origin = origin.split('://')[1]
if origin not in converted_origins:
converted_origins.append(origin)
elif origin not in converted_origins:
converted_origins.append(origin)
settings['*']['gateone'].update({key: converted_origins})
elif key == 'api_keys':
if not value:
continue
# API keys/secrets are now a dict instead of a string
settings['*']['gateone']['api_keys'] = {}
for pair in value.split(','):
api_key, secret = pair.split(':', 1)
if bytes == str: # Python 2
api_key = api_key.decode('UTF-8')
secret = secret.decode('UTF-8')
settings['*']['gateone']['api_keys'].update(
{api_key: secret})
else:
settings['*']['gateone'].update({key: value})
return settings
def start(self):
def kill_server(sig, frame):
LOG.warning('Catch SIG: %d' % sig)
tornado.ioloop.IOLoop.instance().stop()
# ignore Broken Pipe signal
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
# catch kill signal
signal.signal(signal.SIGINT, kill_server)
signal.signal(signal.SIGQUIT, kill_server)
signal.signal(signal.SIGTERM, kill_server)
signal.signal(signal.SIGHUP, kill_server)
for log_name in self.log_list:
mownfish.util.log.setup(log_name)
LOG.info('START TORNADO WEB SERVER ...')
for key, value in sorted(options.items(), key=lambda d: d[0]):
if key not in ('help', 'log_file_prefix', 'log_to_stderr') \
and value.value() is None:
sys.stderr.write('must specify %s\n' % key)
options.print_help()
sys.exit(0)
LOG.info('Options: (%s, %s)', key, value.value())
try:
sockets = tornado.netutil.bind_sockets(options.port,
address=options.bind_ip,
backlog=128)
if not options.multiports:
task_id = tornado.process.fork_processes(options.num_process)
http_server = \
tornado.httpserver.HTTPServer(xheaders=True,
request_callback=self.application)
http_server.add_sockets(sockets)
self.prepare()
tornado.ioloop.IOLoop.instance().start()
http_server.stop()
tornado.ioloop.IOLoop.instance().stop()
LOG.info('STOP TORNADO WEB SERVER ...')
except socket.error as e:
LOG.warning('Socket Error: %s' % str(e))
except KeyboardInterrupt as e:
LOG.warning('Gently Quit')
except Exception as e:
LOG.error('UnCaught Exception: %s' % e, exc_info=True)
def options_to_settings(options):
"""
Converts the given Tornado-style *options* to new-style settings. Returns
an :class:`RUDict` containing all the settings.
"""
settings = RUDict({'*': {'gateone': {}, 'terminal': {}}})
# In the new settings format some options have moved to the terminal app.
# These settings are below and will be placed in the 'terminal' sub-dict.
terminal_options = [
'command', 'dtach', 'session_logging', 'session_logs_max_age',
'syslog_session_logging'
]
non_options = [
# These are things that don't really belong in settings
'new_api_key', 'help', 'kill', 'config'
]
for key, value in options.items():
if key in terminal_options:
settings['*']['terminal'].update({key: value})
elif key in non_options:
continue
else:
if key == 'origins':
#if value == '*':
#continue
# Convert to the new format (a list with no http://)
origins = value.split(';')
converted_origins = []
for origin in origins:
if '://' in origin:
# The new format doesn't bother with http:// or https://
origin = origin.split('://')[1]
if origin not in converted_origins:
converted_origins.append(origin)
elif origin not in converted_origins:
converted_origins.append(origin)
settings['*']['gateone'].update({key: converted_origins})
elif key == 'api_keys':
if not value:
continue
# API keys/secrets are now a dict instead of a string
settings['*']['gateone']['api_keys'] = {}
for pair in value.split(','):
api_key, secret = pair.split(':', 1)
if bytes == str: # Python 2
api_key = api_key.decode('UTF-8')
secret = secret.decode('UTF-8')
settings['*']['gateone']['api_keys'].update(
{api_key: secret})
else:
settings['*']['gateone'].update({key: value})
return settings
def parse_config():
options.parse_command_line()
if options.config != "":
logger.info("parse config from config file: {config}".format(
config=options.config))
options.parse_config_file(options.config)
if options.slack_token == "":
logger.error("slack token is required!!")
sys.exit(1)
alert.SLACK_TOKEN = options.slack_token
logger.info("config: {config}".format(config=options.items()))
def run(service, port):
if not options.local_debug:
logging.info('\n--\nRun ' + service)
for k, v in options.items():
logging.info('[Option] ' + k + ' = ' + str(v))
logging.info('--\n')
# Load specific app from module if available.
app_cls = Application
try:
module = __import__(service + '.application', fromlist=['Application'])
app_cls = getattr(module, 'Application')
except:
pass
application = app_cls(service, port)
return application
def app_options():
original_settings = dict(options.items())
yield options
for key, value in original_settings.items():
setattr(options, key, value)
import os
from tornado.options import options
from tornado.options import define
dbname = os.path.join(os.path.dirname(__file__), os.pardir, "base.sql")
epub_path = os.path.join(os.path.dirname(__file__), os.pardir, "files")
define("DBNAME", default=dbname, help="Name of the local DB cache")
define("PORT", default=8080, help="Port to listen to")
define("DB_UPDATE_TIMEOUT", default=1000000, help="Interval to update DB")
define("EPUB_FILES_PATH", default=epub_path, help="Files storage directory")
options.parse_command_line()
for i in options.items(): # print the init parameters on boot
print "{} = {}".format(i[0], i[1])
# map options to constants / too lazy to refactor in a elegant way
EPUB_FILES_PATH = options.EPUB_FILES_PATH
DBNAME = os.path.join(os.path.dirname(__file__), os.pardir, options.DBNAME)
PORT, DB_UPDATE_TIMEOUT = options.PORT, options.DB_UPDATE_TIMEOUT
self.write(json.dumps(ret))
if __name__ == "__main__":
SETTINGS = {
"cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
"debug": False,
}
APPLICATION = tornado.web.Application([
(r"/api/dmp/taguser", GetUserHandler),
(r"/api/dmp/usertag", GetLabelHandler),
], **SETTINGS)
http_server = tornado.httpserver.HTTPServer(APPLICATION)
'''
Get the option(s) from the startup command line if ever.
In this tutorial, we define own "port" option to change the
port via the command line, and then we can run multiple tornado
processes at different ports.
'''
tornado.options.parse_command_line()
# This line should be after the parse_command_line()
http_server.listen(8888)
REDIS_HOST = options.redis_host
REDIS_PORT = options.redis_port
REDIS_AUTH = options.redis_auth
VALID_SECONDS = options.valid_seconds
logging.info(options.items())
tornado.ioloop.IOLoop.instance().start()
def __init__(self, *args, **kwargs):
kwargs.update(options.items())
kwargs.setdefault('handlers', []).extend(routes())
super().__init__(*args, **kwargs)
def app_options():
original_settings = dict(options.items())
yield options
for key, value in original_settings.items():
setattr(options, key, value)
define('haze',default='http://api-howler.io.bittorrent.com', type=str)
define('datapath',default='/home/kyle/virtualbox-shared/ktorrent', type=str)
define('ktorrent_path',default='/home/kyle/ktorrent', type=str)
define('utserver_username',default='admin', type=str)
define('utserver_password',default='', type=str)
define('utserver_webui_port',default=8080, type=int)
tornado.options.parse_command_line()
try:
import config_prod as config
options.utserver_username = config.utserver_username
options.utserver_password = config.utserver_password
except:
logging.warn('no config.py found -- using default settings')
settings = dict( (k, v.value()) for k,v in options.items() )
ioloop = tornado.ioloop.IOLoop.instance()
from handlers import IndexHandler, TestHandler
routes = [
('/?', IndexHandler),
('/test?', TestHandler),
]
application = tornado.web.Application(routes, **settings)
frontend_application = tornado.web.Application(routes, **settings)
frontend_server = tornado.httpserver.HTTPServer(frontend_application, io_loop=ioloop)
frontend_server.bind(options.frontend_port, '')
frontend_server.start()
logging.info('ioloop starting')
def main():
try:
tornado.options.parse_command_line()
if options.docker:
# Translate the environment variables Docker creates
# when containers are linked to a Python dictionary.
from docker_links import parse_links
links = parse_links(os.environ)
logger.info("Docker mode (links):")
logger.info("\n%s" % json.dumps(links, indent=4))
# The user can specify another alias to look for.
if options.docker_db_alias not in links:
raise Exception(
'No Docker alias named "%s" found. Available:\n%s'
% (options.docker_db_alias, "\n".join(links.keys()))
)
db = links[options.docker_db_alias]
options.rethinkdb_host = db["hostname"]
logger.info("Options:\n%s" % "\n".join(map(lambda x: "%25s: %s" % (x[0], x[1]), options.items())))
http_server = tornado.httpserver.HTTPServer(Application())
http_server.listen(options.http_port)
tornado.ioloop.IOLoop.instance().start()
except Exception as ex:
logger.error("Error: %s" % ex)
exit(-1)