def __init__(self):
"""
Initialize tornado server for receiving/grading submissions
"""
endpoint = ARGS.endpoint or os.environ.get("OTTER_ENDPOINT", None)
assert endpoint is not None, "no endpoint address provided"
assert os.path.isdir(OTTER_SERVICE_DIR), "{} does not exist".format(
OTTER_SERVICE_DIR)
settings = dict(google_oauth={
"key":
ARGS.google_key or os.environ.get("GOOGLE_CLIENT_KEY", None),
"secret":
ARGS.google_secret or os.environ.get("GOOGLE_CLIENT_SECRET", None)
},
notebook_dir=os.path.join(OTTER_SERVICE_DIR,
"submissions"),
auth_redirect_uri=os.path.join(endpoint,
"auth/callback"))
handlers = [(r"/submit", SubmissionHandler),
(r"/auth/google", GoogleOAuth2LoginHandler),
(r"/auth/callback", GoogleOAuth2LoginHandler),
(r"/auth", LoginHandler)]
tornado.web.Application.__init__(self, handlers, **settings)
# Initialize database session
self.db = queries.TornadoSession(
queries.uri(host=ARGS.db_host,
port=ARGS.db_port,
dbname='otter_db',
user=ARGS.db_user,
password=ARGS.db_pass))
async def infinite_update_loop():
while True:
await asyncio.sleep(
UPDATE_PERIOD
) # switch to other code and continue execution in 5 seconds
db = options.group_dict('db')
dbpass = db['password']
dbuser = db['username']
dbhost = db['host']
dbname = db['name']
dbport = db['port']
session = queries.TornadoSession(
f'postgresql://{dbuser}:{dbpass}@{dbhost}:{dbport}/{dbname}')
## DETERMINE IF THERE WAS AN UPDATE OR NOT BY CHECKING IN THE PASSED XX SECONDS (which should coincide with
## HOW EVER LONG WE WAIT BAHHHH!HHH!HH!H!!HHHH
if update:
logger.debug(f"sending update, next update in {UPDATE_PERIOD}")
for connection in connections:
await to_asyncio_future(
connection.write_message(msg)
) # send message to each connected client
else:
logger.debug(f"next update in {UPDATE_PERIOD}")
def __init__(self, host, dbport, dbname, user, password):
# Escape password as it may contain special characters.
# Strip whitespace from other parameters.
# Strip trailing '/' from host.
connection_uri = 'postgresql://{user}:{pw}@{host}:{dbport}/{dbname}'.format(
user=user.strip(),
pw=urllib.parse.quote_plus(password),
host=host.strip().rstrip('/'),
dbport=dbport,
dbname=dbname.strip(),
)
# Check that connection can be established
try:
with queries.Session(connection_uri) as session:
session.query('SELECT 1')
except Exception as exception:
print('ERROR: Unable to connect to database:')
raise exception
# Check that test archive contains data
try:
with queries.Session(connection_uri) as session:
session.query('SELECT count(*) FROM test_run')
except Exception:
print(
'ERROR: Given database is empty. Consider archiving some results first.'
)
sys.exit(1)
self.session = queries.TornadoSession(connection_uri)
def __init__(self):
settings = dict(auth_redirect_uri=None)
handlers = [(r"/auth/google", start.GoogleOAuth2LoginHandler),
(r"/auth/callback",
start.GoogleOAuth2LoginHandler),
(r"/auth", start.LoginHandler)]
Application.__init__(self, handlers, **settings)
self.db = queries.TornadoSession(queries.uri(**dsn))
def __init__(self):
pqsql_uri = queries.uri(options.pqsql_host, options.pqsql_port,
options.pqsql_db, options.pqsql_user,
options.pqsql_password)
self.pool = queries.TornadoSession(pqsql_uri,
pool_max_size=options.pool_max_size,
pool_idle_ttl=options.pool_idle_ttl)
self.cur = ''
def initialize(self):
logger.debug("setting up db session")
db = options.group_dict('db')
dbpass = db['password']
dbuser = db['username']
dbhost = db['host']
dbname = db['name']
dbport = db['port']
self.session = queries.TornadoSession(
f'postgresql://{dbuser}:{dbpass}@{dbhost}:{dbport}/{dbname}')
def __init__(self, host, dbname, user, password):
# Escape password as it may contain special characters.
# Strip whitespace from other parameters.
# Strip trailing '/' from host.
connection_uri = 'postgresql://{user}:{pw}@{host}/{dbname}'.format(
user=user.strip(),
pw=urllib.parse.quote_plus(password),
host=host.strip().rstrip('/'),
dbname=dbname.strip(),
)
self.session = queries.TornadoSession(connection_uri)
def __init__(self, host, dbname, user, password):
connection_uri = 'postgresql://{user}:{pw}@{host}/{dbname}'.format(
user=user.strip(),
pw=urllib.parse.quote_plus(password),
host=host.strip().rstrip('/'),
dbname=dbname.strip(),
)
if sys.platform == 'win32':
asyncio.set_event_loop_policy(
asyncio.WindowsSelectorEventLoopPolicy())
self.session = queries.TornadoSession(connection_uri)
def session(connect=None, pool_size=10, is_tornado=False):
if connect is None:
connect = DB_CONNECT
if connect is None:
raise Exception('Please provide a connection string or set the NHLDB_CONNECT env. variable')
if is_tornado:
s = queries.TornadoSession(connect, pool_max_size=pool_size)
else:
s = queries.Session(connect, pool_max_size=pool_size)
try:
yield s
finally:
s.close()
def __init__(self):
handlers = [(r"/submit", SubmissionHandler),
(r"/google_auth", GoogleOAuth2LoginHandler)]
with open("conf.yml") as f:
config = yaml.safe_load(f)
settings = dict(google_oauth={
'key': config['google_auth_key'],
'secret': config['google_auth_secret'],
},
notebook_dir=config['notebook_dir'],
auth_redirect_uri=config['auth_redirect_uri'])
tornado.web.Application.__init__(self, handlers, **settings)
# Initialize database session
self.db = queries.TornadoSession(
queries.uri(host=config['db_host'],
port=config['db_port'],
dbname='otter_db',
user=config['db_user'],
password=config['db_pass']))
def test_polling_stops_after_connection_error(self):
# Abort the test right away if postgres isn't running.
yield self.assertPostgresConnected()
# Use an invalid user to force an OperationalError during connection
bad_uri = queries.uri(os.getenv('PGHOST', 'localhost'),
int(os.getenv('PGPORT', '5432')), 'invalid')
session = queries.TornadoSession(bad_uri)
self.count = 0
real_poll_connection = session._poll_connection
def count_polls(*args, **kwargs):
self.count += 1
real_poll_connection(*args, **kwargs)
session._poll_connection = count_polls
with self.assertRaises(queries.OperationalError):
yield session.query('SELECT 1')
yield gen.sleep(0.05)
self.assertLess(self.count, 20)
def main():
'''
Manage Asynchronous Number General ORGs
Organizations of Roman Generality.
'''
# Now we can run a few servers and processes
# Server daemons and ports
server_push_port = "5556"
server_pub_port = "5558"
# System frontend & backend routers and ports
frontend_port = "4144"
backend_port = "4188"
# Servers
Process(target=server_push, args=(server_push_port, )).start()
Process(target=server_pub, args=(server_pub_port, )).start()
Process(target=server_router, args=(
frontend_port,
backend_port,
)).start()
# Clients
Process(target=client, args=(
server_push_port,
server_pub_port,
)).start()
# Start background tasks
def start(task, *args):
process = Process(target=task, args=args)
process.daemon = True
process.start()
for i in range(NBR_CLIENTS):
start(client_task, i)
for i in range(NBR_WORKERS):
start(worker_task, i)
# Initialize main loop state
count = NBR_CLIENTS
# daemon options
opts = options.options()
# Set document database
document = motor.MotorClient(opts.mongo_host, opts.mongo_port).mango
# Set memcached backend
memcache = mc.Client([opts.memcached_host],
binary=opts.memcached_binary,
behaviors={
"tcp_nodelay": opts.memcached_tcp_nodelay,
"ketama": opts.memcached_ketama
})
# Set SQL URI
postgresql_uri = queries.uri(host=opts.sql_host,
port=opts.sql_port,
dbname=opts.sql_database,
user=opts.sql_user,
password=None)
# Set kvalue database
kvalue = False
# Set default cache
global cache
cache = memcache
# Set SQL session
global sql
sql = queries.TornadoSession(uri=postgresql_uri)
# Set default database
global db
db = document
# logging database hosts
logging.info('MongoDB server: {0}:{1}'.format(opts.mongo_host,
opts.mongo_port))
logging.info('PostgreSQL server: {0}:{1}'.format(opts.sql_host,
opts.sql_port))
# Ensure
if opts.ensure_indexes:
logging.info('Ensuring indexes...')
indexes.ensure_indexes(db)
logging.info('DONE.')
# base url
base_url = opts.base_url
# system cache
cache_enabled = opts.cache_enabled
if cache_enabled:
logging.info('Memcached server: {0}:{1}'.format(
opts.memcached_host, opts.memcached_port))
# mango web application daemon
application = web.Application(
[
# Mango system knowledge (quotes) and realtime events.
(r'/system/?', MangoHandler),
# Basic-Auth session
(r'/login/?', LoginHandler),
(r'/logout/?', LogoutHandler),
# ORGs records
(r'/orgs/(?P<account>.+)/records/?', accounts.RecordsHandler),
(r'/orgs/(?P<account>.+)/records/page/(?P<page_num>\d+)/?',
accounts.RecordsHandler),
(r'/orgs/(?P<account>.+)/records/?', accounts.RecordsHandler),
(r'/orgs/(?P<account>.+)/records/page/(?P<page_num>\d+)/?',
accounts.RecordsHandler),
# ORGs teams
(r'/orgs/(?P<account>.+)/teams/page/(?P<page_num>\d+)/?',
accounts.TeamsHandler),
(r'/orgs/(?P<account>.+)/teams/(?P<team_uuid>.+)/?',
accounts.TeamsHandler),
(r'/orgs/(?P<account>.+)/teams/?', accounts.TeamsHandler),
# ORGs members
(r'/orgs/(?P<account>.+)/members/page/(?P<page_num>\d+)/?',
accounts.MembersHandler),
(r'/orgs/(?P<account>.+)/members/(?P<user>.+)/?',
accounts.MembersHandler),
(r'/orgs/(?P<account>.+)/members/?', accounts.MembersHandler),
# Organizations of Random Generality.
(r'/orgs/?', accounts.OrgsHandler),
(r'/orgs/(?P<account>.+)/?', accounts.OrgsHandler),
# Users records
(r'/users/(?P<account>.+)/records/?', accounts.RecordsHandler),
(r'/users/(?P<account>.+)/records/page/(?P<page_num>\d+)/?',
accounts.RecordsHandler),
# Users billing routes
(r'/users/(?P<account>.+)/routes/?', accounts.RoutesHandler),
# Users
(r'/users/?', accounts.UsersHandler),
(r'/users/(?P<account>.+)/?', accounts.UsersHandler),
# Records
(r'/records/start/(?P<start>.*)/end/(?P<end>.*)/?', records.Handler
),
(r'/records/start/(?P<start>.*)/?', records.Handler),
(r'/records/end/(?P<end>.*)/?', records.Handler),
(r'/records/page/(?P<page_num>\d+)/?', records.Handler),
# Public records
(r'/records/public/?', records.PublicHandler),
(r'/records/public/page/(?P<page_num>\d+)/?',
records.PublicHandler),
# Unassigned records
(r'/records/unassigned/?', records.UnassignedHandler),
(r'/records/unassigned/page/(?P<page_num>\d+)/?',
records.UnassignedHandler),
# Records summary
# (r'/records/summary/<lapse>/<value>/?', records.SummaryHandler),
# Return last (n) of lapse
# (r'/records/summary/<lapse>/lasts/(?P<int>\d+)/?', records.SummaryHandler),
# Statistical projection based on the previous data.
# (r'/records/summary/<lapse>/nexts/(?P<int>\d+)/?', records.SummaryHandler),
# Records summary
(r'/records/summary/start/(?P<start>.*)/end/(?P<end>.*)/?',
records.SummaryHandler),
(r'/records/summary/start/(?P<start>.*)/?',
records.SummaryHandler),
(r'/records/summary/end/(?P<end>.*)/?', records.SummaryHandler),
(r'/records/summary/(?P<lapse>.*)/start/(?P<start>.*)/end/(?P<end>.*)/?',
records.SummaryHandler),
(r'/records/summary/(?P<lapse>.*)/start/(?P<start>.*)/?',
records.SummaryHandler),
(r'/records/summary/(?P<lapse>.*)/end/(?P<end>.*)/?',
records.SummaryHandler),
# Return last (n) of lapse
# (r'/records/summary/(?P<lapse>.*)/lasts/(?P<int>\d+)/?', records.SummaryHandler),
(r'/records/summary/(?P<lapse>.*)/?', records.SummaryHandler),
(r'/records/summary/?', records.SummaryHandler),
# Records summaries
(r'/records/summaries/start/(?P<start>.*)/end/(?P<end>.*)/?',
records.SummariesHandler),
(r'/records/summaries/start/(?P<start>.*)/?',
records.SummariesHandler),
(r'/records/summaries/end/(?P<end>.*)/?',
records.SummariesHandler),
(r'/records/summaries/(?P<lapse>.*)/start/(?P<start>.*)/end/(?P<end>.*)/?',
records.SummariesHandler),
(r'/records/summaries/(?P<lapse>.*)/start/(?P<start>.*)/?',
records.SummariesHandler),
(r'/records/summaries/(?P<lapse>.*)/end/(?P<end>.*)/?',
records.SummariesHandler),
(r'/records/summaries/(?P<lapse>.*)/?', records.SummariesHandler),
(r'/records/summaries/?', records.SummariesHandler),
# Records
(r'/records/(?P<record_uuid>.+)/?', records.Handler),
(r'/records/?', records.Handler),
# Records
(r'/tasks/(?P<task_uuid>.+)/?', tasks.Handler),
(r'/tasks/?', tasks.Handler),
# Billings
(r'/billings/(?P<billing_uuid>.+)/?', billings.RecordsHandler),
(r'/billings/?', billings.RecordsHandler),
# Billings records
(r'/billings/records/start/(?P<start>.*)/end/(?P<end>.*)/?',
billings.RecordsHandler),
(r'/billings/records/start/(?P<start>.*)/?',
billings.RecordsHandler),
(r'/billings/records/end/(?P<end>.*)/?', billings.RecordsHandler),
(r'/billings/records/?', billings.RecordsHandler)
],
# system database
db=db,
# system cache
cache=cache,
# cache enabled flag
cache_enabled=cache_enabled,
# document datastorage
document=document,
# kvalue datastorage
kvalue=kvalue,
# sql datastorage
sql=sql,
# debug mode
debug=opts.debug,
# application domain
domain=opts.domain,
# application timezone
timezone=opts.timezone,
# pagination page size
page_size=opts.page_size,
# cookie settings
cookie_secret=opts.cookie_secret,
# login url
login_url='/login/')
# Mango periodic cast callbacks
periodic_records = PeriodicCast(periodic_get_records, 5000)
periodic_records.start()
# Setting up mango processor
application.listen(opts.port)
logging.info('Listening on http://%s:%s' % (opts.host, opts.port))
ioloop.IOLoop.instance().start()
def setUp(self):
super(TornadoSessionIntegrationTests, self).setUp()
self.session = queries.TornadoSession(self.pg_uri,
pool_max_size=10,
io_loop=self.io_loop)
def setUp(self):
super(TornadoSessionIntegrationTests, self).setUp()
self.session = queries.TornadoSession(queries.uri(
'localhost', 5432, 'postgres', 'postgres'),
pool_max_size=10,
io_loop=self.io_loop)
def initialize(self):
self.session = queries.TornadoSession('postgresql://*****:*****@localhost:5432/tornado_poc')
def __init__(self):
settings = dict(notebook_dir="test_submissions")
handlers = [(r"/submit", start.SubmissionHandler)]
Application.__init__(self, handlers, **settings)
self.db = queries.TornadoSession(queries.uri(**dsn))
result = yield self.application.session.query(self.SQL)
except queries.OperationalError as error:
logging.error('Error connecting to the database: %s', error)
raise web.HTTPError(503)
rows = []
for row in result.items():
row = dict([
(k, v.isoformat() if isinstance(v, datetime.datetime) else v)
for k, v in row.items()
])
rows.append(row)
result.free()
self.finish({'pg_stat_activity': rows})
class ReportHandler(web.RequestHandler):
@gen.coroutine
def get(self):
self.finish(pool.PoolManager.report())
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
application = web.Application([(r'/', ExampleHandler),
(r'/report', ReportHandler)],
debug=True)
application.session = queries.TornadoSession()
application.listen(8000)
ioloop.IOLoop.instance().start()
