def __init__(self): """ Initialize tornado server for receiving/grading submissions """ endpoint = ARGS.endpoint or os.environ.get("OTTER_ENDPOINT", None) assert endpoint is not None, "no endpoint address provided" assert os.path.isdir(OTTER_SERVICE_DIR), "{} does not exist".format( OTTER_SERVICE_DIR) settings = dict(google_oauth={ "key": ARGS.google_key or os.environ.get("GOOGLE_CLIENT_KEY", None), "secret": ARGS.google_secret or os.environ.get("GOOGLE_CLIENT_SECRET", None) }, notebook_dir=os.path.join(OTTER_SERVICE_DIR, "submissions"), auth_redirect_uri=os.path.join(endpoint, "auth/callback")) handlers = [(r"/submit", SubmissionHandler), (r"/auth/google", GoogleOAuth2LoginHandler), (r"/auth/callback", GoogleOAuth2LoginHandler), (r"/auth", LoginHandler)] tornado.web.Application.__init__(self, handlers, **settings) # Initialize database session self.db = queries.TornadoSession( queries.uri(host=ARGS.db_host, port=ARGS.db_port, dbname='otter_db', user=ARGS.db_user, password=ARGS.db_pass))
async def infinite_update_loop(): while True: await asyncio.sleep( UPDATE_PERIOD ) # switch to other code and continue execution in 5 seconds db = options.group_dict('db') dbpass = db['password'] dbuser = db['username'] dbhost = db['host'] dbname = db['name'] dbport = db['port'] session = queries.TornadoSession( f'postgresql://{dbuser}:{dbpass}@{dbhost}:{dbport}/{dbname}') ## DETERMINE IF THERE WAS AN UPDATE OR NOT BY CHECKING IN THE PASSED XX SECONDS (which should coincide with ## HOW EVER LONG WE WAIT BAHHHH!HHH!HH!H!!HHHH if update: logger.debug(f"sending update, next update in {UPDATE_PERIOD}") for connection in connections: await to_asyncio_future( connection.write_message(msg) ) # send message to each connected client else: logger.debug(f"next update in {UPDATE_PERIOD}")
def __init__(self, host, dbport, dbname, user, password): # Escape password as it may contain special characters. # Strip whitespace from other parameters. # Strip trailing '/' from host. connection_uri = 'postgresql://{user}:{pw}@{host}:{dbport}/{dbname}'.format( user=user.strip(), pw=urllib.parse.quote_plus(password), host=host.strip().rstrip('/'), dbport=dbport, dbname=dbname.strip(), ) # Check that connection can be established try: with queries.Session(connection_uri) as session: session.query('SELECT 1') except Exception as exception: print('ERROR: Unable to connect to database:') raise exception # Check that test archive contains data try: with queries.Session(connection_uri) as session: session.query('SELECT count(*) FROM test_run') except Exception: print( 'ERROR: Given database is empty. Consider archiving some results first.' ) sys.exit(1) self.session = queries.TornadoSession(connection_uri)
def __init__(self): settings = dict(auth_redirect_uri=None) handlers = [(r"/auth/google", start.GoogleOAuth2LoginHandler), (r"/auth/callback", start.GoogleOAuth2LoginHandler), (r"/auth", start.LoginHandler)] Application.__init__(self, handlers, **settings) self.db = queries.TornadoSession(queries.uri(**dsn))
def __init__(self): pqsql_uri = queries.uri(options.pqsql_host, options.pqsql_port, options.pqsql_db, options.pqsql_user, options.pqsql_password) self.pool = queries.TornadoSession(pqsql_uri, pool_max_size=options.pool_max_size, pool_idle_ttl=options.pool_idle_ttl) self.cur = ''
def initialize(self): logger.debug("setting up db session") db = options.group_dict('db') dbpass = db['password'] dbuser = db['username'] dbhost = db['host'] dbname = db['name'] dbport = db['port'] self.session = queries.TornadoSession( f'postgresql://{dbuser}:{dbpass}@{dbhost}:{dbport}/{dbname}')
def __init__(self, host, dbname, user, password): # Escape password as it may contain special characters. # Strip whitespace from other parameters. # Strip trailing '/' from host. connection_uri = 'postgresql://{user}:{pw}@{host}/{dbname}'.format( user=user.strip(), pw=urllib.parse.quote_plus(password), host=host.strip().rstrip('/'), dbname=dbname.strip(), ) self.session = queries.TornadoSession(connection_uri)
def __init__(self, host, dbname, user, password): connection_uri = 'postgresql://{user}:{pw}@{host}/{dbname}'.format( user=user.strip(), pw=urllib.parse.quote_plus(password), host=host.strip().rstrip('/'), dbname=dbname.strip(), ) if sys.platform == 'win32': asyncio.set_event_loop_policy( asyncio.WindowsSelectorEventLoopPolicy()) self.session = queries.TornadoSession(connection_uri)
def session(connect=None, pool_size=10, is_tornado=False): if connect is None: connect = DB_CONNECT if connect is None: raise Exception('Please provide a connection string or set the NHLDB_CONNECT env. variable') if is_tornado: s = queries.TornadoSession(connect, pool_max_size=pool_size) else: s = queries.Session(connect, pool_max_size=pool_size) try: yield s finally: s.close()
def __init__(self): handlers = [(r"/submit", SubmissionHandler), (r"/google_auth", GoogleOAuth2LoginHandler)] with open("conf.yml") as f: config = yaml.safe_load(f) settings = dict(google_oauth={ 'key': config['google_auth_key'], 'secret': config['google_auth_secret'], }, notebook_dir=config['notebook_dir'], auth_redirect_uri=config['auth_redirect_uri']) tornado.web.Application.__init__(self, handlers, **settings) # Initialize database session self.db = queries.TornadoSession( queries.uri(host=config['db_host'], port=config['db_port'], dbname='otter_db', user=config['db_user'], password=config['db_pass']))
def test_polling_stops_after_connection_error(self): # Abort the test right away if postgres isn't running. yield self.assertPostgresConnected() # Use an invalid user to force an OperationalError during connection bad_uri = queries.uri(os.getenv('PGHOST', 'localhost'), int(os.getenv('PGPORT', '5432')), 'invalid') session = queries.TornadoSession(bad_uri) self.count = 0 real_poll_connection = session._poll_connection def count_polls(*args, **kwargs): self.count += 1 real_poll_connection(*args, **kwargs) session._poll_connection = count_polls with self.assertRaises(queries.OperationalError): yield session.query('SELECT 1') yield gen.sleep(0.05) self.assertLess(self.count, 20)
def main(): ''' Manage Asynchronous Number General ORGs Organizations of Roman Generality. ''' # Now we can run a few servers and processes # Server daemons and ports server_push_port = "5556" server_pub_port = "5558" # System frontend & backend routers and ports frontend_port = "4144" backend_port = "4188" # Servers Process(target=server_push, args=(server_push_port, )).start() Process(target=server_pub, args=(server_pub_port, )).start() Process(target=server_router, args=( frontend_port, backend_port, )).start() # Clients Process(target=client, args=( server_push_port, server_pub_port, )).start() # Start background tasks def start(task, *args): process = Process(target=task, args=args) process.daemon = True process.start() for i in range(NBR_CLIENTS): start(client_task, i) for i in range(NBR_WORKERS): start(worker_task, i) # Initialize main loop state count = NBR_CLIENTS # daemon options opts = options.options() # Set document database document = motor.MotorClient(opts.mongo_host, opts.mongo_port).mango # Set memcached backend memcache = mc.Client([opts.memcached_host], binary=opts.memcached_binary, behaviors={ "tcp_nodelay": opts.memcached_tcp_nodelay, "ketama": opts.memcached_ketama }) # Set SQL URI postgresql_uri = queries.uri(host=opts.sql_host, port=opts.sql_port, dbname=opts.sql_database, user=opts.sql_user, password=None) # Set kvalue database kvalue = False # Set default cache global cache cache = memcache # Set SQL session global sql sql = queries.TornadoSession(uri=postgresql_uri) # Set default database global db db = document # logging database hosts logging.info('MongoDB server: {0}:{1}'.format(opts.mongo_host, opts.mongo_port)) logging.info('PostgreSQL server: {0}:{1}'.format(opts.sql_host, opts.sql_port)) # Ensure if opts.ensure_indexes: logging.info('Ensuring indexes...') indexes.ensure_indexes(db) logging.info('DONE.') # base url base_url = opts.base_url # system cache cache_enabled = opts.cache_enabled if cache_enabled: logging.info('Memcached server: {0}:{1}'.format( opts.memcached_host, opts.memcached_port)) # mango web application daemon application = web.Application( [ # Mango system knowledge (quotes) and realtime events. (r'/system/?', MangoHandler), # Basic-Auth session (r'/login/?', LoginHandler), (r'/logout/?', LogoutHandler), # ORGs records (r'/orgs/(?P<account>.+)/records/?', accounts.RecordsHandler), (r'/orgs/(?P<account>.+)/records/page/(?P<page_num>\d+)/?', accounts.RecordsHandler), (r'/orgs/(?P<account>.+)/records/?', accounts.RecordsHandler), (r'/orgs/(?P<account>.+)/records/page/(?P<page_num>\d+)/?', accounts.RecordsHandler), # ORGs teams (r'/orgs/(?P<account>.+)/teams/page/(?P<page_num>\d+)/?', accounts.TeamsHandler), (r'/orgs/(?P<account>.+)/teams/(?P<team_uuid>.+)/?', accounts.TeamsHandler), (r'/orgs/(?P<account>.+)/teams/?', accounts.TeamsHandler), # ORGs members (r'/orgs/(?P<account>.+)/members/page/(?P<page_num>\d+)/?', accounts.MembersHandler), (r'/orgs/(?P<account>.+)/members/(?P<user>.+)/?', accounts.MembersHandler), (r'/orgs/(?P<account>.+)/members/?', accounts.MembersHandler), # Organizations of Random Generality. (r'/orgs/?', accounts.OrgsHandler), (r'/orgs/(?P<account>.+)/?', accounts.OrgsHandler), # Users records (r'/users/(?P<account>.+)/records/?', accounts.RecordsHandler), (r'/users/(?P<account>.+)/records/page/(?P<page_num>\d+)/?', accounts.RecordsHandler), # Users billing routes (r'/users/(?P<account>.+)/routes/?', accounts.RoutesHandler), # Users (r'/users/?', accounts.UsersHandler), (r'/users/(?P<account>.+)/?', accounts.UsersHandler), # Records (r'/records/start/(?P<start>.*)/end/(?P<end>.*)/?', records.Handler ), (r'/records/start/(?P<start>.*)/?', records.Handler), (r'/records/end/(?P<end>.*)/?', records.Handler), (r'/records/page/(?P<page_num>\d+)/?', records.Handler), # Public records (r'/records/public/?', records.PublicHandler), (r'/records/public/page/(?P<page_num>\d+)/?', records.PublicHandler), # Unassigned records (r'/records/unassigned/?', records.UnassignedHandler), (r'/records/unassigned/page/(?P<page_num>\d+)/?', records.UnassignedHandler), # Records summary # (r'/records/summary/<lapse>/<value>/?', records.SummaryHandler), # Return last (n) of lapse # (r'/records/summary/<lapse>/lasts/(?P<int>\d+)/?', records.SummaryHandler), # Statistical projection based on the previous data. # (r'/records/summary/<lapse>/nexts/(?P<int>\d+)/?', records.SummaryHandler), # Records summary (r'/records/summary/start/(?P<start>.*)/end/(?P<end>.*)/?', records.SummaryHandler), (r'/records/summary/start/(?P<start>.*)/?', records.SummaryHandler), (r'/records/summary/end/(?P<end>.*)/?', records.SummaryHandler), (r'/records/summary/(?P<lapse>.*)/start/(?P<start>.*)/end/(?P<end>.*)/?', records.SummaryHandler), (r'/records/summary/(?P<lapse>.*)/start/(?P<start>.*)/?', records.SummaryHandler), (r'/records/summary/(?P<lapse>.*)/end/(?P<end>.*)/?', records.SummaryHandler), # Return last (n) of lapse # (r'/records/summary/(?P<lapse>.*)/lasts/(?P<int>\d+)/?', records.SummaryHandler), (r'/records/summary/(?P<lapse>.*)/?', records.SummaryHandler), (r'/records/summary/?', records.SummaryHandler), # Records summaries (r'/records/summaries/start/(?P<start>.*)/end/(?P<end>.*)/?', records.SummariesHandler), (r'/records/summaries/start/(?P<start>.*)/?', records.SummariesHandler), (r'/records/summaries/end/(?P<end>.*)/?', records.SummariesHandler), (r'/records/summaries/(?P<lapse>.*)/start/(?P<start>.*)/end/(?P<end>.*)/?', records.SummariesHandler), (r'/records/summaries/(?P<lapse>.*)/start/(?P<start>.*)/?', records.SummariesHandler), (r'/records/summaries/(?P<lapse>.*)/end/(?P<end>.*)/?', records.SummariesHandler), (r'/records/summaries/(?P<lapse>.*)/?', records.SummariesHandler), (r'/records/summaries/?', records.SummariesHandler), # Records (r'/records/(?P<record_uuid>.+)/?', records.Handler), (r'/records/?', records.Handler), # Records (r'/tasks/(?P<task_uuid>.+)/?', tasks.Handler), (r'/tasks/?', tasks.Handler), # Billings (r'/billings/(?P<billing_uuid>.+)/?', billings.RecordsHandler), (r'/billings/?', billings.RecordsHandler), # Billings records (r'/billings/records/start/(?P<start>.*)/end/(?P<end>.*)/?', billings.RecordsHandler), (r'/billings/records/start/(?P<start>.*)/?', billings.RecordsHandler), (r'/billings/records/end/(?P<end>.*)/?', billings.RecordsHandler), (r'/billings/records/?', billings.RecordsHandler) ], # system database db=db, # system cache cache=cache, # cache enabled flag cache_enabled=cache_enabled, # document datastorage document=document, # kvalue datastorage kvalue=kvalue, # sql datastorage sql=sql, # debug mode debug=opts.debug, # application domain domain=opts.domain, # application timezone timezone=opts.timezone, # pagination page size page_size=opts.page_size, # cookie settings cookie_secret=opts.cookie_secret, # login url login_url='/login/') # Mango periodic cast callbacks periodic_records = PeriodicCast(periodic_get_records, 5000) periodic_records.start() # Setting up mango processor application.listen(opts.port) logging.info('Listening on http://%s:%s' % (opts.host, opts.port)) ioloop.IOLoop.instance().start()
def setUp(self): super(TornadoSessionIntegrationTests, self).setUp() self.session = queries.TornadoSession(self.pg_uri, pool_max_size=10, io_loop=self.io_loop)
def setUp(self): super(TornadoSessionIntegrationTests, self).setUp() self.session = queries.TornadoSession(queries.uri( 'localhost', 5432, 'postgres', 'postgres'), pool_max_size=10, io_loop=self.io_loop)
def initialize(self): self.session = queries.TornadoSession('postgresql://*****:*****@localhost:5432/tornado_poc')
def __init__(self): settings = dict(notebook_dir="test_submissions") handlers = [(r"/submit", start.SubmissionHandler)] Application.__init__(self, handlers, **settings) self.db = queries.TornadoSession(queries.uri(**dsn))
result = yield self.application.session.query(self.SQL) except queries.OperationalError as error: logging.error('Error connecting to the database: %s', error) raise web.HTTPError(503) rows = [] for row in result.items(): row = dict([ (k, v.isoformat() if isinstance(v, datetime.datetime) else v) for k, v in row.items() ]) rows.append(row) result.free() self.finish({'pg_stat_activity': rows}) class ReportHandler(web.RequestHandler): @gen.coroutine def get(self): self.finish(pool.PoolManager.report()) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) application = web.Application([(r'/', ExampleHandler), (r'/report', ReportHandler)], debug=True) application.session = queries.TornadoSession() application.listen(8000) ioloop.IOLoop.instance().start()