def test_get_indices(self): session = flexmock(default_consistency_level=None) cluster = flexmock(connect=lambda keyspace: session) flexmock(appscale_info).should_receive('get_db_ips') flexmock(Cluster).new_instances(cluster) flexmock(DatastoreProxy).should_receive('range_query').and_return({}) db_batch = DatastoreProxy() self.assertEquals(db_batch.get_indices("appid"), [])
def main(): """ Main function which initializes and starts the tornado server. """ # Parse command line arguments parser = argparse.ArgumentParser(description='A taskqueue API server') parser.add_argument('--port', '-p', default='17447', help='TaskQueue server port') parser.add_argument('--verbose', action='store_true', help='Output debug-level logging') args = parser.parse_args() if args.verbose: logging.getLogger('appscale').setLevel(logging.DEBUG) # Configure zookeeper and db access zk_client = KazooClient(hosts=','.join(appscale_info.get_zk_node_ips()), connection_retry=ZK_PERSISTENT_RECONNECTS) zk_client.start() db_access = DatastoreProxy() # Initialize tornado server task_queue = distributed_tq.DistributedTaskQueue(db_access, zk_client) tq_application = prepare_taskqueue_application(task_queue) # Automatically decompress incoming requests. server = httpserver.HTTPServer(tq_application, decompress_request=True) server.listen(args.port) # Make sure taskqueue shuts down gracefully when signal is received graceful_shutdown = prepare_graceful_shutdown(zk_client, server) signal.signal(signal.SIGTERM, graceful_shutdown) signal.signal(signal.SIGINT, graceful_shutdown) logger.info('Starting TaskQueue server on port {}'.format(args.port)) ioloop.IOLoop.current().start()
def main(): """ Main function which initializes and starts the tornado server. """ parser = argparse.ArgumentParser(description='A taskqueue API server') parser.add_argument('--port', '-p', default='17447', help='TaskQueue server port') parser.add_argument('--verbose', action='store_true', help='Output debug-level logging') args = parser.parse_args() if args.verbose: logger.setLevel(logging.DEBUG) global task_queue db_access = DatastoreProxy() task_queue = distributed_tq.DistributedTaskQueue(db_access) handlers = [ # Takes JSON requests from AppController. (r"/startworker", StartWorkerHandler), (r"/stopworker", StopWorkerHandler), (r"/reloadworker", ReloadWorkerHandler), # Takes protocol buffers from the AppServers. (r"/*", MainHandler) ] # Provides compatibility with the v1beta2 REST API. handlers.extend([(RESTQueue.PATH, RESTQueue, { 'queue_handler': task_queue }), (RESTTasks.PATH, RESTTasks, { 'queue_handler': task_queue }), (RESTLease.PATH, RESTLease, { 'queue_handler': task_queue }), (RESTTask.PATH, RESTTask, { 'queue_handler': task_queue })]) tq_application = tornado.web.Application(handlers) server = tornado.httpserver.HTTPServer( tq_application, decompress_request=True) # Automatically decompress incoming requests. server.listen(args.port) while 1: try: logger.info('Starting TaskQueue server on port {}'.format( args.port)) tornado.ioloop.IOLoop.instance().start() except KeyboardInterrupt: logger.warning('Server interrupted by user, terminating...') sys.exit(1)
def main(): """ Main function which initializes and starts the tornado server. """ parser = argparse.ArgumentParser(description='A taskqueue API server') parser.add_argument('--port', '-p', default='17447', help='TaskQueue server port') parser.add_argument('--verbose', action='store_true', help='Output debug-level logging') args = parser.parse_args() if args.verbose: logger.setLevel(logging.DEBUG) global task_queue zk_client = KazooClient(hosts=','.join(appscale_info.get_zk_node_ips()), connection_retry=ZK_PERSISTENT_RECONNECTS) zk_client.start() db_access = DatastoreProxy() task_queue = distributed_tq.DistributedTaskQueue(db_access, zk_client) handlers = [ # Takes protocol buffers from the AppServers. (r"/*", MainHandler) ] # Provides compatibility with the v1beta2 REST API. handlers.extend([(RESTQueue.PATH, RESTQueue, { 'queue_handler': task_queue }), (RESTTasks.PATH, RESTTasks, { 'queue_handler': task_queue }), (RESTLease.PATH, RESTLease, { 'queue_handler': task_queue }), (RESTTask.PATH, RESTTask, { 'queue_handler': task_queue })]) tq_application = tornado.web.Application(handlers) global server server = tornado.httpserver.HTTPServer( tq_application, decompress_request=True) # Automatically decompress incoming requests. server.listen(args.port) signal.signal(signal.SIGTERM, graceful_shutdown) signal.signal(signal.SIGINT, graceful_shutdown) logger.info('Starting TaskQueue server on port {}'.format(args.port)) IOLoop.current().start()
relevant_ips = set(args.zookeeper) | set(args.database) for ip in relevant_ips: utils.ssh(ip, args.keyname, 'service monit start') start_zookeeper(args.zookeeper, args.keyname) conn = KazooClient(hosts=",".join(args.zookeeper)) conn.start() if not conn.exists(ZK_CASSANDRA_CONFIG): conn.create(ZK_CASSANDRA_CONFIG, json.dumps({"num_tokens": 256}), makepath=True) start_cassandra(args.database, args.db_master, args.keyname, args.zookeeper) datastore_upgrade.wait_for_quorum(args.keyname, args.db_master, len(args.database), args.replication) db_access = DatastoreProxy(hosts=args.database) # Exit early if a data layout upgrade is not needed. if db_access.valid_data_version(): status = { 'status': 'complete', 'message': 'The data layout is valid' } sys.exit() zookeeper = datastore_upgrade.get_zookeeper(args.zookeeper) try: total_entities = datastore_upgrade.estimate_total_entities( db_access.session, args.db_master, args.keyname) except AppScaleDBError: total_entities = None
try: # Ensure monit is running. relevant_ips = set(args.zookeeper) | set(args.database) for ip in relevant_ips: ssh(ip, args.keyname, 'service monit start') start_zookeeper(args.zookeeper, args.keyname) conn = KazooClient(hosts=",".join(args.zookeeper)) conn.start() if not conn.exists(ZK_CASSANDRA_CONFIG): conn.create(ZK_CASSANDRA_CONFIG, json.dumps({"num_tokens":256}), makepath=True) start_cassandra(args.database, args.db_master, args.keyname, args.zookeeper) datastore_upgrade.wait_for_quorum( args.keyname, args.db_master, len(args.database), args.replication) db_access = DatastoreProxy(hosts=args.database) # Exit early if a data layout upgrade is not needed. if db_access.valid_data_version_sync(): status = {'status': 'complete', 'message': 'The data layout is valid'} sys.exit() zookeeper = datastore_upgrade.get_zookeeper(args.zookeeper) try: total_entities = datastore_upgrade.estimate_total_entities( db_access.session, args.db_master, args.keyname) except AppScaleDBError: total_entities = None run_datastore_upgrade(db_access, zookeeper, args.log_postfix, total_entities) status = {'status': 'complete', 'message': 'Data layout upgrade complete'}