def main(): file_io.set_logging_format() logging.getLogger().setLevel(logging.INFO) zk_ips = appscale_info.get_zk_node_ips() zk_client = KazooClient(hosts=','.join(zk_ips)) zk_client.start() deployment_config = DeploymentConfig(zk_client) projects_manager = GlobalProjectsManager(zk_client) thread_pool = ThreadPoolExecutor(MAX_BACKGROUND_WORKERS) source_manager = SourceManager(zk_client, thread_pool) source_manager.configure_automatic_fetch(projects_manager) monit_operator = MonitOperator() options.define('private_ip', appscale_info.get_private_ip()) options.define('syslog_server', appscale_info.get_headnode_ip()) options.define('db_proxy', appscale_info.get_db_proxy()) options.define('load_balancer_ip', appscale_info.get_load_balancer_ips()[0]) options.define('tq_proxy', appscale_info.get_tq_proxy()) options.define('secret', appscale_info.get_secret()) routing_client = RoutingClient(zk_client, options.private_ip, options.secret) instance_manager = InstanceManager( zk_client, monit_operator, routing_client, projects_manager, deployment_config, source_manager, options.syslog_server, thread_pool, options.private_ip) instance_manager.start() logger.info('Starting AppManager') io_loop = IOLoop.current() io_loop.run_sync(instance_manager.populate_api_servers) io_loop.start()
def main(): file_io.set_logging_format() logging.getLogger().setLevel(logging.INFO) zk_ips = appscale_info.get_zk_node_ips() zk_client = KazooClient(hosts=','.join(zk_ips)) zk_client.start() deployment_config = DeploymentConfig(zk_client) projects_manager = GlobalProjectsManager(zk_client) thread_pool = ThreadPoolExecutor(MAX_BACKGROUND_WORKERS) source_manager = SourceManager(zk_client, thread_pool) source_manager.configure_automatic_fetch(projects_manager) monit_operator = MonitOperator() options.define('private_ip', appscale_info.get_private_ip()) options.define('syslog_server', appscale_info.get_headnode_ip()) options.define('db_proxy', appscale_info.get_db_proxy()) options.define('tq_proxy', appscale_info.get_tq_proxy()) options.define('secret', appscale_info.get_secret()) routing_client = RoutingClient(zk_client, options.private_ip, options.secret) instance_manager = InstanceManager(zk_client, monit_operator, routing_client, projects_manager, deployment_config, source_manager, options.syslog_server, thread_pool, options.private_ip) instance_manager.start() logger.info('Starting AppManager') io_loop = IOLoop.current() io_loop.run_sync(instance_manager.populate_api_servers) io_loop.start()
def main(): logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger = logging.getLogger(__name__) zookeeper_locations = appscale_info.get_zk_locations_string() gc_zookeeper = zk.ZKTransaction(host=zookeeper_locations) logger.info("Using ZK locations {0}".format(zookeeper_locations)) datastore_location = ':'.join([appscale_info.get_db_proxy(), str(DB_SERVER_PORT)]) ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra", datastore_location) try: ds_groomer.start() except Exception, exception: logger.warning("An exception slipped through:") logger.exception(exception) logger.warning("Exiting service.")
def main(): logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger = logging.getLogger(__name__) zookeeper_locations = appscale_info.get_zk_locations_string() gc_zookeeper = zk.ZKTransaction(host=zookeeper_locations) logger.info("Using ZK locations {0}".format(zookeeper_locations)) datastore_location = ':'.join( [appscale_info.get_db_proxy(), str(DB_SERVER_PORT)]) ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra", datastore_location) try: ds_groomer.start() except Exception, exception: logger.warning("An exception slipped through:") logger.exception(exception) logger.warning("Exiting service.")
def __init__(self, db_access, zk_client): """ DistributedTaskQueue Constructor. Args: db_access: A DatastoreProxy object. zk_client: A KazooClient. """ setup_env() db_proxy = appscale_info.get_db_proxy() connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT)) ds_distrib = datastore_distributed.DatastoreDistributed( constants.DASHBOARD_APP_ID, connection_str, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib) os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID self.db_access = db_access self.queue_manager = GlobalQueueManager(zk_client, db_access)
def create_java_start_cmd(app_name, port, load_balancer_host, max_heap, pidfile): """ Creates the start command to run the java application server. Args: app_name: The name of the application to run port: The local port the application server will bind to load_balancer_host: The host of the load balancer max_heap: An integer specifying the max heap size in MB. pidfile: A string specifying the pidfile location. Returns: A string of the start command. """ db_proxy = appscale_info.get_db_proxy() tq_proxy = appscale_info.get_tq_proxy() java_start_script = os.path.join(constants.JAVA_APPSERVER, 'appengine-java-sdk-repacked', 'bin', 'dev_appserver.sh') # The Java AppServer needs the NGINX_PORT flag set so that it will read the # local FS and see what port it's running on. The value doesn't matter. cmd = [ java_start_script, "--port=" + str(port), #this jvm flag allows javax.email to connect to the smtp server "--jvm_flag=-Dsocket.permit_connect=true", '--jvm_flag=-Xmx{}m'.format(max_heap), '--jvm_flag=-Djava.security.egd=file:/dev/./urandom', "--disable_update_check", "--address=" + appscale_info.get_private_ip(), "--datastore_path=" + db_proxy, "--login_server=" + load_balancer_host, "--appscale_version=1", "--APP_NAME=" + app_name, "--NGINX_ADDRESS=" + load_balancer_host, "--TQ_PROXY=" + tq_proxy, "--pidfile={}".format(pidfile), os.path.dirname( locate_dir("/var/apps/" + app_name + "/app/", "WEB-INF")) ] return ' '.join(cmd)
def __init__(self, db_access, zk_client): """ DistributedTaskQueue Constructor. Args: db_access: A DatastoreProxy object. zk_client: A KazooClient. """ setup_env() db_proxy = appscale_info.get_db_proxy() connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT)) ds_distrib = datastore_distributed.DatastoreDistributed( constants.DASHBOARD_APP_ID, connection_str, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib) os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID self.db_access = db_access self.load_balancers = appscale_info.get_load_balancer_ips() self.queue_manager = GlobalQueueManager(zk_client, db_access) self.service_manager = GlobalServiceManager(zk_client)
def create_python27_start_cmd(app_name, login_ip, port, pidfile): """ Creates the start command to run the python application server. Args: app_name: The name of the application to run login_ip: The public IP of this deployment port: The local port the application server will bind to pidfile: A string specifying the pidfile location. Returns: A string of the start command. """ db_proxy = appscale_info.get_db_proxy() cmd = [ "/usr/bin/python2", constants.APPSCALE_HOME + "/AppServer/dev_appserver.py", "--port " + str(port), "--admin_port " + str(port + 10000), "--login_server " + login_ip, "--skip_sdk_update_check", "--nginx_host " + str(login_ip), "--require_indexes", "--enable_sendmail", "--xmpp_path " + login_ip, "--php_executable_path=" + str(PHP_CGI_LOCATION), "--uaserver_path " + db_proxy + ":"\ + str(constants.UA_SERVER_PORT), "--datastore_path " + db_proxy + ":"\ + str(constants.DB_SERVER_PORT), "/var/apps/" + app_name + "/app", "--host " + appscale_info.get_private_ip(), "--admin_host " + appscale_info.get_private_ip(), "--automatic_restart", "no", "--pidfile", pidfile] if app_name in TRUSTED_APPS: cmd.extend([TRUSTED_FLAG]) return ' '.join(cmd)
def main(): logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger = logging.getLogger(__name__) zookeeper_locations = appscale_info.get_zk_locations_string() retry_policy = KazooRetry(max_tries=5) zk_client = KazooClient(zookeeper_locations, connection_retry=ZK_PERSISTENT_RECONNECTS, command_retry=retry_policy) zk_client.start() gc_zookeeper = zk.ZKTransaction(zk_client) logger.info("Using ZK locations {0}".format(zookeeper_locations)) datastore_location = ':'.join( [appscale_info.get_db_proxy(), str(DB_SERVER_PORT)]) ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra", datastore_location) try: ds_groomer.start() except Exception, exception: logger.warning("An exception slipped through:") logger.exception(exception) logger.warning("Exiting service.")
from google.appengine.api import datastore_distributed from google.appengine.ext import db app_id = os.environ['APP_ID'] remote_host = os.environ['HOST'] with open(get_celery_configuration_path(app_id)) as config_file: rates = json.load(config_file) celery = create_celery_for_app(app_id, rates) logger = get_task_logger(__name__) logger.setLevel(logging.INFO) db_proxy = appscale_info.get_db_proxy() connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT)) ds_distrib = datastore_distributed.DatastoreDistributed( 'appscaledashboard', connection_str, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib) os.environ['APPLICATION_ID'] = 'appscaledashboard' def get_wait_time(retries, args): """ Calculates how long we should wait to execute a failed task, based on how many times it's failed in the past. Args: retries: An int that indicates how many times this task has failed. args: A dict that contains information about when the user wants to retry the failed task.
file_io.set_logging_format() logging.getLogger().setLevel(logging.INFO) zk_ips = appscale_info.get_zk_node_ips() zk_client = KazooClient(hosts=','.join(zk_ips)) zk_client.start() deployment_config = DeploymentConfig(zk_client) projects_manager = GlobalProjectsManager(zk_client) thread_pool = ThreadPoolExecutor(MAX_BACKGROUND_WORKERS) source_manager = SourceManager(zk_client, thread_pool) source_manager.configure_automatic_fetch(projects_manager) options.define('private_ip', appscale_info.get_private_ip()) options.define('syslog_server', appscale_info.get_headnode_ip()) options.define('db_proxy', appscale_info.get_db_proxy()) options.define('tq_proxy', appscale_info.get_tq_proxy()) options.define('secret', appscale_info.get_secret()) running_instances = recover_state(zk_client) PeriodicCallback(stop_failed_instances, INSTANCE_CLEANUP_INTERVAL * 1000).start() app = tornado.web.Application([('/versions/([a-z0-9-_]+)', VersionHandler), ('/versions/([a-z0-9-_]+)/([0-9-]+)', InstanceHandler)]) app.listen(constants.APP_MANAGER_PORT) logging.info('Starting AppManager on {}'.format( constants.APP_MANAGER_PORT))
def test_get_tq_proxy(self, read_mock): read_mock.return_value = '192.168.0.1\n129.168.0.2\n184.48.65.89' self.assertEquals(appscale_info.get_db_proxy(), '192.168.0.1') read_mock.assert_called_once_with('/etc/appscale/load_balancer_ips')
def test_get_tq_proxy(self): flexmock(file_io).should_receive("read").\ and_return("192.168.0.1\n129.168.0.2\n184.48.65.89") self.assertEquals("192.168.0.1", appscale_info.get_db_proxy())
################################ if __name__ == "__main__": file_io.set_logging_format() logging.getLogger().setLevel(logging.INFO) zk_ips = appscale_info.get_zk_node_ips() zk_client = KazooClient(hosts=','.join(zk_ips)) zk_client.start() deployment_config = DeploymentConfig(zk_client) projects_manager = GlobalProjectsManager(zk_client) thread_pool = ThreadPoolExecutor(MAX_BACKGROUND_WORKERS) source_manager = SourceManager(zk_client, thread_pool) options.define('private_ip', appscale_info.get_private_ip()) options.define('syslog_server', appscale_info.get_headnode_ip()) options.define('db_proxy', appscale_info.get_db_proxy()) options.define('tq_proxy', appscale_info.get_tq_proxy()) app = tornado.web.Application([ ('/versions/([a-z0-9-_]+)', VersionHandler), ('/versions/([a-z0-9-_]+)/([0-9-]+)', InstanceHandler) ]) app.listen(constants.APP_MANAGER_PORT) logging.info('Starting AppManager on {}'.format(constants.APP_MANAGER_PORT)) io_loop = IOLoop.current() io_loop.run_sync(populate_api_servers) io_loop.start()