def main():
file_io.set_logging_format()
logging.getLogger().setLevel(logging.INFO)
zk_ips = appscale_info.get_zk_node_ips()
zk_client = KazooClient(hosts=','.join(zk_ips))
zk_client.start()
deployment_config = DeploymentConfig(zk_client)
projects_manager = GlobalProjectsManager(zk_client)
thread_pool = ThreadPoolExecutor(MAX_BACKGROUND_WORKERS)
source_manager = SourceManager(zk_client, thread_pool)
source_manager.configure_automatic_fetch(projects_manager)
monit_operator = MonitOperator()
options.define('private_ip', appscale_info.get_private_ip())
options.define('syslog_server', appscale_info.get_headnode_ip())
options.define('db_proxy', appscale_info.get_db_proxy())
options.define('load_balancer_ip', appscale_info.get_load_balancer_ips()[0])
options.define('tq_proxy', appscale_info.get_tq_proxy())
options.define('secret', appscale_info.get_secret())
routing_client = RoutingClient(zk_client, options.private_ip, options.secret)
instance_manager = InstanceManager(
zk_client, monit_operator, routing_client, projects_manager,
deployment_config, source_manager, options.syslog_server, thread_pool,
options.private_ip)
instance_manager.start()
logger.info('Starting AppManager')
io_loop = IOLoop.current()
io_loop.run_sync(instance_manager.populate_api_servers)
io_loop.start()
def main():
file_io.set_logging_format()
logging.getLogger().setLevel(logging.INFO)
zk_ips = appscale_info.get_zk_node_ips()
zk_client = KazooClient(hosts=','.join(zk_ips))
zk_client.start()
deployment_config = DeploymentConfig(zk_client)
projects_manager = GlobalProjectsManager(zk_client)
thread_pool = ThreadPoolExecutor(MAX_BACKGROUND_WORKERS)
source_manager = SourceManager(zk_client, thread_pool)
source_manager.configure_automatic_fetch(projects_manager)
monit_operator = MonitOperator()
options.define('private_ip', appscale_info.get_private_ip())
options.define('syslog_server', appscale_info.get_headnode_ip())
options.define('db_proxy', appscale_info.get_db_proxy())
options.define('tq_proxy', appscale_info.get_tq_proxy())
options.define('secret', appscale_info.get_secret())
routing_client = RoutingClient(zk_client, options.private_ip,
options.secret)
instance_manager = InstanceManager(zk_client, monit_operator,
routing_client, projects_manager,
deployment_config, source_manager,
options.syslog_server, thread_pool,
options.private_ip)
instance_manager.start()
logger.info('Starting AppManager')
io_loop = IOLoop.current()
io_loop.run_sync(instance_manager.populate_api_servers)
io_loop.start()
def main():
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger = logging.getLogger(__name__)
zookeeper_locations = appscale_info.get_zk_locations_string()
gc_zookeeper = zk.ZKTransaction(host=zookeeper_locations)
logger.info("Using ZK locations {0}".format(zookeeper_locations))
datastore_location = ':'.join([appscale_info.get_db_proxy(),
str(DB_SERVER_PORT)])
ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra",
datastore_location)
try:
ds_groomer.start()
except Exception, exception:
logger.warning("An exception slipped through:")
logger.exception(exception)
logger.warning("Exiting service.")
def main():
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger = logging.getLogger(__name__)
zookeeper_locations = appscale_info.get_zk_locations_string()
gc_zookeeper = zk.ZKTransaction(host=zookeeper_locations)
logger.info("Using ZK locations {0}".format(zookeeper_locations))
datastore_location = ':'.join(
[appscale_info.get_db_proxy(),
str(DB_SERVER_PORT)])
ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra",
datastore_location)
try:
ds_groomer.start()
except Exception, exception:
logger.warning("An exception slipped through:")
logger.exception(exception)
logger.warning("Exiting service.")
def __init__(self, db_access, zk_client):
""" DistributedTaskQueue Constructor.
Args:
db_access: A DatastoreProxy object.
zk_client: A KazooClient.
"""
setup_env()
db_proxy = appscale_info.get_db_proxy()
connection_str = '{}:{}'.format(db_proxy,
str(constants.DB_SERVER_PORT))
ds_distrib = datastore_distributed.DatastoreDistributed(
constants.DASHBOARD_APP_ID, connection_str, require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID
self.db_access = db_access
self.queue_manager = GlobalQueueManager(zk_client, db_access)
def create_java_start_cmd(app_name, port, load_balancer_host, max_heap,
pidfile):
""" Creates the start command to run the java application server.
Args:
app_name: The name of the application to run
port: The local port the application server will bind to
load_balancer_host: The host of the load balancer
max_heap: An integer specifying the max heap size in MB.
pidfile: A string specifying the pidfile location.
Returns:
A string of the start command.
"""
db_proxy = appscale_info.get_db_proxy()
tq_proxy = appscale_info.get_tq_proxy()
java_start_script = os.path.join(constants.JAVA_APPSERVER,
'appengine-java-sdk-repacked', 'bin',
'dev_appserver.sh')
# The Java AppServer needs the NGINX_PORT flag set so that it will read the
# local FS and see what port it's running on. The value doesn't matter.
cmd = [
java_start_script,
"--port=" + str(port),
#this jvm flag allows javax.email to connect to the smtp server
"--jvm_flag=-Dsocket.permit_connect=true",
'--jvm_flag=-Xmx{}m'.format(max_heap),
'--jvm_flag=-Djava.security.egd=file:/dev/./urandom',
"--disable_update_check",
"--address=" + appscale_info.get_private_ip(),
"--datastore_path=" + db_proxy,
"--login_server=" + load_balancer_host,
"--appscale_version=1",
"--APP_NAME=" + app_name,
"--NGINX_ADDRESS=" + load_balancer_host,
"--TQ_PROXY=" + tq_proxy,
"--pidfile={}".format(pidfile),
os.path.dirname(
locate_dir("/var/apps/" + app_name + "/app/", "WEB-INF"))
]
return ' '.join(cmd)
def __init__(self, db_access, zk_client):
""" DistributedTaskQueue Constructor.
Args:
db_access: A DatastoreProxy object.
zk_client: A KazooClient.
"""
setup_env()
db_proxy = appscale_info.get_db_proxy()
connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT))
ds_distrib = datastore_distributed.DatastoreDistributed(
constants.DASHBOARD_APP_ID, connection_str, require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID
self.db_access = db_access
self.load_balancers = appscale_info.get_load_balancer_ips()
self.queue_manager = GlobalQueueManager(zk_client, db_access)
self.service_manager = GlobalServiceManager(zk_client)
def create_python27_start_cmd(app_name, login_ip, port, pidfile):
""" Creates the start command to run the python application server.
Args:
app_name: The name of the application to run
login_ip: The public IP of this deployment
port: The local port the application server will bind to
pidfile: A string specifying the pidfile location.
Returns:
A string of the start command.
"""
db_proxy = appscale_info.get_db_proxy()
cmd = [
"/usr/bin/python2",
constants.APPSCALE_HOME + "/AppServer/dev_appserver.py",
"--port " + str(port),
"--admin_port " + str(port + 10000),
"--login_server " + login_ip,
"--skip_sdk_update_check",
"--nginx_host " + str(login_ip),
"--require_indexes",
"--enable_sendmail",
"--xmpp_path " + login_ip,
"--php_executable_path=" + str(PHP_CGI_LOCATION),
"--uaserver_path " + db_proxy + ":"\
+ str(constants.UA_SERVER_PORT),
"--datastore_path " + db_proxy + ":"\
+ str(constants.DB_SERVER_PORT),
"/var/apps/" + app_name + "/app",
"--host " + appscale_info.get_private_ip(),
"--admin_host " + appscale_info.get_private_ip(),
"--automatic_restart", "no",
"--pidfile", pidfile]
if app_name in TRUSTED_APPS:
cmd.extend([TRUSTED_FLAG])
return ' '.join(cmd)
def main():
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger = logging.getLogger(__name__)
zookeeper_locations = appscale_info.get_zk_locations_string()
retry_policy = KazooRetry(max_tries=5)
zk_client = KazooClient(zookeeper_locations,
connection_retry=ZK_PERSISTENT_RECONNECTS,
command_retry=retry_policy)
zk_client.start()
gc_zookeeper = zk.ZKTransaction(zk_client)
logger.info("Using ZK locations {0}".format(zookeeper_locations))
datastore_location = ':'.join(
[appscale_info.get_db_proxy(),
str(DB_SERVER_PORT)])
ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra",
datastore_location)
try:
ds_groomer.start()
except Exception, exception:
logger.warning("An exception slipped through:")
logger.exception(exception)
logger.warning("Exiting service.")
from google.appengine.api import datastore_distributed
from google.appengine.ext import db
app_id = os.environ['APP_ID']
remote_host = os.environ['HOST']
with open(get_celery_configuration_path(app_id)) as config_file:
rates = json.load(config_file)
celery = create_celery_for_app(app_id, rates)
logger = get_task_logger(__name__)
logger.setLevel(logging.INFO)
db_proxy = appscale_info.get_db_proxy()
connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT))
ds_distrib = datastore_distributed.DatastoreDistributed(
'appscaledashboard', connection_str, require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = 'appscaledashboard'
def get_wait_time(retries, args):
""" Calculates how long we should wait to execute a failed task, based on
how many times it's failed in the past.
Args:
retries: An int that indicates how many times this task has failed.
args: A dict that contains information about when the user wants to retry
the failed task.
file_io.set_logging_format()
logging.getLogger().setLevel(logging.INFO)
zk_ips = appscale_info.get_zk_node_ips()
zk_client = KazooClient(hosts=','.join(zk_ips))
zk_client.start()
deployment_config = DeploymentConfig(zk_client)
projects_manager = GlobalProjectsManager(zk_client)
thread_pool = ThreadPoolExecutor(MAX_BACKGROUND_WORKERS)
source_manager = SourceManager(zk_client, thread_pool)
source_manager.configure_automatic_fetch(projects_manager)
options.define('private_ip', appscale_info.get_private_ip())
options.define('syslog_server', appscale_info.get_headnode_ip())
options.define('db_proxy', appscale_info.get_db_proxy())
options.define('tq_proxy', appscale_info.get_tq_proxy())
options.define('secret', appscale_info.get_secret())
running_instances = recover_state(zk_client)
PeriodicCallback(stop_failed_instances,
INSTANCE_CLEANUP_INTERVAL * 1000).start()
app = tornado.web.Application([('/versions/([a-z0-9-_]+)', VersionHandler),
('/versions/([a-z0-9-_]+)/([0-9-]+)',
InstanceHandler)])
app.listen(constants.APP_MANAGER_PORT)
logging.info('Starting AppManager on {}'.format(
constants.APP_MANAGER_PORT))
def test_get_tq_proxy(self, read_mock):
read_mock.return_value = '192.168.0.1\n129.168.0.2\n184.48.65.89'
self.assertEquals(appscale_info.get_db_proxy(), '192.168.0.1')
read_mock.assert_called_once_with('/etc/appscale/load_balancer_ips')
def test_get_tq_proxy(self):
flexmock(file_io).should_receive("read").\
and_return("192.168.0.1\n129.168.0.2\n184.48.65.89")
self.assertEquals("192.168.0.1", appscale_info.get_db_proxy())
################################
if __name__ == "__main__":
file_io.set_logging_format()
logging.getLogger().setLevel(logging.INFO)
zk_ips = appscale_info.get_zk_node_ips()
zk_client = KazooClient(hosts=','.join(zk_ips))
zk_client.start()
deployment_config = DeploymentConfig(zk_client)
projects_manager = GlobalProjectsManager(zk_client)
thread_pool = ThreadPoolExecutor(MAX_BACKGROUND_WORKERS)
source_manager = SourceManager(zk_client, thread_pool)
options.define('private_ip', appscale_info.get_private_ip())
options.define('syslog_server', appscale_info.get_headnode_ip())
options.define('db_proxy', appscale_info.get_db_proxy())
options.define('tq_proxy', appscale_info.get_tq_proxy())
app = tornado.web.Application([
('/versions/([a-z0-9-_]+)', VersionHandler),
('/versions/([a-z0-9-_]+)/([0-9-]+)', InstanceHandler)
])
app.listen(constants.APP_MANAGER_PORT)
logging.info('Starting AppManager on {}'.format(constants.APP_MANAGER_PORT))
io_loop = IOLoop.current()
io_loop.run_sync(populate_api_servers)
io_loop.start()
def test_get_tq_proxy(self):
flexmock(file_io).should_receive("read").\
and_return("192.168.0.1\n129.168.0.2\n184.48.65.89")
self.assertEquals("192.168.0.1", appscale_info.get_db_proxy())
def test_get_tq_proxy(self, read_mock):
read_mock.return_value = '192.168.0.1\n129.168.0.2\n184.48.65.89'
self.assertEquals(appscale_info.get_db_proxy(), '192.168.0.1')
read_mock.assert_called_once_with('/etc/appscale/load_balancer_ips')
from google.appengine.api import datastore_distributed
from google.appengine.ext import db
app_id = os.environ['APP_ID']
remote_host = os.environ['HOST']
with open(get_celery_configuration_path(app_id)) as config_file:
rates = json.load(config_file)
celery = create_celery_for_app(app_id, rates)
logger = get_task_logger(__name__)
logger.setLevel(logging.INFO)
db_proxy = appscale_info.get_db_proxy()
connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT))
ds_distrib = datastore_distributed.DatastoreDistributed(
'appscaledashboard', connection_str, require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = 'appscaledashboard'
def get_wait_time(retries, args):
""" Calculates how long we should wait to execute a failed task, based on
how many times it's failed in the past.
Args:
retries: An int that indicates how many times this task has failed.
args: A dict that contains information about when the user wants to retry
the failed task.
