def create_celery_worker_scripts(self):
""" Creates the task worker python script. It uses a configuration file
for setup.
Returns:
The full path of the worker script.
"""
header_template = file_io.read(self.HEADER_LOC)
task_template = file_io.read(self.TASK_LOC)
header_template = header_template.replace("APP_ID", self._app_id)
script = header_template.replace("CELERY_CONFIGURATION", self._app_id) + \
'\n'
for name, queue in self.queues.iteritems():
# Celery only handles push queues.
if not isinstance(queue, PushQueue):
continue
# The queue name is used as a function name so replace invalid chars
queue_name = queue.name.replace('-', '_')
new_task = task_template.\
replace("QUEUE_NAME", self.get_queue_function_name(queue_name))
# For tasks generated by mapreduce, or destined to be run by a module,
# the hostname may have a prefix that corresponds to a different
# subdomain.
# AppScale does not support that type of routing currently, so the main
# loadbalancer IP/hostname is used here for the execution of a task.
new_task = new_task.\
replace("PUBLIC_IP", "\"{}\"".format(self.get_public_ip()))
script += new_task + '\n'
worker_file = self.get_celery_worker_script_path(self._app_id)
file_io.write(worker_file, script)
return worker_file
def create_celery_file(self):
""" Creates the Celery configuration file describing queues and exchanges
for an application. Uses the queue.yaml/queue.xml input.
Returns:
A string representing the full path location of the
configuration file.
"""
celery_queues = []
annotations = []
for name, queue in self.queues.iteritems():
# Celery only handles push queues.
if not isinstance(queue, PushQueue):
continue
celery_name = TaskQueueConfig.get_celery_queue_name(
self._app_id, queue.name)
queue_str = "Queue('{name}', Exchange('{app}'), routing_key='{key}'),"\
.format(name=celery_name, app=self._app_id, key=celery_name)
celery_queues.append(queue_str)
annotation_name = TaskQueueConfig.get_celery_annotation_name(
self._app_id, queue.name)
annotation = "'{name}': {{'rate_limit': '{rate}'}},".format(
name=annotation_name, rate=queue.rate)
annotations.append(annotation)
config = """
from kombu import Exchange
from kombu import Queue
CELERY_QUEUES = (
{queues}
)
CELERY_ANNOTATIONS = {{
{annotations}
}}
# Everytime a task is enqueued a temporary queue is created to store
# results into rabbitmq. This can be bad in a high enqueue environment
# We use the following to make sure these temp queues are not created.
# See http://stackoverflow.com/questions/7144025/temporary-queue-made-in-celery
# for more information on this issue.
CELERY_IGNORE_RESULT = True
CELERY_STORE_ERRORS_EVEN_IF_IGNORED = False
# One month expiration date because a task can be deferred that long.
CELERY_AMQP_TASK_RESULT_EXPIRES = 2678400
# Disable prefetching for celery workers. If tasks are small in duration this
# should be set to a higher value (64-128) for increased performance.
# See: http://celery.readthedocs.org/en/latest/userguide/optimizing.html#worker-settings
CELERYD_PREFETCH_MULTIPLIER = 1
""".format(queues='\n'.join(celery_queues), annotations='\n'.join(annotations))
config_file = self._app_id + ".py"
file_io.write(self.CELERY_CONFIG_DIR + config_file, config)
return self.CELERY_CONFIG_DIR + config_file
def update_worker(self, queue_config):
""" Updates a worker's configuration and restarts it.
Args:
queue_config: A JSON string specifying queue configuration.
"""
self._write_worker_configuration(queue_config)
# Start the worker if it doesn't exist. Restart it if it does.
logger.info('(Re)starting push worker for {}'.format(self.project_id))
file_io.write('/run/appscale/appscale-celery.env',
'HOST={}\n'.format(options.load_balancers[0]))
yield self.service_operator.restart_async(self.service_name)
yield self.service_operator.start_async(self.service_name)
def create_test_yaml():
file_loc = FILE_LOC
config = \
"""
queue:
- name: default
rate: 5/s
- name: foo
rate: 10/m
"""
try:
os.mkdir("/var/apps/test_app")
os.mkdir("/var/apps/test_app/app/")
except OSError:
pass
file_io.write(file_loc, config)
def create_test_yaml():
file_loc = FILE_LOC
config = \
"""
queue:
- name: default
rate: 5/s
- name: foo
rate: 10/m
"""
try:
os.mkdir("/var/apps/test_app")
os.mkdir("/var/apps/test_app/app/")
except OSError:
pass
file_io.write(file_loc, config)
def create_test_yaml(): file_loc = FILE_LOC config = \ """ queue: - name: default rate: 5/s - name: foo rate: 10/m """ FILE = file_io.write(config, file_loc)
def create_test_yaml():
file_loc = FILE_LOC
config = \
"""
queue:
- name: default
rate: 5/s
- name: foo
rate: 10/m
"""
FILE = file_io.write(config, file_loc)
def _ensure_api_server(self, project_id, runtime):
""" Make sure there is a running API server for a project.
Args:
project_id: A string specifying the project ID.
runtime: The runtime for the project
Returns:
An integer specifying the API server port and list of api services.
"""
ensure_app_server_api = runtime == JAVA8
if project_id in self._api_servers:
api_server_ports = self._api_servers[project_id]
if not ensure_app_server_api:
raise gen.Return((api_server_ports[0], [
'appscale-api-server@{}-{}'.format(
project_id, str(api_server_ports[0]))
]))
elif len(api_server_ports) > 1:
raise gen.Return((api_server_ports[1], [
'appscale-api-server@{}-{}'.format(
project_id, str(api_server_ports[0])),
'appscale-api-server@1_{}-{}'.format(
project_id, str(api_server_ports[1]))
]))
server_port = MAX_API_SERVER_PORT
for ports in self._api_servers.values():
for port in ports:
if port <= server_port:
server_port = port - 1
api_services = []
if not project_id in self._api_servers:
watch = ''.join([API_SERVER_PREFIX, project_id])
zk_locations = appscale_info.get_zk_node_ips()
start_cmd = ' '.join([
API_SERVER_LOCATION, '--port',
str(server_port), '--project-id', project_id,
'--zookeeper-locations', ' '.join(zk_locations)
])
api_command_file_path = (
'/run/appscale/apps/api_command_{}-{}'.format(
project_id, str(server_port)))
api_command_content = 'exec {}'.format(start_cmd)
file_io.write(api_command_file_path, api_command_content)
api_server_port = server_port
else:
api_server_port = self._api_servers[project_id][0]
api_services.append('appscale-api-server@{}-{}'.format(
project_id, str(api_server_port)))
if ensure_app_server_api:
# Start an Python 27 runtime API server
if api_server_port == server_port:
server_port -= 1
start_cmd = create_python_api_start_cmd(project_id,
self._login_server,
server_port,
api_server_port)
api_command_file_path = (
'/run/appscale/apps/api_command_1_{}-{}'.format(
project_id, str(server_port)))
api_command_content = 'exec {}'.format(start_cmd)
file_io.write(api_command_file_path, api_command_content)
api_services.append('appscale-api-server@{}-{}'.format(
project_id, str(server_port)))
self._api_servers[project_id] = [api_server_port, server_port]
else:
self._api_servers[project_id] = [server_port]
raise gen.Return((server_port, api_services))
def _start_instance(self, version, port):
""" Starts a Google App Engine application on this machine. It
will start it up and then proceed to fetch the main page.
Args:
version: A Version object.
port: An integer specifying a port to use.
"""
version_details = version.version_details
runtime = version_details['runtime']
env_vars = version_details.get('envVariables', {})
runtime_params = self._deployment_config.get_config(
'runtime_parameters')
max_memory = runtime_params.get('default_max_appserver_memory',
DEFAULT_MAX_APPSERVER_MEMORY)
if 'instanceClass' in version_details:
max_memory = INSTANCE_CLASSES.get(version_details['instanceClass'],
max_memory)
source_archive = version_details['deployment']['zip']['sourceUrl']
http_port = version_details['appscaleExtensions']['httpPort']
api_server_port, api_services = yield self._ensure_api_server(
version.project_id, runtime)
yield self._source_manager.ensure_source(version.revision_key,
source_archive, runtime)
logger.info('Starting {}:{}'.format(version, port))
pidfile = PIDFILE_TEMPLATE.format(revision=version.revision_key,
port=port)
if runtime == GO:
env_vars['GOPATH'] = os.path.join(UNPACK_ROOT,
version.revision_key, 'gopath')
env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot')
if runtime in (PYTHON27, GO, PHP):
start_cmd = create_python27_start_cmd(version.project_id,
self._login_server, port,
pidfile,
version.revision_key,
api_server_port)
env_vars.update(
create_python_app_env(self._login_server, version.project_id))
elif runtime in (JAVA, JAVA8):
# Account for MaxPermSize (~170MB), the parent process (~50MB), and thread
# stacks (~20MB).
max_heap = max_memory - 250
if max_heap <= 0:
raise BadConfigurationException(
'Memory for Java applications must be greater than 250MB')
start_cmd = create_java_start_cmd(version.project_id, port,
http_port, self._login_server,
max_heap, pidfile,
version.revision_key,
api_server_port, runtime)
env_vars.update(
create_java_app_env(self._deployment_config, runtime,
version.project_id))
else:
raise BadConfigurationException('Unknown runtime {} for {}'.format(
runtime, version.project_id))
logger.info("Start command: " + str(start_cmd))
logger.info("Environment variables: " + str(env_vars))
env_content = ' '.join(
['{}="{}"'.format(k, str(v)) for k, v in env_vars.items()])
command_content = 'exec env {} {}'.format(env_content, start_cmd)
service_inst = '{}-{}'.format(version.revision_key, port)
service_name = 'appscale-instance-run@{}'.format(service_inst)
service_props = {'MemoryLimit': '{}M'.format(max_memory)}
command_file_path = '/run/appscale/apps/command_{}'.format(
service_inst)
file_io.write(command_file_path, command_content)
yield self._service_operator.start_async(service_name,
wants=api_services,
properties=service_props)
# Make sure the version registration node exists.
self._zk_client.ensure_path('/'.join(
[VERSION_REGISTRATION_NODE, version.version_key]))
instance = Instance(version.revision_key, port)
yield self._add_routing(instance)
if version.project_id == DASHBOARD_PROJECT_ID:
log_size = DASHBOARD_LOG_SIZE
else:
log_size = APP_LOG_SIZE
if not setup_logrotate(version.project_id, log_size):
logger.error(
"Error while setting up log rotation for application: {}".
format(version.project_id))