def identify_as_hoster(self, project_id, service_id, version): """ Marks this machine as having a version's source code. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version: A dictionary containing version details. """ revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id'], str(version['revision'])]) hoster_node = '/apps/{}/{}'.format(revision_key, options.private_ip) source_location = version['deployment']['zip']['sourceUrl'] md5 = yield self.thread_pool.submit(get_md5, source_location) try: self.zk_client.create(hoster_node, md5, makepath=True) except NodeExistsError: raise CustomHTTPError(HTTPCodes.INTERNAL_ERROR, message='Revision already exists') # Remove old revision nodes. version_prefix = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id']]) old_revisions = [ node for node in self.zk_client.get_children('/apps') if node.startswith(version_prefix) and node < revision_key ] for node in old_revisions: logger.info('Removing hosting entries for {}'.format(node)) self.zk_client.delete('/apps/{}'.format(node), recursive=True)
def stop_hosting_revision(self, project_id, service_id, version): """ Removes a revision and its hosting entry. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version: A dictionary containing version details. """ revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id'], str(version['revision'])]) revision_node = '/apps/{}'.format(revision_key) hoster_node = '/'.join([revision_node, options.private_ip]) try: self.zk_client.delete(hoster_node) except NoNodeError: pass # Clean up revision container since it's probably empty. try: self.zk_client.delete(revision_node) except (NotEmptyError, NoNodeError): pass source_location = version['deployment']['zip']['sourceUrl'] try: os.remove(source_location) except OSError as error: if error.errno != errno.ENOENT: raise
def declare_instance_nodes(self, running_instances): """ Removes dead ZooKeeper instance entries and adds running ones. Args: running_instances: An iterable of Instances. """ registered_instances = set() for version_key in self._zk_client.get_children(VERSION_REGISTRATION_NODE): version_node = '/'.join([VERSION_REGISTRATION_NODE, version_key]) for instance_entry in self._zk_client.get_children(version_node): machine_ip = instance_entry.split(':')[0] if machine_ip != self._private_ip: continue port = int(instance_entry.split(':')[-1]) instance_node = '/'.join([version_node, instance_entry]) revision = self._zk_client.get(instance_node)[0] revision_key = VERSION_PATH_SEPARATOR.join([version_key, revision]) registered_instances.add(Instance(revision_key, port)) # Remove outdated nodes. for instance in registered_instances - running_instances: self.unregister_instance(instance) # Add nodes for running instances. for instance in running_instances - registered_instances: self.register_instance(instance)
def clean_up_revision_nodes(self, project_id, service_id, version): """ Removes old revision nodes. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version: A dictionary containing version details. """ revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id'], str(version['revision'])]) version_prefix = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id']]) old_revisions = [node for node in self.zk_client.get_children('/apps') if node.startswith(version_prefix) and node < revision_key] for node in old_revisions: logger.info('Removing hosting entries for {}'.format(node)) self.zk_client.delete('/apps/{}'.format(node), recursive=True)
def post(self, project_id, service_id): """ Creates or updates a version. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. """ self.authenticate(project_id, self.ua_client) version = self.version_from_payload() version_exists = self.version_exists(project_id, service_id, version['id']) revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id'], str(version['revision'])]) try: yield self.thread_pool.submit( utils.extract_source, revision_key, version['deployment']['zip']['sourceUrl'], version['runtime']) except IOError: message = '{} does not exist'.format( version['deployment']['zip']['sourceUrl']) raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=message) except constants.InvalidSource as error: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=str(error)) new_path = utils.rename_source_archive(project_id, service_id, version) version['deployment']['zip']['sourceUrl'] = new_path yield self.identify_as_hoster(project_id, service_id, version) yield self.thread_pool.submit(self.version_update_lock.acquire) try: version = self.put_version(project_id, service_id, version) except VersionNotChanged as warning: logger.info(str(warning)) self.stop_hosting_revision(project_id, service_id, version) return finally: self.version_update_lock.release() self.clean_up_revision_nodes(project_id, service_id, version) utils.remove_old_archives(project_id, service_id, version) self.begin_deploy(project_id, service_id, version['id']) operation = CreateVersionOperation(project_id, service_id, version) operations[operation.id] = operation pre_wait = REDEPLOY_WAIT if version_exists else 0 logging.debug('Starting operation {} in {}s'.format( operation.id, pre_wait)) IOLoop.current().call_later(pre_wait, wait_for_deploy, operation.id, self.acc) self.write(json_encode(operation.rest_repr()))
def post(self, project_id, service_id): """ Creates or updates a version. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. """ self.authenticate(project_id, self.ua_client) version = self.version_from_payload() version_exists = self.version_exists(project_id, service_id, version['id']) revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id'], str(version['revision'])]) try: yield self.thread_pool.submit( utils.extract_source, revision_key, version['deployment']['zip']['sourceUrl'], version['runtime']) except IOError: message = '{} does not exist'.format( version['deployment']['zip']['sourceUrl']) raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=message) except constants.InvalidSource as error: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=str(error)) new_path = utils.rename_source_archive(project_id, service_id, version) version['deployment']['zip']['sourceUrl'] = new_path yield self.identify_as_hoster(project_id, service_id, version) yield self.thread_pool.submit(self.version_update_lock.acquire) try: version = self.put_version(project_id, service_id, version) except VersionNotChanged as warning: logger.info(str(warning)) self.stop_hosting_revision(project_id, service_id, version) return finally: self.version_update_lock.release() self.clean_up_revision_nodes(project_id, service_id, version) utils.remove_old_archives(project_id, service_id, version) self.begin_deploy(project_id, service_id, version['id']) operation = CreateVersionOperation(project_id, service_id, version) operations[operation.id] = operation pre_wait = REDEPLOY_WAIT if version_exists else 0 logging.debug( 'Starting operation {} in {}s'.format(operation.id, pre_wait)) IOLoop.current().call_later(pre_wait, wait_for_deploy, operation.id, self.acc) self.write(json_encode(operation.rest_repr()))
def revision_key(self): if self.version_details is None: return None try: revision_id = self.version_details['revision'] except KeyError: return None return VERSION_PATH_SEPARATOR.join([ self.project_id, self.service_id, self.version_id, str(revision_id) ])
def remove_old_archives(project_id, service_id, version): """ Cleans up old revision archives. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version: A dictionary containing version details. """ prefix = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id']]) current_name = os.path.basename(version['deployment']['zip']['sourceUrl']) old_sources = [os.path.join(SOURCES_DIRECTORY, archive) for archive in os.listdir(SOURCES_DIRECTORY) if archive.startswith(prefix) and archive < current_name] for archive in old_sources: os.remove(archive)
def rename_source_archive(project_id, service_id, version): """ Renames the given source archive to keep track of it. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version: A dictionary containing version details. Returns: A string specifying the new location of the archive. """ new_filename = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id'], '{}.tar.gz'.format(version['revision'])]) new_location = os.path.join(SOURCES_DIRECTORY, new_filename) os.rename(version['deployment']['zip']['sourceUrl'], new_location) return new_location
def _clean_old_sources(self): """ Removes source code for obsolete revisions. """ monit_entries = yield self._monit_operator.get_entries() active_revisions = { entry[len(MONIT_INSTANCE_PREFIX):].rsplit('-', 1)[0] for entry in monit_entries if entry.startswith(MONIT_INSTANCE_PREFIX)} for project_id, project_manager in self._projects_manager.items(): for service_id, service_manager in project_manager.items(): for version_id, version_manager in service_manager.items(): revision_id = version_manager.version_details['revision'] revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version_id, str(revision_id)]) active_revisions.add(revision_key) self._source_manager.clean_old_revisions(active_revisions=active_revisions)
def begin_deploy(self, project_id, service_id, version_id): """ Triggers the deployment process. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version_id: A string specifying a version ID. Raises: CustomHTTPError if unable to start the deployment process. """ version_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version_id]) try: self.acc.update([version_key]) except AppControllerException as error: message = 'Error while updating version: {}'.format(error) raise CustomHTTPError(HTTPCodes.INTERNAL_ERROR, message=message)
def identify_as_hoster(self, project_id, service_id, version): """ Marks this machine as having a version's source code. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version: A dictionary containing version details. """ revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id'], str(version['revision'])]) hoster_node = '/apps/{}/{}'.format(revision_key, options.private_ip) source_location = version['deployment']['zip']['sourceUrl'] md5 = yield self.thread_pool.submit(get_md5, source_location) try: self.zk_client.create(hoster_node, md5, makepath=True) except NodeExistsError: raise CustomHTTPError( HTTPCodes.INTERNAL_ERROR, message='Revision already exists')
def start_delete_version(self, project_id, service_id, version_id): """ Starts the process of deleting a version by calling stop_version on the AppController and deleting the version node. Returns the version's port that will be closing, the caller should wait for this port to close. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version_id: A string specifying a version ID. Returns: The version's port. """ version = self.get_version(project_id, service_id, version_id) try: http_port = int(version['appscaleExtensions']['httpPort']) except KeyError: raise CustomHTTPError(HTTPCodes.NOT_FOUND, message='Version serving port not found') version_node = constants.VERSION_NODE_TEMPLATE.format( project_id=project_id, service_id=service_id, version_id=version_id) yield self.thread_pool.submit(self.version_update_lock.acquire) try: try: self.zk_client.delete(version_node) except NoNodeError: pass finally: self.version_update_lock.release() version_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version_id]) try: self.acc.stop_version(version_key) except AppControllerException as error: message = 'Error while stopping version: {}'.format(error) raise CustomHTTPError(HTTPCodes.INTERNAL_ERROR, message=message) raise gen.Return(http_port)
def update_version(self, new_version): """ Caches new version details. Args: new_version: A JSON string specifying version details. """ if new_version is not None: self.version_details = json.loads(new_version) # Update port file. http_port = self.version_details['appscaleExtensions']['httpPort'] version_key = VERSION_PATH_SEPARATOR.join( [self.project_id, self.service_id, self.version_id]) port_file_location = os.path.join( CONFIG_DIR, 'port-{}.txt'.format(version_key)) with open(port_file_location, 'w') as port_file: port_file.write(str(http_port)) logger.info('Updated version details: {}'.format(version_key)) if self.callback is not None: self.callback()
def start_delete_version(self, project_id, service_id, version_id): """ Starts the process of deleting a version by calling stop_version on the AppController and deleting the version node. Returns the version's port that will be closing, the caller should wait for this port to close. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. version_id: A string specifying a version ID. Returns: The version's port. """ version = self.get_version(project_id, service_id, version_id) try: http_port = int(version['appscaleExtensions']['httpPort']) except KeyError: raise CustomHTTPError(HTTPCodes.NOT_FOUND, message='Version serving port not found') version_node = constants.VERSION_NODE_TEMPLATE.format( project_id=project_id, service_id=service_id, version_id=version_id) yield self.thread_pool.submit(self.version_update_lock.acquire) try: try: self.zk_client.delete(version_node) except NoNodeError: pass finally: self.version_update_lock.release() version_key = VERSION_PATH_SEPARATOR.join([project_id, service_id, version_id]) try: self.acc.stop_version(version_key) except AppControllerException as error: message = 'Error while stopping version: {}'.format(error) raise CustomHTTPError(HTTPCodes.INTERNAL_ERROR, message=message) raise gen.Return(http_port)
def affects_version(self, version_key): """ Reports whether or not the event affects a given version. Args: version_key: A string specifying the relevant version key. Returns: A boolean specifying whether or not the version is affected. """ project_id, service_id, _ = version_key.split(VERSION_PATH_SEPARATOR) if self.type in (self.PROJECT_CREATED, self.PROJECT_DELETED): return self.resource == project_id if self.type in (self.SERVICE_CREATED, self.SERVICE_DELETED): service_key = VERSION_PATH_SEPARATOR.join([project_id, service_id]) return self.resource == service_key if self.type in (self.VERSION_CREATED, self.VERSION_DELETED, self.VERSION_UPDATED): return self.resource == version_key return False
def xmpp_message(self, _, event): """Responds to the receipt of an XMPP message, by finding an App Server that hosts the given application and POSTing the message's payload to it. Args: _: The connection that the message was received on (not used). event: The actual message that was received. """ logging.info("received a message from {0}, with body {1}" \ .format(event.getFrom().getStripped(), event.getBody())) logging.info("message type is {0}".format(event.getType)) from_jid = event.getFrom().getStripped() params = {} params['from'] = from_jid params['to'] = self.my_jid params['body'] = event.getBody() encoded_params = urllib.urlencode(params) version_key = VERSION_PATH_SEPARATOR.join([self.appid, DEFAULT_SERVICE, DEFAULT_VERSION]) port_file_location = os.path.join( '/', 'etc', 'appscale', 'port-{}.txt'.format(version_key)) with open(port_file_location) as port_file: app_port = int(port_file.read().strip()) try: logging.debug("Attempting to open connection to {0}:{1}".format( self.login_ip, app_port)) connection = httplib.HTTPConnection(self.login_ip, app_port) connection.request('POST', '/_ah/xmpp/message/chat/', encoded_params, self.HEADERS) response = connection.getresponse() logging.info("POST XMPP message returned status of {0}".format( response.status)) connection.close() except Exception as e: logging.exception(e)
def xmpp_message(self, _, event): """Responds to the receipt of an XMPP message, by finding an App Server that hosts the given application and POSTing the message's payload to it. Args: _: The connection that the message was received on (not used). event: The actual message that was received. """ logging.info("received a message from {0}, with body {1}" \ .format(event.getFrom().getStripped(), event.getBody())) logging.info("message type is {0}".format(event.getType)) from_jid = event.getFrom().getStripped() params = {} params['from'] = from_jid params['to'] = self.my_jid params['body'] = event.getBody() encoded_params = urllib.urlencode(params) version_key = VERSION_PATH_SEPARATOR.join( [self.appid, DEFAULT_SERVICE, DEFAULT_VERSION]) port_file_location = os.path.join('/', 'etc', 'appscale', 'port-{}.txt'.format(version_key)) with open(port_file_location) as port_file: app_port = int(port_file.read().strip()) try: logging.debug("Attempting to open connection to {0}:{1}".format( self.login_ip, app_port)) connection = httplib.HTTPConnection(self.login_ip, app_port) connection.request('POST', '/_ah/xmpp/message/chat/', encoded_params, self.HEADERS) response = connection.getresponse() logging.info("POST XMPP message returned status of {0}".format( response.status)) connection.close() except Exception as e: logging.exception(e)
def start_app(version_key, config): """ Starts a Google App Engine application on this machine. It will start it up and then proceed to fetch the main page. Args: version_key: A string specifying a version key. config: a dictionary that contains app_port: An integer specifying the port to use. login_server: The server address the AppServer will use for login urls. """ if 'app_port' not in config: raise BadConfigurationException('app_port is required') if 'login_server' not in config or not config['login_server']: raise BadConfigurationException('login_server is required') login_server = config['login_server'] project_id, service_id, version_id = version_key.split( VERSION_PATH_SEPARATOR) if not misc.is_app_name_valid(project_id): raise BadConfigurationException( 'Invalid project ID: {}'.format(project_id)) try: service_manager = projects_manager[project_id][service_id] version_details = service_manager[version_id].version_details except KeyError: raise BadConfigurationException('Version not found') runtime = version_details['runtime'] env_vars = version_details.get('envVariables', {}) runtime_params = deployment_config.get_config('runtime_parameters') max_memory = runtime_params.get('default_max_appserver_memory', DEFAULT_MAX_APPSERVER_MEMORY) if 'instanceClass' in version_details: max_memory = INSTANCE_CLASSES.get(version_details['instanceClass'], max_memory) version_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version_id]) revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version_id, str(version_details['revision'])]) source_archive = version_details['deployment']['zip']['sourceUrl'] api_server_port = ensure_api_server(project_id) yield source_manager.ensure_source(revision_key, source_archive, runtime) logging.info('Starting {} application {}'.format(runtime, project_id)) pidfile = PIDFILE_TEMPLATE.format(revision=revision_key, port=config['app_port']) if runtime == constants.GO: env_vars['GOPATH'] = os.path.join(UNPACK_ROOT, revision_key, 'gopath') env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot') watch = ''.join([MONIT_INSTANCE_PREFIX, revision_key]) if runtime in (constants.PYTHON27, constants.GO, constants.PHP): start_cmd = create_python27_start_cmd( project_id, login_server, config['app_port'], pidfile, revision_key, api_server_port) env_vars.update(create_python_app_env( login_server, project_id)) elif runtime == constants.JAVA: # Account for MaxPermSize (~170MB), the parent process (~50MB), and thread # stacks (~20MB). max_heap = max_memory - 250 if max_heap <= 0: raise BadConfigurationException( 'Memory for Java applications must be greater than 250MB') start_cmd = create_java_start_cmd( project_id, config['app_port'], login_server, max_heap, pidfile, revision_key ) env_vars.update(create_java_app_env(project_id)) else: raise BadConfigurationException( 'Unknown runtime {} for {}'.format(runtime, project_id)) logging.info("Start command: " + str(start_cmd)) logging.info("Environment variables: " + str(env_vars)) monit_app_configuration.create_config_file( watch, start_cmd, pidfile, config['app_port'], env_vars, max_memory, options.syslog_server, check_port=True, kill_exceeded_memory=True) # We want to tell monit to start the single process instead of the # group, since monit can get slow if there are quite a few processes in # the same group. full_watch = '{}-{}'.format(watch, config['app_port']) assert monit_interface.start(full_watch, is_group=False), ( 'Monit was unable to start {}:{}'.format(project_id, config['app_port'])) # Since we are going to wait, possibly for a long time for the # application to be ready, we do it in a thread. threading.Thread(target=add_routing, args=(version_key, config['app_port'])).start() if project_id == DASHBOARD_PROJECT_ID: log_size = DASHBOARD_LOG_SIZE else: log_size = APP_LOG_SIZE if not setup_logrotate(project_id, log_size): logging.error("Error while setting up log rotation for application: {}". format(project_id))
def post(self, project_id, service_id): """ Creates or updates a version. Args: project_id: A string specifying a project ID. service_id: A string specifying a service ID. """ if not self.PROJECT_ID_RE.match(project_id): raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message='Invalid project ID. ' 'It must be 6 to 30 lowercase letters, digits, ' 'or hyphens. It must start with a letter.') if not self.SERVICE_ID_RE.match(service_id): raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message='Invalid service ID. ' 'May only contain lowercase letters, digits, ' 'and hyphens. Must begin and end with a letter ' 'or digit. Must not exceed 63 characters.') self.authenticate(project_id, self.ua_client) version = self.version_from_payload() version_exists = self.version_exists(project_id, service_id, version['id']) revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version['id'], str(version['revision'])]) try: yield self.thread_pool.submit( utils.extract_source, revision_key, version['deployment']['zip']['sourceUrl'], version['runtime']) except IOError: message = '{} does not exist'.format( version['deployment']['zip']['sourceUrl']) raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=message) except constants.InvalidSource as error: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=six.text_type(error)) new_path = utils.rename_source_archive(project_id, service_id, version) version['deployment']['zip']['sourceUrl'] = new_path yield self.identify_as_hoster(project_id, service_id, version) yield self.thread_pool.submit(self.version_update_lock.acquire) try: version = self.put_version(project_id, service_id, version) except VersionNotChanged as warning: logger.info(six.text_type(warning)) self.stop_hosting_revision(project_id, service_id, version) return finally: self.version_update_lock.release() self.clean_up_revision_nodes(project_id, service_id, version) utils.remove_old_archives(project_id, service_id, version) operation = CreateVersionOperation(project_id, service_id, version) operations[operation.id] = operation pre_wait = REDEPLOY_WAIT if version_exists else 0 logger.debug( 'Starting operation {} in {}s'.format(operation.id, pre_wait)) IOLoop.current().call_later(pre_wait, wait_for_deploy, operation.id, self.controller_state) # Update the project's cron configuration. This is a bit messy because it # means acc.update_cron is often called twice when deploying a version. # However, it's needed for now to handle the following case: # 1. The user updates a project's cron config, referencing a module that # isn't deployed yet. # 2. The user deploys the referenced module from a directory that does not # have any cron configuration. # In order for the cron entries to use the correct location, # acc.update_cron needs to be called again even though the client did not # request a cron configuration update. This can be eliminated in the # future by routing requests based on the host header like in GAE. if not version_exists: self.acc.update_cron(project_id) self.write(json_encode(operation.rest_repr()))
def start_app(version_key, config): """ Starts a Google App Engine application on this machine. It will start it up and then proceed to fetch the main page. Args: version_key: A string specifying a version key. config: a dictionary that contains app_port: An integer specifying the port to use. login_server: The server address the AppServer will use for login urls. """ if 'app_port' not in config: raise BadConfigurationException('app_port is required') if 'login_server' not in config or not config['login_server']: raise BadConfigurationException('login_server is required') login_server = config['login_server'] project_id, service_id, version_id = version_key.split( VERSION_PATH_SEPARATOR) if not misc.is_app_name_valid(project_id): raise BadConfigurationException( 'Invalid project ID: {}'.format(project_id)) try: service_manager = projects_manager[project_id][service_id] version_details = service_manager[version_id].version_details except KeyError: raise BadConfigurationException('Version not found') runtime = version_details['runtime'] env_vars = version_details.get('envVariables', {}) runtime_params = deployment_config.get_config('runtime_parameters') max_memory = runtime_params.get('default_max_appserver_memory', DEFAULT_MAX_APPSERVER_MEMORY) if 'instanceClass' in version_details: max_memory = INSTANCE_CLASSES.get(version_details['instanceClass'], max_memory) revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version_id, str(version_details['revision'])]) source_archive = version_details['deployment']['zip']['sourceUrl'] api_server_port = yield ensure_api_server(project_id) yield source_manager.ensure_source(revision_key, source_archive, runtime) logging.info('Starting {} application {}'.format(runtime, project_id)) pidfile = PIDFILE_TEMPLATE.format(revision=revision_key, port=config['app_port']) if runtime == constants.GO: env_vars['GOPATH'] = os.path.join(UNPACK_ROOT, revision_key, 'gopath') env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot') watch = ''.join([MONIT_INSTANCE_PREFIX, revision_key]) if runtime in (constants.PYTHON27, constants.GO, constants.PHP): start_cmd = create_python27_start_cmd(project_id, login_server, config['app_port'], pidfile, revision_key, api_server_port) env_vars.update(create_python_app_env(login_server, project_id)) elif runtime == constants.JAVA: # Account for MaxPermSize (~170MB), the parent process (~50MB), and thread # stacks (~20MB). max_heap = max_memory - 250 if max_heap <= 0: raise BadConfigurationException( 'Memory for Java applications must be greater than 250MB') start_cmd = create_java_start_cmd(project_id, config['app_port'], login_server, max_heap, pidfile, revision_key, api_server_port) env_vars.update(create_java_app_env(project_id)) else: raise BadConfigurationException('Unknown runtime {} for {}'.format( runtime, project_id)) logging.info("Start command: " + str(start_cmd)) logging.info("Environment variables: " + str(env_vars)) monit_app_configuration.create_config_file(watch, start_cmd, pidfile, config['app_port'], env_vars, max_memory, options.syslog_server, check_port=True, kill_exceeded_memory=True) full_watch = '{}-{}'.format(watch, config['app_port']) monit_operator = MonitOperator() yield monit_operator.reload(thread_pool) yield monit_operator.send_command_retry_process(full_watch, 'start') # Make sure the version node exists. zk_client.ensure_path('/'.join([VERSION_REGISTRATION_NODE, version_key])) # Since we are going to wait, possibly for a long time for the # application to be ready, we do it later. IOLoop.current().spawn_callback(add_routing, Instance(revision_key, config['app_port'])) if project_id == DASHBOARD_PROJECT_ID: log_size = DASHBOARD_LOG_SIZE else: log_size = APP_LOG_SIZE if not setup_logrotate(project_id, log_size): logging.error( "Error while setting up log rotation for application: {}".format( project_id))
def start_app(version_key, config): """ Starts a Google App Engine application on this machine. It will start it up and then proceed to fetch the main page. Args: version_key: A string specifying a version key. config: a dictionary that contains app_port: An integer specifying the port to use. """ if 'app_port' not in config: raise BadConfigurationException('app_port is required') project_id, service_id, version_id = version_key.split( VERSION_PATH_SEPARATOR) if not misc.is_app_name_valid(project_id): raise BadConfigurationException( 'Invalid project ID: {}'.format(project_id)) try: service_manager = projects_manager[project_id][service_id] version_details = service_manager[version_id].version_details except KeyError: raise BadConfigurationException('Version not found') runtime = version_details['runtime'] env_vars = version_details.get('envVariables', {}) runtime_params = deployment_config.get_config('runtime_parameters') max_memory = runtime_params.get('default_max_appserver_memory', DEFAULT_MAX_APPSERVER_MEMORY) if 'instanceClass' in version_details: max_memory = INSTANCE_CLASSES.get(version_details['instanceClass'], max_memory) version_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version_id]) revision_key = VERSION_PATH_SEPARATOR.join( [project_id, service_id, version_id, str(version_details['revision'])]) source_archive = version_details['deployment']['zip']['sourceUrl'] yield source_manager.ensure_source(revision_key, source_archive, runtime) logging.info('Starting {} application {}'.format(runtime, project_id)) pidfile = PIDFILE_TEMPLATE.format(revision=revision_key, port=config['app_port']) if runtime == constants.GO: env_vars['GOPATH'] = os.path.join(UNPACK_ROOT, revision_key, 'gopath') env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot') watch = ''.join([MONIT_INSTANCE_PREFIX, revision_key]) if runtime in (constants.PYTHON27, constants.GO, constants.PHP): start_cmd = create_python27_start_cmd(project_id, options.login_ip, config['app_port'], pidfile, revision_key) env_vars.update(create_python_app_env(options.login_ip, project_id)) elif runtime == constants.JAVA: # Account for MaxPermSize (~170MB), the parent process (~50MB), and thread # stacks (~20MB). max_heap = max_memory - 250 if max_heap <= 0: raise BadConfigurationException( 'Memory for Java applications must be greater than 250MB') start_cmd = create_java_start_cmd(project_id, config['app_port'], options.login_ip, max_heap, pidfile, revision_key) env_vars.update(create_java_app_env(project_id)) else: raise BadConfigurationException('Unknown runtime {} for {}'.format( runtime, project_id)) logging.info("Start command: " + str(start_cmd)) logging.info("Environment variables: " + str(env_vars)) monit_app_configuration.create_config_file(watch, start_cmd, pidfile, config['app_port'], env_vars, max_memory, options.syslog_server, check_port=True) # We want to tell monit to start the single process instead of the # group, since monit can get slow if there are quite a few processes in # the same group. full_watch = '{}-{}'.format(watch, config['app_port']) assert monit_interface.start( full_watch, is_group=False), ('Monit was unable to start {}:{}'.format( project_id, config['app_port'])) # Since we are going to wait, possibly for a long time for the # application to be ready, we do it in a thread. threading.Thread(target=add_routing, args=(version_key, config['app_port'])).start() if project_id == DASHBOARD_PROJECT_ID: log_size = DASHBOARD_LOG_SIZE else: log_size = APP_LOG_SIZE if not setup_logrotate(project_id, log_size): logging.error( "Error while setting up log rotation for application: {}".format( project_id))
def version_key(self): return VERSION_PATH_SEPARATOR.join( [self.project_id, self.service_id, self.version_id])
def version_key(self): revision_parts = self.revision_key.split(VERSION_PATH_SEPARATOR) return VERSION_PATH_SEPARATOR.join(revision_parts[:3])