def get_resource(blueprint_id, deployment_id, tenant_name, resource_path): """ Get resource from the manager file server with path relative to the deployment or blueprint denoted by ``deployment_id`` or ``blueprint_id``. An attempt will first be made for getting the resource from the deployment folder. If not found, an attempt will be made for getting the resource from the blueprint folder. :param blueprint_id: the blueprint id of the blueprint to download the resource from :param deployment_id: the deployment id of the deployment to download the resource from :param tenant_name: tenant name :param resource_path: path to resource relative to blueprint folder :returns: resource content """ def _get_resource(base_url): try: return get_resource_from_manager(resource_path, base_url=base_url) except HttpException as e: if e.code != 404: raise return None resource = None if deployment_id is not None: relative_deployment_path = os.path.join( constants.FILE_SERVER_RESOURCES_FOLDER, constants.FILE_SERVER_DEPLOYMENTS_FOLDER, tenant_name, deployment_id) deployment_base_url = urljoin(utils.get_manager_file_server_url(), relative_deployment_path).replace( '\\', '/') resource = _get_resource(deployment_base_url) if resource is None: client = get_rest_client() blueprint = client.blueprints.get(blueprint_id) if blueprint['visibility'] == VisibilityState.GLOBAL: tenant_name = blueprint['tenant_name'] relative_blueprint_path = os.path.join( constants.FILE_SERVER_RESOURCES_FOLDER, constants.FILE_SERVER_BLUEPRINTS_FOLDER, tenant_name, blueprint_id) blueprint_base_url = urljoin(utils.get_manager_file_server_url(), relative_blueprint_path).replace( '\\', '/') resource = _get_resource(blueprint_base_url) if resource is None: if deployment_id is None: url = blueprint_base_url else: url = ','.join([deployment_base_url, blueprint_base_url]) raise HttpException( url, 404, 'Resource not found: {0}'.format(resource_path)) return resource
def installation_attributes(cloudify_agent, runner): if (not cloudify_agent.get('source_url') and not cloudify_agent.get('package_url')): if cloudify_agent['windows']: # no distribution difference in windows installation cloudify_agent['package_url'] = '{0}/packages/agents' \ '/cloudify-windows-agent.exe'\ .format(cloudify_utils.get_manager_file_server_url()) else: if not cloudify_agent.get('distro'): if cloudify_agent['local']: cloudify_agent['distro'] = platform.dist()[0].lower() elif cloudify_agent['remote_execution']: dist = runner.machine_distribution() cloudify_agent['distro'] = dist[0].lower() if not cloudify_agent.get('distro_codename'): if cloudify_agent['local']: cloudify_agent['distro_codename'] = platform.dist()[ 2].lower() elif cloudify_agent['remote_execution']: dist = runner.machine_distribution() cloudify_agent['distro_codename'] = dist[2].lower() if ('distro' in cloudify_agent and 'distro_codename' in cloudify_agent): cloudify_agent['package_url'] = '{0}/packages/agents' \ '/{1}-{2}-agent.tar.gz' \ .format(cloudify_utils.get_manager_file_server_url(), cloudify_agent['distro'], cloudify_agent['distro_codename']) if not cloudify_agent.get('basedir'): if cloudify_agent['local']: basedir = agent_utils.get_home_dir(cloudify_agent['user']) else: if cloudify_agent['windows']: # can't seem to figure out how to get the home_dir remotely # on windows. same was as fabric wont work because the # 'pwd' module does not exists in a windows python # installation. # TODO - maybe use some environment variables heuristics? basedir = \ agent_utils.get_windows_home_dir(cloudify_agent['user']) elif cloudify_agent['remote_execution']: basedir = runner.home_dir(cloudify_agent['user']) else: basedir = '~{0}'.format(cloudify_agent['user']) cloudify_agent['basedir'] = basedir directory_attributes(cloudify_agent)
def installation_attributes(cloudify_agent, runner): if (not cloudify_agent.get('source_url') and not cloudify_agent.get('package_url')): if cloudify_agent['windows']: # no distribution difference in windows installation cloudify_agent['package_url'] = '{0}/packages/agents' \ '/cloudify-windows-agent.exe'\ .format(get_manager_file_server_url()) else: if not cloudify_agent.get('distro'): if cloudify_agent['local']: cloudify_agent['distro'] = platform.dist()[0].lower() elif cloudify_agent['remote_execution']: dist = runner.machine_distribution() cloudify_agent['distro'] = dist[0].lower() if not cloudify_agent.get('distro_codename'): if cloudify_agent['local']: cloudify_agent['distro_codename'] = platform.dist( )[2].lower() elif cloudify_agent['remote_execution']: dist = runner.machine_distribution() cloudify_agent['distro_codename'] = dist[2].lower() if ('distro' in cloudify_agent and 'distro_codename' in cloudify_agent): cloudify_agent['package_url'] = '{0}/packages/agents' \ '/{1}-{2}-agent.tar.gz' \ .format(get_manager_file_server_url(), cloudify_agent['distro'], cloudify_agent['distro_codename']) if not cloudify_agent.get('basedir'): if cloudify_agent['local']: basedir = utils.get_home_dir(cloudify_agent['user']) else: if cloudify_agent['windows']: # can't seem to figure out how to get the home_dir remotely # on windows. same was as fabric wont work because the # 'pwd' module does not exists in a windows python # installation. # TODO - maybe use some environment variables heuristics? basedir = utils.get_windows_home_dir(cloudify_agent['user']) elif cloudify_agent['remote_execution']: basedir = runner.home_dir(cloudify_agent['user']) else: basedir = '~{0}'.format(cloudify_agent['user']) cloudify_agent['basedir'] = basedir directory_attributes(cloudify_agent)
def installation_attributes(cloudify_agent, runner): if (not cloudify_agent.get('source_url') and not cloudify_agent.get('package_url')): if cloudify_agent['windows']: # no distribution difference in windows installation cloudify_agent['package_url'] = '{0}/packages/agents' \ '/cloudify-windows-agent.exe'\ .format(cloudify_utils.get_manager_file_server_url()) else: if not cloudify_agent.get('distro'): if cloudify_agent['local']: cloudify_agent['distro'] = platform.dist()[0].lower() elif cloudify_agent['remote_execution']: dist = runner.machine_distribution() cloudify_agent['distro'] = dist[0].lower() if not cloudify_agent.get('distro_codename'): if cloudify_agent['local']: cloudify_agent['distro_codename'] = platform.dist( )[2].lower() elif cloudify_agent['remote_execution']: dist = runner.machine_distribution() cloudify_agent['distro_codename'] = dist[2].lower() if ('distro' in cloudify_agent and 'distro_codename' in cloudify_agent): cloudify_agent['package_url'] = '{0}/packages/agents' \ '/{1}-{2}-agent.tar.gz' \ .format(cloudify_utils.get_manager_file_server_url(), cloudify_agent['distro'], cloudify_agent['distro_codename']) if not cloudify_agent.get('basedir'): if cloudify_agent['local']: basedir = agent_utils.get_home_dir(cloudify_agent['user']) else: if cloudify_agent['windows']: # TODO: Get the program files directory from the machine iself # instead of hardcoding it an assuming it's in C:\ basedir = 'C:\\Program Files\\Cloudify Agents' elif cloudify_agent['remote_execution']: basedir = runner.home_dir(cloudify_agent['user']) else: basedir = '~{0}'.format(cloudify_agent['user']) cloudify_agent['basedir'] = basedir directory_attributes(cloudify_agent)
def get_disable_requiretty_script_url(distro): """ Returns the disable requiretty script url the script will be downloaded from. """ return '{0}/{1}'.format(utils.get_manager_file_server_url(), DISABLE_REQUIRETTY_SCRIPT_URL.format(distro))
def _run_install_script(old_agent, timeout, validate_only=False): # Assuming that if there is no version info in the agent then # this agent was installed by current manager. old_agent = copy.deepcopy(old_agent) if 'version' not in old_agent: old_agent['version'] = _get_manager_version() new_agent = create_new_agent_dict(old_agent) with _celery_client(ctx, old_agent) as celery_client: old_agent_name = old_agent['name'] _assert_agent_alive(old_agent_name, celery_client) script_format = '{0}/cloudify/install_agent.py' script_url = script_format.format(get_manager_file_server_url()) result = celery_client.send_task( 'script_runner.tasks.run', args=[script_url], kwargs={ 'cloudify_agent': new_agent, 'validate_only': validate_only}, queue=old_agent['queue'] ) returned_agent = result.get(timeout=timeout) if returned_agent['name'] != new_agent['name']: raise NonRecoverableError( 'Expected agent name {0}, received {1}'.format( new_agent['name'], returned_agent['name']) ) returned_agent.pop('old_agent_version', None) return { 'old': old_agent, 'new': returned_agent }
def _run_install_script(old_agent, timeout, validate_only=False): # Assuming that if there is no version info in the agent then # this agent was installed by current manager. old_agent = copy.deepcopy(old_agent) if 'version' not in old_agent: old_agent['version'] = _get_manager_version() new_agent = create_new_agent_dict(old_agent) with _celery_client(ctx, old_agent) as celery_client: old_agent_name = old_agent['name'] _assert_agent_alive(old_agent_name, celery_client) script_format = '{0}/cloudify/install_agent.py' script_url = script_format.format(get_manager_file_server_url()) result = celery_client.send_task('script_runner.tasks.run', args=[script_url], kwargs={ 'cloudify_agent': new_agent, 'validate_only': validate_only }, queue=old_agent['queue']) returned_agent = result.get(timeout=timeout) if returned_agent['name'] != new_agent['name']: raise NonRecoverableError( 'Expected agent name {0}, received {1}'.format( new_agent['name'], returned_agent['name'])) returned_agent.pop('old_agent_version', None) return {'old': old_agent, 'new': returned_agent}
def install_script(self, add_ssl_cert=True): """Render the agent installation script. :return: Install script content :rtype: str """ template = self._get_template(self.install_script_template) # Called before creating the agent env to populate all the variables cert_content = self._get_local_cert_content() if add_ssl_cert else '' remote_ssl_cert_path = self._get_remote_ssl_cert_path() # Called before rendering the template to populate all the variables daemon_env = self._create_agent_env() return template.render( conf=self.cloudify_agent, daemon_env=daemon_env, pm_options=self._create_process_management_options(), custom_env=self.custom_env, custom_env_path=self.custom_env_path, file_server_url=cloudify_utils.get_manager_file_server_url(), configure_flags=self._configure_flags(), ssl_cert_content=cert_content, ssl_cert_path=remote_ssl_cert_path, auth_token_header=CLOUDIFY_TOKEN_AUTHENTICATION_HEADER, auth_token_value=ctx.rest_token, install=not self.cloudify_agent.is_provided, configure=True, start=True, add_ssl_cert=add_ssl_cert)
def get_resource_from_manager(resource_path, base_url=None, base_urls=None): """Get resource from the manager file server. :param resource_path: path to resource on the file server :param base_url: The base URL to manager file server. Deprecated. :param base_urls: A list of base URL to cluster manager file servers. :param resource_path: path to resource on the file server. :returns: resource content """ base_urls = base_urls or [] base_urls += utils.get_manager_file_server_url() if base_url is not None: base_urls.insert(0, base_url) # if we have multiple managers to try, set connect_timeout so that # we're not waiting forever for a single non-responding manager if len(base_urls) > 1: timeout = (10, None) else: timeout = None verify = utils.get_local_rest_certificate() headers = {} try: headers[constants.CLOUDIFY_EXECUTION_TOKEN_HEADER] = \ ctx.execution_token except NotInContext: headers[constants.CLOUDIFY_EXECUTION_TOKEN_HEADER] = \ workflow_ctx.execution_token for ix, next_url in enumerate(base_urls): url = '{0}/{1}'.format(next_url.rstrip('/'), resource_path.lstrip('/')) try: response = requests.get(url, verify=verify, headers=headers, timeout=timeout) except requests.ConnectionError: continue if not response.ok: is_last = (ix == len(base_urls) - 1) if not is_last: # if there's more managers to try, try them: due to filesystem # replication lag, they might have files that the previous # manager didn't continue raise HttpException(url, response.status_code, response.reason) return response.content raise NonRecoverableError( 'Failed to download {0}: unable to connect to any manager (tried: {1})' .format(resource_path, ', '.join(base_urls)))
def create_new_agent_dict(old_agent): new_agent = {} new_agent['name'] = utils.internal.generate_new_agent_name( old_agent['name']) new_agent['remote_execution'] = True fields_to_copy = ['windows', 'ip', 'basedir', 'user'] for field in fields_to_copy: if field in old_agent: new_agent[field] = old_agent[field] configuration.reinstallation_attributes(new_agent) new_agent['manager_file_server_url'] = get_manager_file_server_url() new_agent['old_agent_version'] = old_agent['version'] return new_agent
def get_agent_resource_url(ctx, agent_config, resource): """returns an agent's resource url The resource will be looked for in the agent's properties. If it isn't found, it will look for it in the default location. """ if agent_config.get(resource): origin = utils.get_manager_file_server_blueprints_root_url() + \ '/' + ctx.blueprint.id + '/' + agent_config[resource] else: resource_path = DEFAULT_AGENT_RESOURCES.get(resource) if not resource_path: raise NonRecoverableError('no such resource: {0}'.format(resource)) if resource == 'agent_package_path': origin = utils.get_manager_file_server_url() + \ resource_path.format(agent_config['distro'], agent_config['distro_codename']) else: origin = utils.get_manager_file_server_url() + \ resource_path.format(agent_config['distro']) ctx.logger.debug('resource origin: {0}'.format(origin)) return origin
def build(self): if self.cloudify_agent['windows']: resource = 'script/windows.ps1.template' else: resource = 'script/linux.sh.template' template = jinja2.Template(utils.get_resource(resource)) # called before so that custom_env and custom_env_path # get populated daemon_env = self._create_agent_env() return template.render( conf=self.cloudify_agent, daemon_env=daemon_env, pm_options=self._create_process_management_options(), custom_env=self.custom_env, custom_env_path=self.custom_env_path, file_server_url=cloudify_utils.get_manager_file_server_url(), configure_flags=self._configure_flags())
def create_new_agent_dict(old_agent): new_agent = {} new_agent['name'] = utils.internal.generate_new_agent_name( old_agent['name']) new_agent['remote_execution'] = True # TODO: broker_ip should be handled as part of fixing agent migration fields_to_copy = [ 'windows', 'ip', 'basedir', 'user', 'ssl_cert_path', 'agent_rest_cert_path' ] for field in fields_to_copy: if field in old_agent: new_agent[field] = old_agent[field] configuration.reinstallation_attributes(new_agent) new_agent['manager_file_server_url'] = get_manager_file_server_url() new_agent['old_agent_version'] = old_agent['version'] return new_agent
def create_env_string(cloudify_agent): env = { LOCAL_IP_KEY: cloudify_agent['host'], MANAGER_IP_KEY: get_manager_ip(), MANAGER_FILE_SERVER_BLUEPRINTS_ROOT_URL_KEY: get_manager_file_server_blueprints_root_url(), MANAGER_FILE_SERVER_URL_KEY: get_manager_file_server_url(), MANAGER_REST_PORT_KEY: get_manager_rest_service_port() } env_string = '' for key, value in env.iteritems(): env_string = '{0} {1}={2}' \ .format(env_string, key, value) return env_string.strip()
def __init__(self, cloudify_agent): super(AgentInstallationScriptBuilder, self).__init__(cloudify_agent) self.custom_env = None self.file_server_root = cloudify_utils.get_manager_file_server_root() self.file_server_url = cloudify_utils.get_manager_file_server_url() basedir = self.cloudify_agent['basedir'] if cloudify_agent.is_windows: self.install_script_template = 'script/windows.ps1.template' self.init_script_template = 'script/windows-download.ps1.template' self.install_script_filename = '{0}.ps1'.format(uuid.uuid4()) self.init_script_filename = '{0}.ps1'.format(uuid.uuid4()) self.custom_env_path = '{0}\\custom_agent_env.bat'.format(basedir) else: self.install_script_template = 'script/linux.sh.template' self.init_script_template = 'script/linux-download.sh.template' self.install_script_filename = '{0}.sh'.format(uuid.uuid4()) self.init_script_filename = '{0}.sh'.format(uuid.uuid4()) self.custom_env_path = '{0}/custom_agent_env.sh'.format(basedir)
def _run_install_script(old_agent, timeout, validate_only=False, install_script=None): # Assuming that if there is no version info in the agent then # this agent was installed by current manager. old_agent = copy.deepcopy(old_agent) if 'version' not in old_agent: old_agent['version'] = str(_get_manager_version()) new_agent = create_new_agent_dict(old_agent) old_agent_version = new_agent['old_agent_version'] with _celery_client(ctx, old_agent) as celery_client: old_agent_name = old_agent['name'] _assert_agent_alive(old_agent_name, celery_client, old_agent_version) if install_script is None: script_format = '{0}/cloudify/install_agent.py' install_script = script_format.format( get_manager_file_server_url()) script_runner_task = 'script_runner.tasks.run' cloudify_context = { 'type': 'operation', 'task_name': script_runner_task, 'task_target': old_agent['queue'] } kwargs = {'script_path': install_script, 'cloudify_agent': new_agent, 'validate_only': validate_only, '__cloudify_context': cloudify_context} task = _celery_task_name(old_agent_version) result = celery_client.send_task( task, kwargs=kwargs, queue=old_agent['queue'] ) returned_agent = result.get(timeout=timeout) if returned_agent['name'] != new_agent['name']: raise NonRecoverableError( 'Expected agent name {0}, received {1}'.format( new_agent['name'], returned_agent['name']) ) returned_agent.pop('old_agent_version', None) return { 'old': old_agent, 'new': returned_agent }
def create_env_string(cloudify_agent): env = { constants.CELERY_WORK_DIR_PATH_KEY: RUNTIME_AGENT_PATH, constants.LOCAL_IP_KEY: cloudify_agent['host'], constants.MANAGER_IP_KEY: utils.get_manager_ip(), constants.MANAGER_FILE_SERVER_BLUEPRINTS_ROOT_URL_KEY: utils.get_manager_file_server_blueprints_root_url(), constants.MANAGER_FILE_SERVER_URL_KEY: utils.get_manager_file_server_url(), constants.MANAGER_REST_PORT_KEY: utils.get_manager_rest_service_port() } env_string = '' for key, value in env.iteritems(): env_string = '{0} {1}={2}' \ .format(env_string, key, value) return env_string.strip()
def _set_package_url(self, runner): if self.get('package_url'): return agent_package_name = None if self.is_windows: # No distribution difference in windows installation agent_package_name = 'cloudify-windows-agent.exe' else: self._set_agent_distro(runner) self._set_agent_distro_codename(runner) if 'distro' in self and 'distro_codename' in self: agent_package_name = '{0}-{1}-agent.tar.gz'.format( self['distro'], self['distro_codename'] ) if agent_package_name: self['package_url'] = posix_join( cloudify_utils.get_manager_file_server_url(), 'packages', 'agents', agent_package_name )
def build(self): if self.cloudify_agent['windows']: resource = 'script/windows.ps1.template' else: resource = 'script/linux.sh.template' template = jinja2.Template(utils.get_resource(resource)) # Called before creating the agent env to populate all the variables local_rest_content = self._get_local_cert_content() remote_ssl_cert_path = self._get_remote_ssl_cert_path() # Called before rendering the template to populate all the variables daemon_env = self._create_agent_env() return template.render( conf=self.cloudify_agent, daemon_env=daemon_env, pm_options=self._create_process_management_options(), custom_env=self.custom_env, custom_env_path=self.custom_env_path, file_server_url=cloudify_utils.get_manager_file_server_url(), configure_flags=self._configure_flags(), ssl_cert_content=local_rest_content, ssl_cert_path=remote_ssl_cert_path, auth_token_header=CLOUDIFY_TOKEN_AUTHENTICATION_HEADER, auth_token_value=ctx.rest_token)
def get_resource_from_manager(resource_path, base_url=None): """ Get resource from the manager file server. :param resource_path: path to resource on the file server :returns: resource content """ if base_url is None: base_url = utils.get_manager_file_server_url() url = '{0}/{1}'.format(base_url, resource_path) verify = utils.get_local_rest_certificate() headers = {} try: headers[constants.CLOUDIFY_TOKEN_AUTHENTICATION_HEADER] = \ ctx.rest_token except NotInContext: headers[constants.CLOUDIFY_TOKEN_AUTHENTICATION_HEADER] = \ workflow_ctx.rest_token response = requests.get(url, verify=verify, headers=headers) if not response.ok: raise HttpException(url, response.status_code, response.reason) return response.content
def _get_creds_url(self): return urljoin(get_manager_file_server_url(), 'cloudify_agent', self._creds_filename)
def get_agent_package_url(distro): """ Returns the agent package url the package will be downloaded from. """ return '{0}/{1}'.format(utils.get_manager_file_server_url(), AGENT_PACKAGE_PATH.format(distro))
def get_agent_package_url(): return '{0}{1}'.format(utils.get_manager_file_server_url(), AGENT_PACKAGE_PATH)
def installation_attributes(cloudify_agent, runner): if 'source_url' not in cloudify_agent: if 'package_url' not in cloudify_agent: if cloudify_agent['windows']: # no distribution difference in windows installation cloudify_agent['package_url'] = '{0}/packages/agents' \ '/cloudify-windows-agent.exe'\ .format(get_manager_file_server_url()) else: # build one from distro and distro_codename if cloudify_agent['local']: cloudify_agent['distro'] = platform.dist()[0].lower() else: dist = runner.machine_distribution() cloudify_agent['distro'] = dist[0].lower() # distro was not specified, try to auto-detect if cloudify_agent['local']: cloudify_agent['distro_codename'] = platform.dist()[ 2].lower() else: dist = runner.machine_distribution() cloudify_agent['distro_codename'] = dist[2].lower() cloudify_agent['package_url'] = '{0}/packages/agents' \ '/{1}-{2}-agent.tar.gz' \ .format(get_manager_file_server_url(), cloudify_agent['distro'], cloudify_agent['distro_codename']) if 'basedir' not in cloudify_agent: if cloudify_agent['local']: basedir = utils.get_home_dir(cloudify_agent['user']) else: if cloudify_agent['windows']: # can't seem to figure out how to get the home_dir remotely # on windows. same was as fabric wont work because the # 'pwd' module does not exists in a windows python # installation. # TODO - maybe use some environment variables heuristics? basedir = 'C:\\Users\\{0}'.format(cloudify_agent['user']) else: basedir = runner.home_dir(cloudify_agent['user']) cloudify_agent['basedir'] = basedir if 'agent_dir' not in cloudify_agent: name = cloudify_agent['name'] basedir = cloudify_agent['basedir'] if cloudify_agent['windows']: agent_dir = '{0}\\{1}'.format(basedir, name) else: agent_dir = os.path.join(basedir, name) cloudify_agent['agent_dir'] = agent_dir if 'workdir' not in cloudify_agent: agent_dir = cloudify_agent['agent_dir'] if cloudify_agent['windows']: workdir = '{0}\\{1}'.format(agent_dir, 'work') else: workdir = os.path.join(agent_dir, 'work') cloudify_agent['workdir'] = workdir if 'envdir' not in cloudify_agent: agent_dir = cloudify_agent['agent_dir'] if cloudify_agent['windows']: envdir = '{0}\\{1}'.format(agent_dir, 'env') else: envdir = os.path.join(agent_dir, 'env') cloudify_agent['envdir'] = envdir