def delete_proxy_user_for_job_execution(job_execution): '''Delete a proxy user based on a JobExecution :param job_execution: The job execution with proxy user information :returns: An updated job_configs dictionary or None ''' proxy_configs = job_execution.job_configs.get('proxy_configs') if proxy_configs is not None: proxy_username = proxy_configs.get('proxy_username') key = key_manager.API().get( context.current(), proxy_configs.get('proxy_password')) proxy_password = key.get_encoded() proxy_trust_id = proxy_configs.get('proxy_trust_id') proxy_user = k.auth_for_proxy(proxy_username, proxy_password, proxy_trust_id) t.delete_trust(proxy_user, proxy_trust_id) proxy_user_delete(proxy_username) key_manager.API().delete(context.current(), proxy_configs.get('proxy_password')) update = job_execution.job_configs.to_dict() del update['proxy_configs'] return update return None
def url_for(service_catalog=None, service_type="identity", endpoint_type="publicURL"): if not service_catalog: service_catalog = context.current().service_catalog try: return keystone_service_catalog.ServiceCatalogV2({"serviceCatalog": json.loads(service_catalog)}).url_for( service_type=service_type, endpoint_type=endpoint_type, region_name=CONF.os_region_name ) except keystone_ex.EndpointNotFound: ctx = context.current() return keystone_service_catalog.ServiceCatalogV3( ctx.auth_token, {"catalog": json.loads(service_catalog)} ).url_for(service_type=service_type, endpoint_type=endpoint_type, region_name=CONF.os_region_name)
def retrieve_auth_url(): """This function return auth url v2.0 api. Hadoop Swift library doesn't support keystone v3 api. """ info = urlparse.urlparse(context.current().auth_uri) return "%s://%s:%s/%s/" % (info.scheme, info.hostname, info.port, 'v2.0')
def check_cluster_update(cluster_id, data, **kwargs): cluster = api.get_cluster(cluster_id) verification = verification_base.validate_verification_ops(cluster, data) acl.check_tenant_for_update(context.current(), cluster) if not verification: acl.check_protected_from_update(cluster, data)
def execute_job(job_id, data): # Elements common to all job types cluster_id = data['cluster_id'] configs = data.get('job_configs', {}) ctx = context.current() cluster = conductor.cluster_get(ctx, cluster_id) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) instance = plugin.get_oozie_server(cluster) extra = {} info = None if CONF.use_namespaces and not CONF.use_floating_ips: info = instance.remote().get_neutron_info() extra['neutron'] = info # Not in Java job types but present for all others input_id = data.get('input_id', None) output_id = data.get('output_id', None) # Since we will use a unified class in the database, we pass # a superset for all job types job_ex_dict = {'input_id': input_id, 'output_id': output_id, 'job_id': job_id, 'cluster_id': cluster_id, 'info': {'status': 'Pending'}, 'job_configs': configs, 'extra': extra} job_execution = conductor.job_execution_create(context.ctx(), job_ex_dict) context.spawn("Starting Job Execution %s" % job_execution.id, manager.run_job, job_execution) return job_execution
def _build_proxy_command(self, command, instance=None, port=None, info=None, rootwrap_command=None): # Accepted keywords in the proxy command template: # {host}, {port}, {tenant_id}, {network_id}, {router_id} keywords = {} if not info: info = self.get_neutron_info(instance) keywords['tenant_id'] = context.current().tenant_id keywords['network_id'] = info['network'] # Query Neutron only if needed if '{router_id}' in command: client = neutron.NeutronClient(info['network'], info['token'], info['tenant']) keywords['router_id'] = client.get_router() keywords['host'] = instance.management_ip keywords['port'] = port try: command = command.format(**keywords) except KeyError as e: LOG.error(_LE('Invalid keyword in proxy_command: {result}').format( result=e)) # Do not give more details to the end-user raise ex.SystemError('Misconfiguration') if rootwrap_command: command = '{0} {1}'.format(rootwrap_command, command) return command
def mount_to_instances(instances): if len(instances) == 0: return use_xfs = _can_use_xfs(instances) for instance in instances: with context.set_current_instance_id(instance.instance_id): devices = _find_instance_devices(instance) if devices: cpo.add_provisioning_step( instance.cluster_id, _("Mount volumes to {inst_name} instance").format( inst_name=instance.instance_name), len(devices)) formatted_devices = [] lock = threading.Lock() with context.ThreadGroup() as tg: # Since formating can take several minutes (for large # disks) and can be done in parallel, launch one thread # per disk. for device in devices: tg.spawn('format-device-%s' % device, _format_device, instance, device, use_xfs, formatted_devices, lock) conductor.instance_update( context.current(), instance, {"storage_devices_number": len(formatted_devices)}) for idx, dev in enumerate(formatted_devices): _mount_volume_to_node(instance, idx+1, dev, use_xfs)
def check_cluster_scaling(data, cluster_id, **kwargs): ctx = context.current() cluster = api.get_cluster(id=cluster_id) if cluster is None: raise ex.NotFoundException( {'id': cluster_id}, _('Object with %s not found')) b.check_plugin_labels( cluster.plugin_name, cluster.hadoop_version) acl.check_tenant_for_update(ctx, cluster) acl.check_protected_from_update(cluster, data) cluster_engine = cluster.sahara_info.get( 'infrastructure_engine') if cluster.sahara_info else None engine_type_and_version = service_api.OPS.get_engine_type_and_version() if (not cluster_engine and not engine_type_and_version.startswith('direct')): raise ex.InvalidReferenceException( _("Cluster created before Juno release " "can't be scaled with %(engine)s engine") % {"engine": engine_type_and_version}) if (cluster.sahara_info and cluster_engine != engine_type_and_version): raise ex.InvalidReferenceException( _("Cluster created with %(old_engine)s infrastructure engine " "can't be scaled with %(new_engine)s engine") % {"old_engine": cluster.sahara_info.get('infrastructure_engine'), "new_engine": engine_type_and_version}) if not (plugin_base.PLUGINS.is_plugin_implements(cluster.plugin_name, 'scale_cluster') and ( plugin_base.PLUGINS.is_plugin_implements(cluster.plugin_name, 'decommission_nodes'))): raise ex.InvalidReferenceException( _("Requested plugin '%s' doesn't support cluster scaling feature") % cluster.plugin_name) if cluster.status != c_u.CLUSTER_STATUS_ACTIVE: raise ex.InvalidReferenceException( _("Cluster cannot be scaled not in 'Active' status. " "Cluster status: %s") % cluster.status) if cluster.user_keypair_id: b.check_keypair_exists(cluster.user_keypair_id) if cluster.default_image_id: b.check_image_registered(cluster.default_image_id) if data.get("resize_node_groups"): b.check_resize(cluster, data['resize_node_groups']) if data.get("add_node_groups"): b.check_add_node_groups(cluster, data['add_node_groups']) b.check_cluster_hostnames_lengths(cluster.name, data['add_node_groups'])
def _retrieve_tenant(): try: return context.current().tenant_name except RuntimeError: LOG.exception("Cannot retrieve tenant for swift integration. " "Stopping cluster creation") #todo(slukjanov?) raise special error here raise RuntimeError("Cannot retrieve tenant for swift integration")
def use_os_admin_auth_token(cluster): if cluster.trust_id: ctx = context.current() ctx.username = CONF.keystone_authtoken.admin_user ctx.tenant_id = cluster.tenant_id client = keystone.client_for_trusts(cluster.trust_id) ctx.token = client.auth_token ctx.service_catalog = json.dumps(client.service_catalog.catalog["catalog"])
def retrieve_auth_url(): version = "v3" if CONF.use_identity_api_v3 else "v2.0" ctx = context.current() if ctx.service_catalog: info = urlparse.urlparse(url_for(ctx.service_catalog, "identity")) else: info = urlparse.urlparse(CONF.keystone_authtoken.auth_uri) return "%s://%s/%s" % (info[:2] + (version,))
def check_cinder_exists(): service_type = 'volumev2' try: base.url_for(context.current().service_catalog, service_type, endpoint_type=CONF.cinder.endpoint_type) return True except ex.SystemError: return False
def retrieve_auth_url(endpoint_type="internalURL", version=None): if not version: version = 'v3' if CONF.use_identity_api_v3 else 'v2.0' ctx = context.current() if ctx.service_catalog: auth_url = url_for(ctx.service_catalog, 'identity', endpoint_type) else: auth_url = CONF.keystone_authtoken.auth_uri return prepare_auth_url(auth_url, version)
def get_raw_data(job_binary, proxy_configs=None): conn_kwargs = {} if proxy_configs: key = key_manager.API().get(context.current(), proxy_configs.get('proxy_password')) password = key.get_encoded() conn_kwargs.update(username=proxy_configs.get('proxy_username'), password=password, trust_id=proxy_configs.get('proxy_trust_id')) else: key = key_manager.API().get(context.current(), job_binary.extra.get('password')) password = key.get_encoded() conn_kwargs.update(username=job_binary.extra.get('user'), password=password) conn = sw.client(**conn_kwargs) return _get_raw_data(job_binary, conn)
def client(): ctx = context.current() heat_url = base.url_for(ctx.service_catalog, 'orchestration', endpoint_type=CONF.heat.endpoint_type) return heat_client.Client('1', heat_url, token=ctx.auth_token, cert_file=CONF.heat.ca_file, insecure=CONF.heat.api_insecure, username=ctx.username, include_pass=True)
def retrieve_auth_url(endpoint_type="internalURL"): version = 'v3' if CONF.use_identity_api_v3 else 'v2.0' ctx = context.current() if ctx.service_catalog: info = urlparse.urlparse(url_for(ctx.service_catalog, 'identity', endpoint_type)) else: info = urlparse.urlparse(CONF.keystone_authtoken.auth_uri) return "%s://%s/%s" % (info[:2] + (version,))
def check_cinder_exists(): if CONF.cinder.api_version == 1: service_type = 'volume' else: service_type = 'volumev2' try: base.url_for(context.current().service_catalog, service_type) return True except ex.SystemError: return False
def delete_secret(id, ctx=None): """delete a secret from the external key manager :param id: The identifier of the secret to delete :param ctx: The context, and associated authentication, to use with this operation (defaults to the current context) """ if ctx is None: ctx = context.current() key_manager.API().delete(ctx, id)
def store_secret(secret, ctx=None): """store a secret and return its identifier :param secret: The secret to store, this should be a string :param ctx: The context, and associated authentication, to use with this operation (defaults to the current context) """ if ctx is None: ctx = context.current() key = passphrase.Passphrase(secret) return key_manager.API().store(ctx, key)
def check_cinder_exists(): if CONF.cinder.api_version == 2: service_type = 'volumev2' else: service_type = 'volumev3' try: base.url_for(context.current().service_catalog, service_type, endpoint_type=CONF.cinder.endpoint_type) return True except keystone_exceptions.EndpointNotFound: return False
def get_secret(id, ctx=None): """get a secret associated with an id :param id: The identifier of the secret to retrieve :param ctx: The context, and associated authentication, to use with this operation (defaults to the current context) """ if ctx is None: ctx = context.current() key = key_manager.API().get(ctx, id) return key.get_encoded()
def get_neutron_info(self): neutron_info = h.HashableDict() neutron_info['network'] = ( self.instance.node_group.cluster.neutron_management_network) ctx = context.current() neutron_info['uri'] = base.url_for(ctx.service_catalog, 'network') neutron_info['token'] = ctx.token neutron_info['tenant'] = ctx.tenant_name neutron_info['host'] = self.instance.management_ip LOG.debug('Returning neutron info: {0}'.format(neutron_info)) return neutron_info
def client(): ctx = context.current() volume_url = base.url_for(ctx.service_catalog, 'volume') cinder = cinder_client.Client(ctx.username, ctx.token, ctx.tenant_id, volume_url) cinder.client.auth_token = ctx.token cinder.client.management_url = volume_url return cinder
def get_neutron_info(self, instance=None): if not instance: instance = self.instance neutron_info = h.HashableDict() neutron_info['network'] = instance.cluster.neutron_management_network ctx = context.current() neutron_info['uri'] = base.url_for(ctx.service_catalog, 'network') neutron_info['token'] = ctx.auth_token neutron_info['tenant'] = ctx.tenant_name neutron_info['host'] = instance.management_ip LOG.debug('Returning neutron info: {info}'.format(info=neutron_info)) return neutron_info
def client(): ctx = context.current() auth_url = base.retrieve_auth_url() compute_url = base.url_for(ctx.service_catalog, 'compute') nova = nova_client.Client(username=ctx.username, api_key=None, project_id=ctx.tenant_id, auth_url=auth_url) nova.client.auth_token = ctx.token nova.client.management_url = compute_url nova.images = images.SaharaImageManager(nova) return nova
def check_job_execution_cancel(job_id, **kwargs): ctx = context.current() je = conductor.job_execution_get(ctx, job_id) if je.tenant_id != ctx.tenant_id: raise ex.CancelingFailed( _("Job execution with id '%s' cannot be canceled " "because it wasn't created in this tenant") % job_id) if je.is_protected: raise ex.CancelingFailed( _("Job Execution with id '%s' cannot be canceled " "because it's marked as protected") % job_id)
def get_neutron_info(self, instance=None): if not instance: instance = self.instance neutron_info = dict() neutron_info['network'] = instance.cluster.neutron_management_network ctx = context.current() neutron_info['token'] = ctx.auth_token neutron_info['tenant'] = ctx.tenant_name neutron_info['host'] = instance.management_ip log_info = copy.deepcopy(neutron_info) del log_info['token'] LOG.debug('Returning neutron info: {info}'.format(info=log_info)) return neutron_info
def get_neutron_info(self, instance=None): if not instance: instance = self.instance neutron_info = h.HashableDict() neutron_info['network'] = ( instance.node_group.cluster.neutron_management_network) ctx = context.current() neutron_info['uri'] = base.url_for(ctx.service_catalog, 'network') neutron_info['token'] = ctx.auth_token neutron_info['tenant'] = ctx.tenant_name neutron_info['host'] = instance.management_ip LOG.debug('Returning neutron info: {0}'.format(neutron_info)) return neutron_info
def check_job_execution_cancel(job_execution_id, **kwargs): ctx = context.current() je = conductor.job_execution_get(ctx, job_execution_id) if je.tenant_id != ctx.tenant_id: raise ex.CancelingFailed( _("Job execution with id '%s' cannot be canceled " "because it wasn't created in this tenant") % job_execution_id) if je.is_protected: raise ex.CancelingFailed( _("Job Execution with id '%s' cannot be canceled " "because it's marked as protected") % job_execution_id)
def url_for(service_catalog=None, service_type='identity', endpoint_type="internalURL"): if not service_catalog: service_catalog = context.current().service_catalog try: return keystone_service_catalog.ServiceCatalogV2( json.loads(service_catalog)).url_for( service_type=service_type, interface=endpoint_type, region_name=CONF.os_region_name) except keystone_ex.EndpointNotFound: return keystone_service_catalog.ServiceCatalogV3( json.loads(service_catalog)).url_for( service_type=service_type, interface=endpoint_type, region_name=CONF.os_region_name)
def client(): ctx = context.current() auth_url = base.retrieve_auth_url() compute_url = base.url_for(ctx.service_catalog, 'compute') nova = nova_client.Client(username=ctx.username, api_key=None, project_id=ctx.tenant_id, auth_url=auth_url) nova.client.auth_token = ctx.auth_token nova.client.management_url = compute_url nova.images = images.SaharaImageManager(nova) return nova
def delete_trust_from_cluster(cluster): """Delete a trust from a cluster If the cluster has a trust delegated to it, then delete it and set the trust id to None. :param cluster: The cluster to delete the trust from. """ if cluster.trust_id: keystone_client = keystone.client_for_admin_from_trust(cluster.trust_id) delete_trust(keystone_client, cluster.trust_id) ctx = context.current() conductor.cluster_update(ctx, cluster, {"trust_id": None})
def get_neutron_info(self, instance=None): if not instance: instance = self.instance neutron_info = dict() neutron_info['network'] = instance.cluster.neutron_management_network ctx = context.current() neutron_info['token'] = context.get_auth_token() neutron_info['tenant'] = ctx.tenant_name neutron_info['host'] = _get_access_ip(instance) log_info = copy.deepcopy(neutron_info) del log_info['token'] LOG.debug('Returning neutron info: {info}'.format(info=log_info)) return neutron_info
def delete_proxy_user_for_cluster(cluster): '''Delete a proxy user based on a Cluster :param cluster: The cluster model with proxy user information ''' proxy_configs = cluster.cluster_configs.get('proxy_configs') if proxy_configs is not None: proxy_username = proxy_configs.get('proxy_username') key = key_manager.API().get( context.current(), proxy_configs.get('proxy_password')) proxy_password = key.get_encoded() proxy_trust_id = proxy_configs.get('proxy_trust_id') proxy_user = k.auth_for_proxy(proxy_username, proxy_password, proxy_trust_id) t.delete_trust(proxy_user, proxy_trust_id) proxy_user_delete(proxy_username) key_manager.API().delete(context.current(), proxy_configs.get('proxy_password')) update = {'cluster_configs': cluster.cluster_configs.to_dict()} del update['cluster_configs']['proxy_configs'] conductor.cluster_update(context.ctx(), cluster, update)
def client(): ctx = context.current() if CONF.cinder_api_version == 1: volume_url = base.url_for(ctx.service_catalog, 'volume') cinder = cinder_client_v1.Client(ctx.username, ctx.auth_token, ctx.tenant_id, volume_url) else: volume_url = base.url_for(ctx.service_catalog, 'volumev2') cinder = cinder_client_v2.Client(ctx.username, ctx.auth_token, ctx.tenant_id, volume_url) cinder.client.auth_token = ctx.auth_token cinder.client.management_url = volume_url return cinder
def create_trust(cluster): client = keystone.client() ctx = context.current() trustee_id = keystone.client_for_admin(ctx.tenant_id).user_id trust = client.trusts.create(trustor_user=client.user_id, trustee_user=trustee_id, impersonation=True, role_names=ctx.roles, project=client.tenant_id) conductor.cluster_update(ctx, cluster, {'trust_id': trust.id})
def get_configs(self, input_data, output_data, proxy_configs=None): configs = {} if proxy_configs: key = key_manager.API().get(context.current(), proxy_configs.get('proxy_password')) configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') configs[sw.HADOOP_SWIFT_PASSWORD] = key.get_encoded() configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name return configs for src in (input_data, output_data): if src.type == "swift" and hasattr(src, "credentials"): if "user" in src.credentials: configs[sw.HADOOP_SWIFT_USERNAME] = src.credentials['user'] if "password" in src.credentials: key = key_manager.API().get(context.current(), src.credentials['password']) configs[sw.HADOOP_SWIFT_PASSWORD] = key.get_encoded() break return configs
def delete_trust_from_cluster(cluster): '''Delete a trust from a cluster If the cluster has a trust delegated to it, then delete it and set the trust id to None. :param cluster: The cluster to delete the trust from. ''' ctx = context.current() cluster = conductor.cluster_get(ctx, cluster) if CONF.use_identity_api_v3 and cluster.trust_id: keystone_auth = keystone.auth_for_admin(trust_id=cluster.trust_id) delete_trust(keystone_auth, cluster.trust_id) conductor.cluster_update(ctx, cluster, {'trust_id': None})
def delete_trust_from_cluster(cluster): '''Delete a trust from a cluster If the cluster has a trust delegated to it, then delete it and set the trust id to None. :param cluster: The cluster to delete the trust from. ''' if cluster.trust_id: keystone_client = keystone.client_for_admin_from_trust( cluster.trust_id) delete_trust(keystone_client, cluster.trust_id) ctx = context.current() conductor.cluster_update(ctx, cluster, {'trust_id': None})
def get_configs(self, proxy_configs=None): configs = {} if proxy_configs: key = key_manager.API().get(context.current(), proxy_configs.get('proxy_password')) password = key.get_encoded() configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') configs[sw.HADOOP_SWIFT_PASSWORD] = password configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name return configs return configs
def get_os_admin_auth_plugin(cluster): '''Return an admin auth plugin based on the cluster trust id or project If a trust id is available for the cluster, then it is used to create an auth plugin scoped to the trust. If not, the project name from the current context is used to scope the auth plugin. :param cluster: The id of the cluster to use for trust identification. ''' ctx = context.current() cluster = conductor.cluster_get(ctx, cluster) if CONF.use_identity_api_v3 and cluster.trust_id: return keystone.auth_for_admin(trust_id=cluster.trust_id) return keystone.auth_for_admin(project_name=ctx.tenant_name)
def retrieve_auth_url(): """This function returns auth url v2.0 api. Hadoop Swift library doesn't support keystone v3 api. """ info = urlparse.urlparse(context.current().auth_uri) if CONF.use_domain_for_proxy_users: url = 'v3/auth' else: url = 'v2.0' return '{scheme}://{hostname}:{port}/{url}/'.format(scheme=info.scheme, hostname=info.hostname, port=info.port, url=url)
def create_trust_for_cluster(cluster): '''Create a trust for a cluster This delegates a trust from the current user to the Sahara admin user based on the current context roles, and then adds the trust identifier to the cluster object. ''' trustor = keystone.client() ctx = context.current() trustee = keystone.client_for_admin() trust_id = create_trust(trustor=trustor, trustee=trustee, role_names=ctx.roles) conductor.cluster_update(ctx, cluster, {'trust_id': trust_id})
def client(): ctx = context.current() auth_url = base.retrieve_auth_url() if CONF.use_identity_api_v3: keystone = keystone_client_v3.Client(username=ctx.username, token=ctx.token, tenant_id=ctx.tenant_id, auth_url=auth_url) keystone.management_url = auth_url else: keystone = keystone_client.Client(username=ctx.username, token=ctx.token, tenant_id=ctx.tenant_id, auth_url=auth_url) return keystone
def add_provisioning_step(cluster_id, step_name, total): if CONF.disable_event_log or not g.check_cluster_exists(cluster_id): return prev_step = get_current_provisioning_step(cluster_id) if prev_step: conductor.cluster_provision_step_update(context.ctx(), prev_step) step_type = context.ctx().current_instance_info.step_type new_step = conductor.cluster_provision_step_add( context.ctx(), cluster_id, { 'step_name': step_name, 'step_type': step_type, 'total': total, 'started_at': timeutils.utcnow(), }) context.current().current_instance_info.step_id = new_step return new_step
def use_os_admin_auth_token(cluster): '''Set the current context to the admin user's trust scoped token This will configure the current context to the admin user's identity with the cluster's tenant. It will also generate an authentication token based on the admin user and a delegated trust associated with the cluster. :param cluster: The cluster to use for tenant and trust identification. ''' if cluster.trust_id: ctx = context.current() ctx.username = CONF.keystone_authtoken.admin_user ctx.tenant_id = cluster.tenant_id client = keystone.client_for_admin_from_trust(cluster.trust_id) ctx.auth_token = client.auth_token ctx.service_catalog = json.dumps(client.service_catalog.get_data())
def client(): ctx = context.current() args = { 'insecure': CONF.cinder.api_insecure, 'cacert': CONF.cinder.ca_file } if CONF.cinder.api_version == 1: volume_url = base.url_for(ctx.service_catalog, 'volume') cinder = cinder_client_v1.Client(ctx.username, ctx.auth_token, ctx.tenant_id, volume_url, **args) else: volume_url = base.url_for(ctx.service_catalog, 'volumev2') cinder = cinder_client_v2.Client(ctx.username, ctx.auth_token, ctx.tenant_id, volume_url, **args) cinder.client.auth_token = ctx.auth_token cinder.client.management_url = volume_url return cinder
def use_os_admin_auth_token(cluster): '''Set the current context to the admin user's trust scoped token This will configure the current context to the admin user's identity with the cluster's tenant. It will also generate an authentication token based on the admin user and a delegated trust associated with the cluster. :param cluster: The cluster to use for tenant and trust identification. ''' ctx = context.current() cluster = conductor.cluster_get(ctx, cluster) if CONF.use_identity_api_v3 and cluster.trust_id: ctx.username = CONF.keystone_authtoken.admin_user ctx.tenant_id = cluster.tenant_id ctx.auth_plugin = keystone.auth_for_admin(trust_id=cluster.trust_id) ctx.auth_token = context.get_auth_token() ctx.service_catalog = json.dumps( keystone.service_catalog_from_auth(ctx.auth_plugin))
def create_proxy_user_for_job_execution(job_execution): '''Creates a proxy user and adds the credentials to the job execution :param job_execution: The job execution model to update ''' username = '******'.format(job_execution.id) key = passphrase.Passphrase(proxy_user_create(username)) password = key_manager.API().store(context.current(), key) current_user = k.auth() proxy_user = k.auth_for_proxy(username, password) trust_id = t.create_trust(trustor=current_user, trustee=proxy_user, role_names=CONF.proxy_user_role_names) update = {'job_configs': job_execution.job_configs.to_dict()} update['job_configs']['proxy_configs'] = { 'proxy_username': username, 'proxy_password': password, 'proxy_trust_id': trust_id } conductor.job_execution_update(context.ctx(), job_execution, update)
def create_trust_for_cluster(cluster, expires=True): '''Create a trust for a cluster This delegates a trust from the current user to the Sahara admin user based on the current context roles, and then adds the trust identifier to the cluster object. :param expires: The trust will expire if this is set to True. ''' ctx = context.current() cluster = conductor.cluster_get(ctx, cluster) if CONF.use_identity_api_v3 and not cluster.trust_id: trustor = keystone.auth() trustee = keystone.auth_for_admin( project_name=CONF.keystone_authtoken.admin_tenant_name) trust_id = create_trust(trustor=trustor, trustee=trustee, role_names=ctx.roles, allow_redelegation=True) conductor.cluster_update(ctx, cluster, {'trust_id': trust_id})
def execute_job(job_id, data): # Elements common to all job types cluster_id = data['cluster_id'] configs = data.get('job_configs', {}) ctx = context.current() cluster = conductor.cluster_get(ctx, cluster_id) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) instance = plugin.get_oozie_server(cluster) extra = {} info = None if CONF.use_namespaces and not CONF.use_floating_ips: info = instance.remote().get_neutron_info() extra['neutron'] = info # Not in Java job types but present for all others input_id = data.get('input_id', None) output_id = data.get('output_id', None) # Since we will use a unified class in the database, we pass # a superset for all job types job_ex_dict = { 'input_id': input_id, 'output_id': output_id, 'job_id': job_id, 'cluster_id': cluster_id, 'info': { 'status': 'Pending' }, 'job_configs': configs, 'extra': extra } job_execution = conductor.job_execution_create(context.ctx(), job_ex_dict) context.spawn("Starting Job Execution %s" % job_execution.id, manager.run_job, job_execution) return job_execution
def _build_proxy_command(self, command, instance=None, port=None, info=None, rootwrap_command=None): # Accepted keywords in the proxy command template: # {host}, {port}, {tenant_id}, {network_id}, {router_id} keywords = {} if not info: info = self.get_neutron_info(instance) keywords['tenant_id'] = context.current().tenant_id keywords['network_id'] = info['network'] # Query Neutron only if needed if '{router_id}' in command: auth = trusts.get_os_admin_auth_plugin(instance.cluster) client = neutron.NeutronClient(info['network'], info['token'], info['tenant'], auth=auth) keywords['router_id'] = client.get_router() keywords['host'] = instance.management_ip keywords['port'] = port try: command = command.format(**keywords) except KeyError as e: LOG.error( _LE('Invalid keyword in proxy_command: {result}').format( result=e)) # Do not give more details to the end-user raise ex.SystemError('Misconfiguration') if rootwrap_command: command = '{0} {1}'.format(rootwrap_command, command) return command
def create_proxy_user_for_cluster(cluster): '''Creates a proxy user and adds the credentials to the cluster :param cluster: The cluster model to update ''' if cluster.cluster_configs.get('proxy_configs'): return cluster username = '******'.format(cluster.id) key = passphrase.Passphrase(proxy_user_create(username)) password = key_manager.API().store(context.current(), key) current_user = k.auth() proxy_user = k.auth_for_proxy(username, password) trust_id = t.create_trust(trustor=current_user, trustee=proxy_user, role_names=CONF.proxy_user_role_names) update = {'cluster_configs': cluster.cluster_configs.to_dict()} update['cluster_configs']['proxy_configs'] = { 'proxy_username': username, 'proxy_password': password, 'proxy_trust_id': trust_id } return conductor.cluster_update(context.ctx(), cluster, update)
def get_swift_configs(): configs = x.load_hadoop_xml_defaults('swift/resources/conf-template.xml') for conf in configs: if conf['name'] == HADOOP_SWIFT_AUTH_URL: conf['value'] = su.retrieve_auth_url() + "auth/tokens/" if conf['name'] == HADOOP_SWIFT_TENANT: conf['value'] = retrieve_tenant() if CONF.os_region_name and conf['name'] == HADOOP_SWIFT_REGION: conf['value'] = CONF.os_region_name if conf['name'] == HADOOP_SWIFT_DOMAIN_NAME: # NOTE(jfreud): Don't be deceived here... Even though there is an # attribute provided by context called domain_name, it is used for # domain scope, and hadoop-swiftfs always authenticates using # project scope. The purpose of the setting below is to override # the default value for project domain and user domain, domain id # as 'default', which may not always be correct. # TODO(jfreud): When hadoop-swiftfs allows it, stop hoping that # project_domain_name is always equal to user_domain_name. conf['value'] = context.current().project_domain_name result = [cfg for cfg in configs if cfg['value']] LOG.info("Swift would be integrated with the following " "params: {result}".format(result=result)) return result
def _upload_wrapper_xml(self, where, job_dir, job_configs): xml_name = 'spark.xml' proxy_configs = job_configs.get('proxy_configs') configs = {} if proxy_configs: key = key_manager.API().get(context.current(), proxy_configs.get('proxy_password')) password = key.get_encoded() configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') configs[sw.HADOOP_SWIFT_PASSWORD] = password configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name else: cfgs = job_configs.get('configs', {}) targets = [sw.HADOOP_SWIFT_USERNAME, sw.HADOOP_SWIFT_PASSWORD] configs = {k: cfgs[k] for k in targets if k in cfgs} content = xmlutils.create_hadoop_xml(configs) with remote.get_remote(where) as r: dst = os.path.join(job_dir, xml_name) r.write_file_to(dst, content) return xml_name
def _release_remote_semaphore(): _global_remote_semaphore.release() context.current().remote_semaphore.release()
def _acquire_remote_semaphore(): context.current().remote_semaphore.acquire() _global_remote_semaphore.acquire()
def call(self, name, **kwargs): ctx = context.current() return self.__client.call(ctx.to_dict(), name, **kwargs)