def _enable_kerberos(self): # Update Kerberos configuration self.cm_api.update_config( message='Updating Kerberos config', body=cm_client.ApiConfigList([ cm_client.ApiConfig(name='KDC_ADMIN_HOST', value='edge2ai-1.dim.local'), cm_client.ApiConfig(name='KDC_HOST', value='edge2ai-1.dim.local'), cm_client.ApiConfig(name='KDC_TYPE', value='MIT KDC'), cm_client.ApiConfig(name='KRB_AUTH_ENABLE', value='true'), cm_client.ApiConfig(name='KRB_ENC_TYPES', value='aes256-cts rc4-hmac'), cm_client.ApiConfig(name='PUBLIC_CLOUD_STATUS', value='ON_PUBLIC_CLOUD'), cm_client.ApiConfig(name='SECURITY_REALM', value='WORKSHOP.COM'), ])) # Import Kerberos credentials cmd = self.cm_api.import_admin_credentials(password=self.krb_pass, username=self.krb_princ) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to import admin credentials')
def _enable_tls(self): # Update TLS configuration self.cm_api.update_config( message='Updating TLS config', body=cm_client.ApiConfigList([ cm_client.ApiConfig(name='AGENT_TLS', value='true'), cm_client.ApiConfig(name='KEYSTORE_PASSWORD', value=the_pwd()), cm_client.ApiConfig( name='KEYSTORE_PATH', value='/opt/cloudera/security/jks/keystore.jks'), cm_client.ApiConfig(name='NEED_AGENT_VALIDATION', value='true'), cm_client.ApiConfig(name='SCM_PROXY_TIMEOUT', value='30000'), cm_client.ApiConfig(name='TRUSTSTORE_PASSWORD', value=the_pwd()), cm_client.ApiConfig( name='TRUSTSTORE_PATH', value='/opt/cloudera/security/jks/truststore.jks'), cm_client.ApiConfig(name='WEB_TLS', value='true'), ])) self.mgmt_api.update_service_config( message='Updating TLS config for Mgmt Services', body=cm_client.ApiServiceConfig([ cm_client.ApiConfig( name='ssl_client_truststore_location', value='/opt/cloudera/security/jks/truststore.jks'), cm_client.ApiConfig(name='ssl_client_truststore_password', value=the_pwd()), ]))
def loadLocalRepo(): new_parcel_repo_urls = parcelsuri cm_api_instance = cm_client.ClouderaManagerResourceApi(api_client) cm_configs = cm_api_instance.get_config(view='full') new_cm_config = cm_client.ApiConfig(name='REMOTE_PARCEL_REPO_URLS', value=new_parcel_repo_urls) new_cm_configs = cm_client.ApiConfigList([new_cm_config]) updated_cm_configs = cm_api_instance.update_config(body=new_cm_configs)
def _enable_kerberos(self): # Update Kerberos configuration self.cm_api.update_config(message='Updating Kerberos config', body=cm_client.ApiConfigList([ cm_client.ApiConfig(name='KDC_ADMIN_HOST', value='edge2ai-1.dim.local'), cm_client.ApiConfig(name='KDC_HOST', value='edge2ai-1.dim.local'), cm_client.ApiConfig(name='KDC_TYPE', value='MIT KDC'), cm_client.ApiConfig(name='KRB_ENC_TYPES', value='aes256-cts rc4-hmac'), cm_client.ApiConfig(name='PUBLIC_CLOUD_STATUS', value='ON_PUBLIC_CLOUD'), cm_client.ApiConfig(name='SECURITY_REALM', value='WORKSHOP.COM'), ])) # Import Kerberos credentials cmd = self.cm_api.import_admin_credentials(password=self.krb_pass, username=self.krb_princ) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to import admin credentials') # Configure Kerberos for the cluster cluster_name = 'OneNodeCluster' cmd = self.cluster_api.configure_for_kerberos(cluster_name, body=cm_client.ApiConfigureForKerberosArguments(datanode_transceiver_port=1004, datanode_web_port=1006)) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to configure services for Kerberos') # Stop cluster cmd = self.cluster_api.stop_command(cluster_name) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to stop cluster') # Stop Mgmt Services cmd = self.mgmt_api.stop_command() cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to stop management services') # Start Mgmt Services cmd = self.mgmt_api.start_command() cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to start management services') # Start cluster cmd = self.cluster_api.start_command(cluster_name) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to start cluster') # Deploy client config cmd = self.cluster_api.deploy_client_config(cluster_name) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to deploy client config')
def _reset_paywall_credentials(self): if cm_major_version() >= 7: try: self.cm_api.update_config(message='Importing paywall credentials', body=cm_client.ApiConfigList([ cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_USER', value=None), cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_PASSWORD', value=None) ]) ) except ApiException: pass
def _import_paywall_credentials(self): if cm_major_version() >= 7: configs = [] if 'REMOTE_REPO_USR' in os.environ and os.environ['REMOTE_REPO_USR']: paywall_usr = os.environ['REMOTE_REPO_USR'] configs.append(cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_USER', value=paywall_usr)) if 'REMOTE_REPO_PWD' in os.environ and os.environ['REMOTE_REPO_PWD']: paywall_pwd = os.environ['REMOTE_REPO_PWD'] configs.append(cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_PASSWORD', value=paywall_pwd)) try: if configs: self.cm_api.update_config(message='Importing paywall credentials', body=cm_client.ApiConfigList(configs)) except ApiException: pass
def appendRemoteRepo(): new_parcel_repo_urls = parcelsuri cm_api_instance = cm_client.ClouderaManagerResourceApi(api_client) cm_configs = cm_api_instance.get_config(view='full') old_parcel_repo_urls = None for cm_config in cm_configs.items: if cm_config.name == 'REMOTE_PARCEL_REPO_URLS': old_parcel_repo_urls = cm_config.value new_parcel_repo_urls = old_parcel_repo_urls + ',' + parcelsuri cm_api_instance = cm_client.ClouderaManagerResourceApi(api_client) cm_configs = cm_api_instance.get_config(view='full') new_cm_config = cm_client.ApiConfig(name='REMOTE_PARCEL_REPO_URLS', value=new_parcel_repo_urls) new_cm_configs = cm_client.ApiConfigList([new_cm_config]) updated_cm_configs = cm_api_instance.update_config(body=new_cm_configs)
def _import_paywall_credentials(self): configs = [] if 'REMOTE_REPO_USR' in os.environ and os.environ['REMOTE_REPO_USR']: paywall_usr = os.environ['REMOTE_REPO_USR'] configs.append( cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_USER', value=paywall_usr)) if 'REMOTE_REPO_PWD' in os.environ and os.environ['REMOTE_REPO_PWD']: paywall_pwd = os.environ['REMOTE_REPO_PWD'] configs.append( cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_PASSWORD', value=paywall_pwd)) # This only works for CM 7 and above. We catch the exception and ignore it for CM 6 # TODO: Check version and only set it if needed try: if configs: self.cm_api.update_config( message='Importing paywall credentials', body=cm_client.ApiConfigList(configs)) except ApiException: pass
host_names=['YourHostname'], user_name='root', private_key=key, cm_repo_url='https://archive.cloudera.com/cm6/6.3.0', java_install_strategy='NONE', ssh_port=22, passphrase='') cmd = cm_api.host_install_command(body=instargs) wait(cmd) # Configure Hosts with property needed by SMM host_api = cm_client.AllHostsResourceApi(api_client) message = 'updating CM Agent safety valve for SMM' body = cm_client.ApiConfigList( ) # ApiConfigList | Configuration changes. (optional) body.items = [ cm_client.ApiConfig( name="host_agent_safety_valve", value="kafka_broker_topic_partition_metrics_for_smm_enabled=true") ] cmd = host_api.update_config(message=message, body=body) # create MGMT/CMS mgmt_api = cm_client.MgmtServiceResourceApi(api_client) api_service = cm_client.ApiService() api_service.roles = [ cm_client.ApiRole(type='SERVICEMONITOR'), cm_client.ApiRole(type='HOSTMONITOR'),
def create_cluster(self): # accept trial licence try: self.cm_api.begin_trial() except ApiException as exc: if exc.status == 400 and 'Trial has been used' in exc.body: pass # This can be ignored else: raise # Install CM Agent on host with open(self.key_file, "r") as f: key = f.read() self._import_paywall_credentials() instargs = cm_client.ApiHostInstallArguments( host_names=[self.host], user_name='root', private_key=key, cm_repo_url=self.cm_repo_url, java_install_strategy='NONE', ssh_port=22, passphrase='') cmd = self.cm_api.host_install_command(body=instargs) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to add host to the cluster') # create MGMT/CMS api_service = cm_client.ApiService() api_service.roles = [ cm_client.ApiRole(type='SERVICEMONITOR'), cm_client.ApiRole(type='HOSTMONITOR'), cm_client.ApiRole(type='EVENTSERVER'), cm_client.ApiRole(type='ALERTPUBLISHER') ] self.mgmt_api.auto_assign_roles() # needed? self.mgmt_api.auto_configure() # needed? self.mgmt_api.setup_cms(body=api_service) cmd = self.mgmt_api.start_command() cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to start Management Services') # Update host-level parameter required by SMM self.all_hosts_api.update_config( message='Updating parameter for SMM', body=cm_client.ApiConfigList([ cm_client.ApiConfig( name='host_agent_safety_valve', value= 'kafka_broker_topic_partition_metrics_for_smm_enabled=true' ) ])) # create the cluster using the template with open(self.template) as f: json_str = f.read() Response = namedtuple("Response", "data") dst_cluster_template = self.api_client.deserialize( response=Response(json_str), response_type=cm_client.ApiClusterTemplate) cmd = self.cm_api.import_cluster_template(add_repositories=True, body=dst_cluster_template) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to deploy cluster template') # All parcel downloads should've already been done at this point, so we can safely remove the paywall credentials self._reset_paywall_credentials() if self.use_kerberos: self._enable_kerberos()
def setupCMS(): service_name = 'mgmt' service_type = 'MGMT'.upper() api_instance = cm_client.MgmtServiceResourceApi(api_client) services_instance = cm_client.MgmtServiceResourceApi(api_client) mgmt_role_instance = cm_client.MgmtRolesResourceApi(api_client) rcg_instance = cm_client.MgmtRoleConfigGroupsResourceApi(api_client) services_instance.setup_cms( body=cm_client.ApiService(name=service_name, type=service_type, display_name='Cloudera Management Service')) host_id = getattr(get_host_resource(hostname), 'host_id', None) role_list = [] for role_type in [ "REPORTSMANAGER", "EVENTSERVER", "HOSTMONITOR", "ALERTPUBLISHER", "SERVICEMONITOR" ]: role_name = service_name + "-" + role_type + "-" + hashlib.md5( hostname.encode('utf-8')).hexdigest() role_list.append({ "name": role_name, "type": role_type, "hostRef": { "hostId": host_id } }) body = cm_client.ApiRoleList(role_list) api_response = mgmt_role_instance.create_roles(body=body) api_response = rcg_instance.read_role_config_groups() for rcg in api_response.items: if rcg.role_type == 'REPORTSMANAGER': rcg_instance.update_config(rcg.name, message=None, body=cm_client.ApiConfigList([{ 'name': 'headlamp_database_host', 'value': hostname }, { 'name': 'headlamp_database_name', 'value': 'rman' }, { 'name': 'headlamp_database_user', 'value': 'rman' }, { 'name': 'headlamp_database_password', 'value': 'supersecret1' }, { 'name': 'headlamp_database_type', 'value': 'postgresql' }])) services_instance.start_command()
api_url = api_host + ':' + port + '/api/' + api_version api_client = cm_client.ApiClient(api_url) role_config_group_resource = cm_client.RoleConfigGroupsResourceApi(api_client) node_manager_base_config = role_config_group_resource.read_config( "cluster", "yarn-NODEMANAGER-BASE", "yarn") #for configItem in node_manager_base_config.items: # print(configItem.name + ": " + configItem.value); #new_config = cm_client.ApiConfig(name="yarn_nodemanager_resource_memory_mb", value="262144") new_config = cm_client.ApiConfig(name=yarn - key, value=yarn - val) new_config_list = cm_client.ApiConfigList([new_config]) res = role_config_group_resource.update_config("cluster", "yarn-NODEMANAGER-BASE", "yarn", message="", body=new_config_list) res = role_config_group_resource.update_config("cluster", "yarn-NODEMANAGER-1", "yarn", message="", body=new_config_list) res = role_config_group_resource.update_config("cluster", "yarn-NODEMANAGER-2", "yarn", message="", body=new_config_list)
def _enable_kerberos(self, kerberos_type, ipa_host): # Update Kerberos configuration config = [ cm_client.ApiConfig(name='KRB_AUTH_ENABLE', value='true'), cm_client.ApiConfig(name='KRB_ENC_TYPES', value='aes256-cts rc4-hmac'), cm_client.ApiConfig(name='PUBLIC_CLOUD_STATUS', value='ON_PUBLIC_CLOUD'), cm_client.ApiConfig(name='SECURITY_REALM', value='WORKSHOP.COM'), ] if kerberos_type == 'MIT': config += [ cm_client.ApiConfig(name='KDC_ADMIN_HOST', value=local_hostname()), cm_client.ApiConfig(name='KDC_HOST', value=local_hostname()), cm_client.ApiConfig(name='KDC_TYPE', value='MIT KDC'), ] else: config += [ cm_client.ApiConfig(name='KDC_ADMIN_HOST', value=ipa_host), cm_client.ApiConfig(name='KDC_HOST', value=ipa_host), cm_client.ApiConfig(name='KDC_TYPE', value='Red Hat IPA'), cm_client.ApiConfig(name='AUTH_BACKEND_ORDER', value='LDAP_THEN_DB'), cm_client.ApiConfig( name='LDAP_BIND_DN', value= 'uid=ldap_bind_user,cn=users,cn=accounts,dc=workshop,dc=com' ), cm_client.ApiConfig(name='LDAP_BIND_PW', value=the_pwd()), cm_client.ApiConfig( name='LDAP_GROUP_SEARCH_BASE', value='cn=groups,cn=accounts,dc=workshop,dc=com'), cm_client.ApiConfig(name='LDAP_GROUP_SEARCH_FILTER', value='(member={0})'), cm_client.ApiConfig(name='LDAP_TYPE', value='LDAP'), cm_client.ApiConfig(name='LDAP_URL', value='ldaps://' + ipa_host), cm_client.ApiConfig( name='LDAP_USER_SEARCH_BASE', value='cn=users,cn=accounts,dc=workshop,dc=com'), cm_client.ApiConfig(name='LDAP_USER_SEARCH_FILTER', value='(uid={0})'), ] if cm_version() < [7, 5, 3]: # These properties were removed in OPSAPS-61384 (CM 7.5.3) config += [ cm_client.ApiConfig( name='LDAP_BIND_DN_MONITORING', value= 'uid=ldap_bind_user,cn=users,cn=accounts,dc=workshop,dc=com' ), cm_client.ApiConfig(name='LDAP_BIND_PW_MONITORING', value=the_pwd()), ] self.cm_api.update_config(message='Updating Kerberos config', body=cm_client.ApiConfigList(config)) # Import Kerberos credentials cmd = self.cm_api.import_admin_credentials(password=the_pwd(), username=self.krb_princ) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to import admin credentials')
def setup_cm(self, key_file, cm_repo_url, use_kerberos, use_tls, kerberos_type, ipa_host): # Accept trial licence try: self.cm_api.begin_trial() except ApiException as exc: if exc.status == 400 and 'Trial has been used' in exc.body: pass # This can be ignored else: raise # Install CM Agent on host with open(key_file, "r") as f: key = f.read() if self.host not in [ h.hostname for h in self.hosts_api.read_hosts().items ]: instargs = cm_client.ApiHostInstallArguments( host_names=[self.host], user_name='root', private_key=key, cm_repo_url=cm_repo_url, java_install_strategy='NONE', ssh_port=22, passphrase='') cmd = self.cm_api.host_install_command(body=instargs) cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to add host to the cluster') # Create MGMT/CMS try: self.mgmt_api.read_service() print("Cloudera Management Services already installed") cms_exists = True except cm_client.rest.ApiException as e: cms_exists = False if not cms_exists: print("Installing Cloudera Management Services") api_service = cm_client.ApiService() api_service.roles = [ cm_client.ApiRole(type='SERVICEMONITOR'), cm_client.ApiRole(type='HOSTMONITOR'), cm_client.ApiRole(type='EVENTSERVER'), cm_client.ApiRole(type='ALERTPUBLISHER') ] self.mgmt_api.setup_cms(body=api_service) cmd = self.mgmt_api.start_command() cmd = self.wait(cmd) if not cmd.success: raise RuntimeError('Failed to start Management Services') # Update cluster banner c_id = cluster_id() banner = 'Cluster ID: {}, Host: {}'.format(c_id, socket.gethostname()) header_color = HEADER_COLORS[c_id % len(HEADER_COLORS)] self.cm_api.update_config( message='Customizing CM header and banner', body=cm_client.ApiConfigList([ cm_client.ApiConfig(name='CUSTOM_BANNER_HTML', value=banner), cm_client.ApiConfig(name='CUSTOM_HEADER_COLOR', value=header_color), ])) # Update host-level parameter required by SMM self.all_hosts_api.update_config( message='Updating parameter for SMM', body=cm_client.ApiConfigList([ cm_client.ApiConfig( name='host_agent_safety_valve', value= 'kafka_broker_topic_partition_metrics_for_smm_enabled=true' ) ])) # Enable kerberos if use_kerberos: self._enable_kerberos(kerberos_type, ipa_host) # Enable TLS if use_tls: self._enable_tls() # Restart Mgmt Services cmd = self.mgmt_api.restart_command() cmd = self.wait(cmd)
print("Exception when calling ClouderaManagerResourceApi->import_cluster_template: %s\n" % e) cm_client.configuration.username = '******' cm_client.configuration.password = '******' api_client = cm_client.ApiClient("http://localhost:7180/api/v40") cm_api = cm_client.ClouderaManagerResourceApi(api_client) # accept trial licence cm_api.begin_trial() # Update Cloudera Manager config for KRB body = cm_client.ApiConfigList() body.items=[ cm_client.ApiConfig(name='KDC_HOST', value='YourHostname'), cm_client.ApiConfig(name='KDC_ADMIN_HOST', value='YourHostname'), cm_client.ApiConfig(name='KDC_TYPE', value='MIT KDC'), cm_client.ApiConfig(name='KRB_ENC_TYPES', value='aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 arcfour-hmac-md5'), cm_client.ApiConfig(name='SECURITY_REALM', value='CLOUDERA.COM'), cm_client.ApiConfig(name='KRB_MANAGE_KRB5_CONF', value='true') ] api_response = cm_api.update_config(message="KRB", body=body) # Import KDC admin credentials cmd = cm_api.import_admin_credentials(password='******', username='******') wait(cmd)
mgmt_api.auto_assign_roles() # needed? mgmt_api.auto_configure() # needed? mgmt_api.setup_cms(body=api_service) cmd = mgmt_api.start_command() cmd = wait(cmd) if not cmd.success: raise RuntimeError('Failed to start Management Services') # Update host-level parameter required by SMM all_hosts_api = cm_client.AllHostsResourceApi(api_client) all_hosts_api.update_config( message='Updating parameter for SMM', body=cm_client.ApiConfigList([ cm_client.ApiConfig( name='host_agent_safety_valve', value='kafka_broker_topic_partition_metrics_for_smm_enabled=true') ])) # create the cluster using the template with open(TEMPLATE) as f: json_str = f.read() Response = namedtuple("Response", "data") dst_cluster_template = api_client.deserialize( response=Response(json_str), response_type=cm_client.ApiClusterTemplate) cmd = cm_api.import_cluster_template(add_repositories=True, body=dst_cluster_template) cmd = wait(cmd) if not cmd.success: raise RuntimeError('Failed to deploy cluster template')