def _enable_kerberos(self):
# Update Kerberos configuration
self.cm_api.update_config(
message='Updating Kerberos config',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(name='KDC_ADMIN_HOST',
value='edge2ai-1.dim.local'),
cm_client.ApiConfig(name='KDC_HOST',
value='edge2ai-1.dim.local'),
cm_client.ApiConfig(name='KDC_TYPE', value='MIT KDC'),
cm_client.ApiConfig(name='KRB_AUTH_ENABLE', value='true'),
cm_client.ApiConfig(name='KRB_ENC_TYPES',
value='aes256-cts rc4-hmac'),
cm_client.ApiConfig(name='PUBLIC_CLOUD_STATUS',
value='ON_PUBLIC_CLOUD'),
cm_client.ApiConfig(name='SECURITY_REALM',
value='WORKSHOP.COM'),
]))
# Import Kerberos credentials
cmd = self.cm_api.import_admin_credentials(password=self.krb_pass,
username=self.krb_princ)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to import admin credentials')
def _enable_tls(self):
# Update TLS configuration
self.cm_api.update_config(
message='Updating TLS config',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(name='AGENT_TLS', value='true'),
cm_client.ApiConfig(name='KEYSTORE_PASSWORD', value=the_pwd()),
cm_client.ApiConfig(
name='KEYSTORE_PATH',
value='/opt/cloudera/security/jks/keystore.jks'),
cm_client.ApiConfig(name='NEED_AGENT_VALIDATION',
value='true'),
cm_client.ApiConfig(name='SCM_PROXY_TIMEOUT', value='30000'),
cm_client.ApiConfig(name='TRUSTSTORE_PASSWORD',
value=the_pwd()),
cm_client.ApiConfig(
name='TRUSTSTORE_PATH',
value='/opt/cloudera/security/jks/truststore.jks'),
cm_client.ApiConfig(name='WEB_TLS', value='true'),
]))
self.mgmt_api.update_service_config(
message='Updating TLS config for Mgmt Services',
body=cm_client.ApiServiceConfig([
cm_client.ApiConfig(
name='ssl_client_truststore_location',
value='/opt/cloudera/security/jks/truststore.jks'),
cm_client.ApiConfig(name='ssl_client_truststore_password',
value=the_pwd()),
]))
def loadLocalRepo():
new_parcel_repo_urls = parcelsuri
cm_api_instance = cm_client.ClouderaManagerResourceApi(api_client)
cm_configs = cm_api_instance.get_config(view='full')
new_cm_config = cm_client.ApiConfig(name='REMOTE_PARCEL_REPO_URLS',
value=new_parcel_repo_urls)
new_cm_configs = cm_client.ApiConfigList([new_cm_config])
updated_cm_configs = cm_api_instance.update_config(body=new_cm_configs)
def _enable_kerberos(self):
# Update Kerberos configuration
self.cm_api.update_config(message='Updating Kerberos config',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(name='KDC_ADMIN_HOST', value='edge2ai-1.dim.local'),
cm_client.ApiConfig(name='KDC_HOST', value='edge2ai-1.dim.local'),
cm_client.ApiConfig(name='KDC_TYPE', value='MIT KDC'),
cm_client.ApiConfig(name='KRB_ENC_TYPES', value='aes256-cts rc4-hmac'),
cm_client.ApiConfig(name='PUBLIC_CLOUD_STATUS', value='ON_PUBLIC_CLOUD'),
cm_client.ApiConfig(name='SECURITY_REALM', value='WORKSHOP.COM'),
]))
# Import Kerberos credentials
cmd = self.cm_api.import_admin_credentials(password=self.krb_pass, username=self.krb_princ)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to import admin credentials')
# Configure Kerberos for the cluster
cluster_name = 'OneNodeCluster'
cmd = self.cluster_api.configure_for_kerberos(cluster_name, body=cm_client.ApiConfigureForKerberosArguments(datanode_transceiver_port=1004, datanode_web_port=1006))
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to configure services for Kerberos')
# Stop cluster
cmd = self.cluster_api.stop_command(cluster_name)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to stop cluster')
# Stop Mgmt Services
cmd = self.mgmt_api.stop_command()
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to stop management services')
# Start Mgmt Services
cmd = self.mgmt_api.start_command()
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to start management services')
# Start cluster
cmd = self.cluster_api.start_command(cluster_name)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to start cluster')
# Deploy client config
cmd = self.cluster_api.deploy_client_config(cluster_name)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to deploy client config')
def _reset_paywall_credentials(self):
if cm_major_version() >= 7:
try:
self.cm_api.update_config(message='Importing paywall credentials',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_USER', value=None),
cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_PASSWORD', value=None)
])
)
except ApiException:
pass
def _import_paywall_credentials(self):
if cm_major_version() >= 7:
configs = []
if 'REMOTE_REPO_USR' in os.environ and os.environ['REMOTE_REPO_USR']:
paywall_usr = os.environ['REMOTE_REPO_USR']
configs.append(cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_USER', value=paywall_usr))
if 'REMOTE_REPO_PWD' in os.environ and os.environ['REMOTE_REPO_PWD']:
paywall_pwd = os.environ['REMOTE_REPO_PWD']
configs.append(cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_PASSWORD', value=paywall_pwd))
try:
if configs:
self.cm_api.update_config(message='Importing paywall credentials',
body=cm_client.ApiConfigList(configs))
except ApiException:
pass
def appendRemoteRepo():
new_parcel_repo_urls = parcelsuri
cm_api_instance = cm_client.ClouderaManagerResourceApi(api_client)
cm_configs = cm_api_instance.get_config(view='full')
old_parcel_repo_urls = None
for cm_config in cm_configs.items:
if cm_config.name == 'REMOTE_PARCEL_REPO_URLS':
old_parcel_repo_urls = cm_config.value
new_parcel_repo_urls = old_parcel_repo_urls + ',' + parcelsuri
cm_api_instance = cm_client.ClouderaManagerResourceApi(api_client)
cm_configs = cm_api_instance.get_config(view='full')
new_cm_config = cm_client.ApiConfig(name='REMOTE_PARCEL_REPO_URLS',
value=new_parcel_repo_urls)
new_cm_configs = cm_client.ApiConfigList([new_cm_config])
updated_cm_configs = cm_api_instance.update_config(body=new_cm_configs)
def _import_paywall_credentials(self):
configs = []
if 'REMOTE_REPO_USR' in os.environ and os.environ['REMOTE_REPO_USR']:
paywall_usr = os.environ['REMOTE_REPO_USR']
configs.append(
cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_USER',
value=paywall_usr))
if 'REMOTE_REPO_PWD' in os.environ and os.environ['REMOTE_REPO_PWD']:
paywall_pwd = os.environ['REMOTE_REPO_PWD']
configs.append(
cm_client.ApiConfig(name='REMOTE_REPO_OVERRIDE_PASSWORD',
value=paywall_pwd))
# This only works for CM 7 and above. We catch the exception and ignore it for CM 6
# TODO: Check version and only set it if needed
try:
if configs:
self.cm_api.update_config(
message='Importing paywall credentials',
body=cm_client.ApiConfigList(configs))
except ApiException:
pass
host_names=['YourHostname'],
user_name='root',
private_key=key,
cm_repo_url='https://archive.cloudera.com/cm6/6.3.0',
java_install_strategy='NONE',
ssh_port=22,
passphrase='')
cmd = cm_api.host_install_command(body=instargs)
wait(cmd)
# Configure Hosts with property needed by SMM
host_api = cm_client.AllHostsResourceApi(api_client)
message = 'updating CM Agent safety valve for SMM'
body = cm_client.ApiConfigList(
) # ApiConfigList | Configuration changes. (optional)
body.items = [
cm_client.ApiConfig(
name="host_agent_safety_valve",
value="kafka_broker_topic_partition_metrics_for_smm_enabled=true")
]
cmd = host_api.update_config(message=message, body=body)
# create MGMT/CMS
mgmt_api = cm_client.MgmtServiceResourceApi(api_client)
api_service = cm_client.ApiService()
api_service.roles = [
cm_client.ApiRole(type='SERVICEMONITOR'),
cm_client.ApiRole(type='HOSTMONITOR'),
def create_cluster(self):
# accept trial licence
try:
self.cm_api.begin_trial()
except ApiException as exc:
if exc.status == 400 and 'Trial has been used' in exc.body:
pass # This can be ignored
else:
raise
# Install CM Agent on host
with open(self.key_file, "r") as f:
key = f.read()
self._import_paywall_credentials()
instargs = cm_client.ApiHostInstallArguments(
host_names=[self.host],
user_name='root',
private_key=key,
cm_repo_url=self.cm_repo_url,
java_install_strategy='NONE',
ssh_port=22,
passphrase='')
cmd = self.cm_api.host_install_command(body=instargs)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to add host to the cluster')
# create MGMT/CMS
api_service = cm_client.ApiService()
api_service.roles = [
cm_client.ApiRole(type='SERVICEMONITOR'),
cm_client.ApiRole(type='HOSTMONITOR'),
cm_client.ApiRole(type='EVENTSERVER'),
cm_client.ApiRole(type='ALERTPUBLISHER')
]
self.mgmt_api.auto_assign_roles() # needed?
self.mgmt_api.auto_configure() # needed?
self.mgmt_api.setup_cms(body=api_service)
cmd = self.mgmt_api.start_command()
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to start Management Services')
# Update host-level parameter required by SMM
self.all_hosts_api.update_config(
message='Updating parameter for SMM',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(
name='host_agent_safety_valve',
value=
'kafka_broker_topic_partition_metrics_for_smm_enabled=true'
)
]))
# create the cluster using the template
with open(self.template) as f:
json_str = f.read()
Response = namedtuple("Response", "data")
dst_cluster_template = self.api_client.deserialize(
response=Response(json_str),
response_type=cm_client.ApiClusterTemplate)
cmd = self.cm_api.import_cluster_template(add_repositories=True,
body=dst_cluster_template)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to deploy cluster template')
# All parcel downloads should've already been done at this point, so we can safely remove the paywall credentials
self._reset_paywall_credentials()
if self.use_kerberos:
self._enable_kerberos()
def setupCMS():
service_name = 'mgmt'
service_type = 'MGMT'.upper()
api_instance = cm_client.MgmtServiceResourceApi(api_client)
services_instance = cm_client.MgmtServiceResourceApi(api_client)
mgmt_role_instance = cm_client.MgmtRolesResourceApi(api_client)
rcg_instance = cm_client.MgmtRoleConfigGroupsResourceApi(api_client)
services_instance.setup_cms(
body=cm_client.ApiService(name=service_name,
type=service_type,
display_name='Cloudera Management Service'))
host_id = getattr(get_host_resource(hostname), 'host_id', None)
role_list = []
for role_type in [
"REPORTSMANAGER", "EVENTSERVER", "HOSTMONITOR", "ALERTPUBLISHER",
"SERVICEMONITOR"
]:
role_name = service_name + "-" + role_type + "-" + hashlib.md5(
hostname.encode('utf-8')).hexdigest()
role_list.append({
"name": role_name,
"type": role_type,
"hostRef": {
"hostId": host_id
}
})
body = cm_client.ApiRoleList(role_list)
api_response = mgmt_role_instance.create_roles(body=body)
api_response = rcg_instance.read_role_config_groups()
for rcg in api_response.items:
if rcg.role_type == 'REPORTSMANAGER':
rcg_instance.update_config(rcg.name,
message=None,
body=cm_client.ApiConfigList([{
'name':
'headlamp_database_host',
'value':
hostname
}, {
'name':
'headlamp_database_name',
'value':
'rman'
}, {
'name':
'headlamp_database_user',
'value':
'rman'
}, {
'name':
'headlamp_database_password',
'value':
'supersecret1'
}, {
'name':
'headlamp_database_type',
'value':
'postgresql'
}]))
services_instance.start_command()
api_url = api_host + ':' + port + '/api/' + api_version
api_client = cm_client.ApiClient(api_url)
role_config_group_resource = cm_client.RoleConfigGroupsResourceApi(api_client)
node_manager_base_config = role_config_group_resource.read_config(
"cluster", "yarn-NODEMANAGER-BASE", "yarn")
#for configItem in node_manager_base_config.items:
# print(configItem.name + ": " + configItem.value);
#new_config = cm_client.ApiConfig(name="yarn_nodemanager_resource_memory_mb", value="262144")
new_config = cm_client.ApiConfig(name=yarn - key, value=yarn - val)
new_config_list = cm_client.ApiConfigList([new_config])
res = role_config_group_resource.update_config("cluster",
"yarn-NODEMANAGER-BASE",
"yarn",
message="",
body=new_config_list)
res = role_config_group_resource.update_config("cluster",
"yarn-NODEMANAGER-1",
"yarn",
message="",
body=new_config_list)
res = role_config_group_resource.update_config("cluster",
"yarn-NODEMANAGER-2",
"yarn",
message="",
body=new_config_list)
def _enable_kerberos(self, kerberos_type, ipa_host):
# Update Kerberos configuration
config = [
cm_client.ApiConfig(name='KRB_AUTH_ENABLE', value='true'),
cm_client.ApiConfig(name='KRB_ENC_TYPES',
value='aes256-cts rc4-hmac'),
cm_client.ApiConfig(name='PUBLIC_CLOUD_STATUS',
value='ON_PUBLIC_CLOUD'),
cm_client.ApiConfig(name='SECURITY_REALM', value='WORKSHOP.COM'),
]
if kerberos_type == 'MIT':
config += [
cm_client.ApiConfig(name='KDC_ADMIN_HOST',
value=local_hostname()),
cm_client.ApiConfig(name='KDC_HOST', value=local_hostname()),
cm_client.ApiConfig(name='KDC_TYPE', value='MIT KDC'),
]
else:
config += [
cm_client.ApiConfig(name='KDC_ADMIN_HOST', value=ipa_host),
cm_client.ApiConfig(name='KDC_HOST', value=ipa_host),
cm_client.ApiConfig(name='KDC_TYPE', value='Red Hat IPA'),
cm_client.ApiConfig(name='AUTH_BACKEND_ORDER',
value='LDAP_THEN_DB'),
cm_client.ApiConfig(
name='LDAP_BIND_DN',
value=
'uid=ldap_bind_user,cn=users,cn=accounts,dc=workshop,dc=com'
),
cm_client.ApiConfig(name='LDAP_BIND_PW', value=the_pwd()),
cm_client.ApiConfig(
name='LDAP_GROUP_SEARCH_BASE',
value='cn=groups,cn=accounts,dc=workshop,dc=com'),
cm_client.ApiConfig(name='LDAP_GROUP_SEARCH_FILTER',
value='(member={0})'),
cm_client.ApiConfig(name='LDAP_TYPE', value='LDAP'),
cm_client.ApiConfig(name='LDAP_URL',
value='ldaps://' + ipa_host),
cm_client.ApiConfig(
name='LDAP_USER_SEARCH_BASE',
value='cn=users,cn=accounts,dc=workshop,dc=com'),
cm_client.ApiConfig(name='LDAP_USER_SEARCH_FILTER',
value='(uid={0})'),
]
if cm_version() < [7, 5, 3]:
# These properties were removed in OPSAPS-61384 (CM 7.5.3)
config += [
cm_client.ApiConfig(
name='LDAP_BIND_DN_MONITORING',
value=
'uid=ldap_bind_user,cn=users,cn=accounts,dc=workshop,dc=com'
),
cm_client.ApiConfig(name='LDAP_BIND_PW_MONITORING',
value=the_pwd()),
]
self.cm_api.update_config(message='Updating Kerberos config',
body=cm_client.ApiConfigList(config))
# Import Kerberos credentials
cmd = self.cm_api.import_admin_credentials(password=the_pwd(),
username=self.krb_princ)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to import admin credentials')
def setup_cm(self, key_file, cm_repo_url, use_kerberos, use_tls,
kerberos_type, ipa_host):
# Accept trial licence
try:
self.cm_api.begin_trial()
except ApiException as exc:
if exc.status == 400 and 'Trial has been used' in exc.body:
pass # This can be ignored
else:
raise
# Install CM Agent on host
with open(key_file, "r") as f:
key = f.read()
if self.host not in [
h.hostname for h in self.hosts_api.read_hosts().items
]:
instargs = cm_client.ApiHostInstallArguments(
host_names=[self.host],
user_name='root',
private_key=key,
cm_repo_url=cm_repo_url,
java_install_strategy='NONE',
ssh_port=22,
passphrase='')
cmd = self.cm_api.host_install_command(body=instargs)
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to add host to the cluster')
# Create MGMT/CMS
try:
self.mgmt_api.read_service()
print("Cloudera Management Services already installed")
cms_exists = True
except cm_client.rest.ApiException as e:
cms_exists = False
if not cms_exists:
print("Installing Cloudera Management Services")
api_service = cm_client.ApiService()
api_service.roles = [
cm_client.ApiRole(type='SERVICEMONITOR'),
cm_client.ApiRole(type='HOSTMONITOR'),
cm_client.ApiRole(type='EVENTSERVER'),
cm_client.ApiRole(type='ALERTPUBLISHER')
]
self.mgmt_api.setup_cms(body=api_service)
cmd = self.mgmt_api.start_command()
cmd = self.wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to start Management Services')
# Update cluster banner
c_id = cluster_id()
banner = 'Cluster ID: {}, Host: {}'.format(c_id, socket.gethostname())
header_color = HEADER_COLORS[c_id % len(HEADER_COLORS)]
self.cm_api.update_config(
message='Customizing CM header and banner',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(name='CUSTOM_BANNER_HTML', value=banner),
cm_client.ApiConfig(name='CUSTOM_HEADER_COLOR',
value=header_color),
]))
# Update host-level parameter required by SMM
self.all_hosts_api.update_config(
message='Updating parameter for SMM',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(
name='host_agent_safety_valve',
value=
'kafka_broker_topic_partition_metrics_for_smm_enabled=true'
)
]))
# Enable kerberos
if use_kerberos:
self._enable_kerberos(kerberos_type, ipa_host)
# Enable TLS
if use_tls:
self._enable_tls()
# Restart Mgmt Services
cmd = self.mgmt_api.restart_command()
cmd = self.wait(cmd)
print("Exception when calling ClouderaManagerResourceApi->import_cluster_template: %s\n" % e)
cm_client.configuration.username = '******'
cm_client.configuration.password = '******'
api_client = cm_client.ApiClient("http://localhost:7180/api/v40")
cm_api = cm_client.ClouderaManagerResourceApi(api_client)
# accept trial licence
cm_api.begin_trial()
# Update Cloudera Manager config for KRB
body = cm_client.ApiConfigList()
body.items=[
cm_client.ApiConfig(name='KDC_HOST', value='YourHostname'),
cm_client.ApiConfig(name='KDC_ADMIN_HOST', value='YourHostname'),
cm_client.ApiConfig(name='KDC_TYPE', value='MIT KDC'),
cm_client.ApiConfig(name='KRB_ENC_TYPES', value='aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 arcfour-hmac-md5'),
cm_client.ApiConfig(name='SECURITY_REALM', value='CLOUDERA.COM'),
cm_client.ApiConfig(name='KRB_MANAGE_KRB5_CONF', value='true')
]
api_response = cm_api.update_config(message="KRB", body=body)
# Import KDC admin credentials
cmd = cm_api.import_admin_credentials(password='******', username='******')
wait(cmd)
mgmt_api.auto_assign_roles() # needed?
mgmt_api.auto_configure() # needed?
mgmt_api.setup_cms(body=api_service)
cmd = mgmt_api.start_command()
cmd = wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to start Management Services')
# Update host-level parameter required by SMM
all_hosts_api = cm_client.AllHostsResourceApi(api_client)
all_hosts_api.update_config(
message='Updating parameter for SMM',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(
name='host_agent_safety_valve',
value='kafka_broker_topic_partition_metrics_for_smm_enabled=true')
]))
# create the cluster using the template
with open(TEMPLATE) as f:
json_str = f.read()
Response = namedtuple("Response", "data")
dst_cluster_template = api_client.deserialize(
response=Response(json_str), response_type=cm_client.ApiClusterTemplate)
cmd = cm_api.import_cluster_template(add_repositories=True,
body=dst_cluster_template)
cmd = wait(cmd)
if not cmd.success:
raise RuntimeError('Failed to deploy cluster template')
