def configure_ozone():
# support async profiler
safey_valve_config = cm_client.ApiConfig(name="ozone-conf/ozone-site.xml_service_safety_valve", value="<property><name>hdds.profiler.endpoint.enabled</name><value>true</value></property>")
body = cm_client.ApiServiceConfig([safey_valve_config])
updated_configs = services_api_instance.update_service_config(cluster.name, ozone.name, body=body)
# TODO: add -XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints java options to export more accurate async profiler output
# add java options
ozone_java_opts_config = cm_client.ApiConfig(name="ozone_java_opts", value="{{java_args}} -XX:NativeMemoryTracking=summary")
#ozone_java_opts_config = cm_client.ApiConfig(name="ozone_java_opts", value="{{java_args}} -XX:NativeMemoryTracking=summary -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=59999")
body = cm_client.ApiServiceConfig([ozone_java_opts_config])
updated_configs = services_api_instance.update_service_config(cluster.name, ozone.name, body=body)
# add environment variable
#env_config = cm_client.ApiConfig(name="OZONE_service_env_safety_valve", value="ASYNC_PROFILER_HOME=/opt/async-profiler-2.0-linux-x64")
env_config = cm_client.ApiConfig(name="OZONE_service_env_safety_valve", value="ASYNC_PROFILER_HOME=/opt/async-profiler-1.8.5-linux-x64")
body = cm_client.ApiServiceConfig([env_config])
updated_configs = services_api_instance.update_service_config(cluster.name, ozone.name, body=body)
# update service id
env_config = cm_client.ApiConfig(name="ozone.service.id", value=ozone_service_id)
body = cm_client.ApiServiceConfig([env_config])
updated_configs = services_api_instance.update_service_config(cluster.name, ozone.name, body=body)
body = cm_client.ApiServiceConfig([safey_valve_config])
updated_configs = services_api_instance.update_service_config(cluster.name, ozone.name, body=body)
print("Ozone updated")
def configure_hbase():
# add java options
role_api_instance = cm_client.RolesResourceApi(api_client)
rcg_configs = role_api_instance.read_roles(cluster.name, hbase.name)
regionserver_groups = [rcg_config.name for rcg_config in rcg_configs.items if rcg_config.type == 'REGIONSERVER']
hbase_java_opts_config = cm_client.ApiConfig(name="hbase_regionserver_java_opts", value="{{JAVA_GC_ARGS}} -XX:ReservedCodeCacheSize=256m -XX:NativeMemoryTracking=summary -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=50100")
body = cm_client.ApiServiceConfig([hbase_java_opts_config])
updated_configs = role_api_instance.update_role_config(cluster.name, regionserver_groups[0], hbase.name, body=body)
# add environment variable
env_config = cm_client.ApiConfig(name="hbase_service_env_safety_valve", value="ASYNC_PROFILER_HOME=/opt/async-profiler-1.8.5-linux-x64")
body = cm_client.ApiServiceConfig([env_config])
updated_configs = services_api_instance.update_service_config(cluster.name, hbase.name, body=body)
# TODO: add -XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints java options to export more accurate async profiler output
# enable offheap bucket cache
env_config = cm_client.ApiConfig(name="hbase_bucketcache_ioengine", value="offheap")
body = cm_client.ApiServiceConfig([env_config])
updated_configs = role_api_instance.update_role_config(cluster.name, regionserver_groups[0], hbase.name, body=body)
# bucket cache size
env_config = cm_client.ApiConfig(name="hbase_bucketcache_size", value="8192")
body = cm_client.ApiServiceConfig([env_config])
updated_configs = role_api_instance.update_role_config(cluster.name, regionserver_groups[0], hbase.name, body=body)
print("HBase updated")
def configure_hive():
if "hive" not in globals():
return
# set up hive warehouse directory to o3
warehouse_directory_config = cm_client.ApiConfig(name="hive_warehouse_directory", value=file_system_prefix + "/managed/hive")
body = cm_client.ApiServiceConfig([warehouse_directory_config])
updated_configs = services_api_instance.update_service_config(cluster.name, hive.name, body=body)
# set up hive external warehouse directory to o3
external_warehouse_directory_config = cm_client.ApiConfig(name="hive_warehouse_external_directory", value=file_system_prefix + "/external/hive")
body = cm_client.ApiServiceConfig([external_warehouse_directory_config])
updated_configs = services_api_instance.update_service_config(cluster.name, hive.name, body=body)
print("Hive updated")
def _enable_tls(self):
# Update TLS configuration
self.cm_api.update_config(
message='Updating TLS config',
body=cm_client.ApiConfigList([
cm_client.ApiConfig(name='AGENT_TLS', value='true'),
cm_client.ApiConfig(name='KEYSTORE_PASSWORD', value=the_pwd()),
cm_client.ApiConfig(
name='KEYSTORE_PATH',
value='/opt/cloudera/security/jks/keystore.jks'),
cm_client.ApiConfig(name='NEED_AGENT_VALIDATION',
value='true'),
cm_client.ApiConfig(name='SCM_PROXY_TIMEOUT', value='30000'),
cm_client.ApiConfig(name='TRUSTSTORE_PASSWORD',
value=the_pwd()),
cm_client.ApiConfig(
name='TRUSTSTORE_PATH',
value='/opt/cloudera/security/jks/truststore.jks'),
cm_client.ApiConfig(name='WEB_TLS', value='true'),
]))
self.mgmt_api.update_service_config(
message='Updating TLS config for Mgmt Services',
body=cm_client.ApiServiceConfig([
cm_client.ApiConfig(
name='ssl_client_truststore_location',
value='/opt/cloudera/security/jks/truststore.jks'),
cm_client.ApiConfig(name='ssl_client_truststore_password',
value=the_pwd()),
]))
def configure_yarn():
role_api_instance = cm_client.RolesResourceApi(api_client)
rcg_configs = role_api_instance.read_roles(cluster.name, yarn.name)
gateway_groups = [rcg_config.name for rcg_config in rcg_configs.items if rcg_config.type == 'GATEWAY']
map_memory_config = cm_client.ApiConfig(name="mapreduce_map_memory_mb", value="4096")
body = cm_client.ApiServiceConfig([map_memory_config])
updated_configs = role_api_instance.update_role_config(cluster.name, gateway_groups[0], yarn.name, body=body)
print("YARN updated")