def adjust_yarn_memory_limits(region, stack_name):
ec2_conn = create_ec2_connection(region)
manager_instance = get_manager_instance(ec2_conn, stack_name)
cm_api = ApiResource("localhost", username="******", password="******", server_port=64999, version=9)
with http_tunnel_ctx(manager_instance, 7180, 64999):
cluster = list(cm_api.get_all_clusters())[0]
host = list(cm_api.get_all_hosts())[0] # all hosts same instance type
yarn = filter(lambda x: x.type == "YARN", list(cluster.get_all_services()))[0]
rm_cg = filter(lambda x: x.roleType == "RESOURCEMANAGER", list(yarn.get_all_role_config_groups()))[0]
nm_cg = filter(lambda x: x.roleType == "NODEMANAGER", list(yarn.get_all_role_config_groups()))[0]
rm_cg.update_config(
{
"yarn_scheduler_maximum_allocation_mb": (int(host.totalPhysMemBytes / 1024.0 / 1024.0)),
"yarn_scheduler_maximum_allocation_vcores": host.numCores,
}
)
nm_cg.update_config(
{
"yarn_nodemanager_resource_memory_mb": (int(host.totalPhysMemBytes / 1024.0 / 1024.0)),
"yarn_nodemanager_resource_cpu_vcores": host.numCores,
}
)
cluster.deploy_client_config().wait()
cluster.restart().wait()
def install_java_8(region, stack_name):
# following general protocol for upgrading to JDK 1.8 here:
# http://www.cloudera.com/content/cloudera/en/documentation/core/v5-3-x/topics/cdh_cm_upgrading_to_jdk8.html
ec2_conn = create_ec2_connection(region)
manager_instance = get_manager_instance(ec2_conn, stack_name)
cluster_instances = get_worker_instances(ec2_conn, stack_name) + [
manager_instance,
get_master_instance(ec2_conn, stack_name),
]
cluster_hosts = [i.ip_address for i in cluster_instances]
# Connect to CM API
cm_api = ApiResource("localhost", username="******", password="******", server_port=64999, version=9)
cloudera_manager = cm_api.get_cloudera_manager()
with http_tunnel_ctx(manager_instance, 7180, 64999):
# Stop Cloudera Management Service
print "Stopping Cloudera Management Service"
mgmt_service = cloudera_manager.get_service()
mgmt_service.stop().wait()
# Stop cluster
print "Stopping the cluster"
clusters = cm_api.get_all_clusters()
cluster = clusters.objects[0]
cluster.stop().wait()
# Stop all Cloudera Manager Agents
@parallel
def stop_cm_agents():
sudo("service cloudera-scm-agent stop")
execute(stop_cm_agents, hosts=cluster_hosts)
# Stop the Cloudera Manager Server
def stop_cm_server():
sudo("service cloudera-scm-server stop")
execute(stop_cm_server, hosts=[manager_instance.ip_address])
# Cleanup other Java versions and install JDK 1.8
@parallel
def swap_jdks():
sudo("rpm -qa | grep jdk | xargs rpm -e")
sudo("rm -rf /usr/java/jdk1.6*")
sudo("rm -rf /usr/java/jdk1.7*")
run(
"wget -O jdk-8-linux-x64.rpm --no-cookies --no-check-certificate "
'--header "Cookie: oraclelicense=accept-securebackup-cookie" '
"http://download.oracle.com/otn-pub/java/jdk/8u51-b16/"
"jdk-8u51-linux-x64.rpm"
)
sudo("yum install -y jdk-8-linux-x64.rpm")
append("/home/ec2-user/.bash_profile", 'export JAVA_HOME=`find /usr/java -name "jdk1.8*"`')
execute(swap_jdks, hosts=cluster_hosts)
# Start the Cloudera Manager Server
def start_cm_server():
sudo("service cloudera-scm-server start")
execute(start_cm_server, hosts=[manager_instance.ip_address])
# Start all Cloudera Manager Agents
@parallel
def start_cm_agents():
sudo("service cloudera-scm-agent start")
execute(start_cm_agents, hosts=cluster_hosts)
with http_tunnel_ctx(manager_instance, 7180, 64999):
# Start the cluster and the mgmt service
print "Starting the cluster"
cluster.start().wait()
print "Starting the Cloudera Management Service"
cloudera_manager = cm_api.get_cloudera_manager()
mgmt_service = cloudera_manager.get_service()
mgmt_service.start().wait()