def adjust_yarn_memory_limits(region, stack_name): ec2_conn = create_ec2_connection(region) manager_instance = get_manager_instance(ec2_conn, stack_name) cm_api = ApiResource("localhost", username="******", password="******", server_port=64999, version=9) with http_tunnel_ctx(manager_instance, 7180, 64999): cluster = list(cm_api.get_all_clusters())[0] host = list(cm_api.get_all_hosts())[0] # all hosts same instance type yarn = filter(lambda x: x.type == "YARN", list(cluster.get_all_services()))[0] rm_cg = filter(lambda x: x.roleType == "RESOURCEMANAGER", list(yarn.get_all_role_config_groups()))[0] nm_cg = filter(lambda x: x.roleType == "NODEMANAGER", list(yarn.get_all_role_config_groups()))[0] rm_cg.update_config( { "yarn_scheduler_maximum_allocation_mb": (int(host.totalPhysMemBytes / 1024.0 / 1024.0)), "yarn_scheduler_maximum_allocation_vcores": host.numCores, } ) nm_cg.update_config( { "yarn_nodemanager_resource_memory_mb": (int(host.totalPhysMemBytes / 1024.0 / 1024.0)), "yarn_nodemanager_resource_cpu_vcores": host.numCores, } ) cluster.deploy_client_config().wait() cluster.restart().wait()
def install_java_8(region, stack_name): # following general protocol for upgrading to JDK 1.8 here: # http://www.cloudera.com/content/cloudera/en/documentation/core/v5-3-x/topics/cdh_cm_upgrading_to_jdk8.html ec2_conn = create_ec2_connection(region) manager_instance = get_manager_instance(ec2_conn, stack_name) cluster_instances = get_worker_instances(ec2_conn, stack_name) + [ manager_instance, get_master_instance(ec2_conn, stack_name), ] cluster_hosts = [i.ip_address for i in cluster_instances] # Connect to CM API cm_api = ApiResource("localhost", username="******", password="******", server_port=64999, version=9) cloudera_manager = cm_api.get_cloudera_manager() with http_tunnel_ctx(manager_instance, 7180, 64999): # Stop Cloudera Management Service print "Stopping Cloudera Management Service" mgmt_service = cloudera_manager.get_service() mgmt_service.stop().wait() # Stop cluster print "Stopping the cluster" clusters = cm_api.get_all_clusters() cluster = clusters.objects[0] cluster.stop().wait() # Stop all Cloudera Manager Agents @parallel def stop_cm_agents(): sudo("service cloudera-scm-agent stop") execute(stop_cm_agents, hosts=cluster_hosts) # Stop the Cloudera Manager Server def stop_cm_server(): sudo("service cloudera-scm-server stop") execute(stop_cm_server, hosts=[manager_instance.ip_address]) # Cleanup other Java versions and install JDK 1.8 @parallel def swap_jdks(): sudo("rpm -qa | grep jdk | xargs rpm -e") sudo("rm -rf /usr/java/jdk1.6*") sudo("rm -rf /usr/java/jdk1.7*") run( "wget -O jdk-8-linux-x64.rpm --no-cookies --no-check-certificate " '--header "Cookie: oraclelicense=accept-securebackup-cookie" ' "http://download.oracle.com/otn-pub/java/jdk/8u51-b16/" "jdk-8u51-linux-x64.rpm" ) sudo("yum install -y jdk-8-linux-x64.rpm") append("/home/ec2-user/.bash_profile", 'export JAVA_HOME=`find /usr/java -name "jdk1.8*"`') execute(swap_jdks, hosts=cluster_hosts) # Start the Cloudera Manager Server def start_cm_server(): sudo("service cloudera-scm-server start") execute(start_cm_server, hosts=[manager_instance.ip_address]) # Start all Cloudera Manager Agents @parallel def start_cm_agents(): sudo("service cloudera-scm-agent start") execute(start_cm_agents, hosts=cluster_hosts) with http_tunnel_ctx(manager_instance, 7180, 64999): # Start the cluster and the mgmt service print "Starting the cluster" cluster.start().wait() print "Starting the Cloudera Management Service" cloudera_manager = cm_api.get_cloudera_manager() mgmt_service = cloudera_manager.get_service() mgmt_service.start().wait()