def web_proxy(region, stack_name):
ec2_conn = create_ec2_connection(region)
manager_instance = get_manager_instance(ec2_conn, stack_name)
master_instance = get_master_instance(ec2_conn, stack_name)
worker_instances = get_worker_instances(ec2_conn, stack_name)
tunnels = []
ts = "{0:<22}{1:<17}{2:<17}{3:<7}localhost:{4}"
print (ts.format("name", "public", "private", "remote", "local"))
# CM
tunnels.append(non_blocking_tunnel(manager_instance, 7180, 7180))
print (ts.format("CM WebUI", manager_instance.ip_address, manager_instance.private_ip_address, 7180, 7180))
# YARN RM
tunnels.append(non_blocking_tunnel(master_instance, 8088, 8088))
print (ts.format("YARN RM", master_instance.ip_address, master_instance.private_ip_address, 8088, 8088))
# YARN JobHistory
tunnels.append(non_blocking_tunnel(master_instance, 19888, 19888))
print (ts.format("YARN JobHistory", master_instance.ip_address, master_instance.private_ip_address, 19888, 19888))
try:
# block on an arbitrary ssh tunnel
tunnels[-1].wait()
finally:
for tunnel in tunnels:
tunnel.terminate()
def describe(region, stack_name):
ec2_conn = create_ec2_connection(region)
print "Launcher", get_launcher_instance(ec2_conn, stack_name).ip_address
print "Manager", get_manager_instance(ec2_conn, stack_name).ip_address
print "Master", get_master_instance(ec2_conn, stack_name).ip_address
for instance in get_worker_instances(ec2_conn, stack_name):
print "Worker", instance.ip_address
def install_java_8(region, stack_name):
# following general protocol for upgrading to JDK 1.8 here:
# http://www.cloudera.com/content/cloudera/en/documentation/core/v5-3-x/topics/cdh_cm_upgrading_to_jdk8.html
ec2_conn = create_ec2_connection(region)
manager_instance = get_manager_instance(ec2_conn, stack_name)
cluster_instances = get_worker_instances(ec2_conn, stack_name) + [
manager_instance,
get_master_instance(ec2_conn, stack_name),
]
cluster_hosts = [i.ip_address for i in cluster_instances]
# Connect to CM API
cm_api = ApiResource("localhost", username="******", password="******", server_port=64999, version=9)
cloudera_manager = cm_api.get_cloudera_manager()
with http_tunnel_ctx(manager_instance, 7180, 64999):
# Stop Cloudera Management Service
print "Stopping Cloudera Management Service"
mgmt_service = cloudera_manager.get_service()
mgmt_service.stop().wait()
# Stop cluster
print "Stopping the cluster"
clusters = cm_api.get_all_clusters()
cluster = clusters.objects[0]
cluster.stop().wait()
# Stop all Cloudera Manager Agents
@parallel
def stop_cm_agents():
sudo("service cloudera-scm-agent stop")
execute(stop_cm_agents, hosts=cluster_hosts)
# Stop the Cloudera Manager Server
def stop_cm_server():
sudo("service cloudera-scm-server stop")
execute(stop_cm_server, hosts=[manager_instance.ip_address])
# Cleanup other Java versions and install JDK 1.8
@parallel
def swap_jdks():
sudo("rpm -qa | grep jdk | xargs rpm -e")
sudo("rm -rf /usr/java/jdk1.6*")
sudo("rm -rf /usr/java/jdk1.7*")
run(
"wget -O jdk-8-linux-x64.rpm --no-cookies --no-check-certificate "
'--header "Cookie: oraclelicense=accept-securebackup-cookie" '
"http://download.oracle.com/otn-pub/java/jdk/8u51-b16/"
"jdk-8u51-linux-x64.rpm"
)
sudo("yum install -y jdk-8-linux-x64.rpm")
append("/home/ec2-user/.bash_profile", 'export JAVA_HOME=`find /usr/java -name "jdk1.8*"`')
execute(swap_jdks, hosts=cluster_hosts)
# Start the Cloudera Manager Server
def start_cm_server():
sudo("service cloudera-scm-server start")
execute(start_cm_server, hosts=[manager_instance.ip_address])
# Start all Cloudera Manager Agents
@parallel
def start_cm_agents():
sudo("service cloudera-scm-agent start")
execute(start_cm_agents, hosts=cluster_hosts)
with http_tunnel_ctx(manager_instance, 7180, 64999):
# Start the cluster and the mgmt service
print "Starting the cluster"
cluster.start().wait()
print "Starting the Cloudera Management Service"
cloudera_manager = cm_api.get_cloudera_manager()
mgmt_service = cloudera_manager.get_service()
mgmt_service.start().wait()