def process_command_bootstrap(args):
deploy_utils.check_admin_priviledge(args)
args.update_config = True
deploy_tool = SERVICE_DEPLOY_TOOL_MAP.get(args.service)
if deploy_tool:
return deploy_tool.bootstrap(args)
Log.print_critical("Not implemented for service: %s", args.service)
def recover_region_server(args, ip):
package_root = deploy_utils.get_hbase_package_root(
args.hbase_config.cluster.version)
Log.print_info("Recover region server: " + ip);
host = socket.gethostbyaddr(ip)[0]
args.command = ["ruby", "%s/bin/region_mover.rb" % package_root,
"load", host]
if run_shell(args) != 0:
Log.print_critical("Load host %s failed." % host);
def pack(args):
get_hbase_service_config(args)
version = args.hbase_config.cluster.version
deploy_utils.make_package_dir(args, "hbase", version)
generate_client_config(args, "hbase", version)
if not args.skip_tarball:
deploy_utils.pack_package(args, "hbase", version)
Log.print_success("Pack client utilities for hbase success!\n")
def pack(args):
get_zk_service_config(args)
version = args.zookeeper_config.cluster.version
deploy_utils.make_package_dir(args, "zookeeper", version)
generate_client_config(args, "zookeeper", version)
if not args.skip_tarball:
deploy_utils.pack_package(args, "zookeeper", version)
Log.print_success("Pack client utilities for zookeeper success!")
def pack(args):
get_impala_service_config(args)
version = args.impala_config.cluster.version
deploy_utils.make_package_dir(args, "impala-shell", version)
if not args.skip_tarball:
deploy_utils.pack_package(args, "impala-shell",
args.impala_config.cluster.version)
Log.print_success("Pack client utilities for hadoop success!\n")
def pack(args):
get_yarn_service_config(args)
version = args.yarn_config.cluster.version
deploy_utils.make_package_dir(args, "hadoop", version)
deploy_hdfs.generate_client_config(args, "hadoop", version)
generate_client_config(args, "hadoop", version)
if not args.skip_tarball:
deploy_utils.pack_package(args, "hadoop", args.hdfs_config.cluster.version)
Log.print_success("Pack client utilities for hadoop success!\n")
def get_hdfs_service_config(args):
args.hdfs_config = deploy_utils.get_service_config(args)
if not args.hdfs_config.cluster.zk_cluster:
Log.print_critical(
"hdfs cluster must depends on a zookeeper clusters: %s" %
args.hdfs_config.cluster.name)
namenode_hosts = args.hdfs_config.jobs["namenode"].hosts
args.hdfs_config.jobs["zkfc"].hosts = namenode_hosts.copy()
args.skip_gen_config_files = False
def rolling_update(args):
get_zk_service_config(args)
job_name = "zookeeper"
if not args.skip_confirm:
deploy_utils.confirm_action(args, "rolling_update")
Log.print_info("Rolling updating %s" % job_name)
hosts = args.zookeeper_config.jobs[job_name].hosts
wait_time = 0
for id in hosts.iterkeys():
deploy_utils.confirm_rolling_update(id, wait_time)
stop_job(args, hosts[id], job_name)
deploy_utils.wait_for_job_stopping("zookeeper",
args.zookeeper_config.cluster.name, job_name, hosts[id])
start_job(args, hosts[id], job_name)
deploy_utils.wait_for_job_starting("zookeeper",
args.zookeeper_config.cluster.name, job_name, hosts[id])
wait_time = args.time_interval
Log.print_success("Rolling updating %s success" % job_name)
def bootstrap(args):
get_hdfs_service_config(args)
cleanup_token = deploy_utils.confirm_bootstrap("hdfs", args.hdfs_config)
for job_name in args.job or ALL_JOBS:
hosts = args.hdfs_config.jobs[job_name].hosts
first = True
if job_name == "namenode":
while not check_journalnode_all_started(args):
Log.print_warning("Wait for journalnode starting")
time.sleep(2)
for id in args.task or hosts.iterkeys():
if job_name == "namenode" and not first:
while not deploy_utils.check_service(hosts[0],
args.hdfs_config.jobs["namenode"].base_port):
Log.print_warning("Wait for active namenode starting")
time.sleep(2)
bootstrap_job(args, hosts[id], job_name, first, cleanup_token)
first = False
def balance_switch(args, flag):
fd, filename = tempfile.mkstemp()
f = os.fdopen(fd, 'w+')
if flag:
Log.print_info("balance_switch on for cluster: %s" % args.cluster)
print >> f, 'balance_switch true'
else:
Log.print_info("balance_switch off for cluster: %s" % args.cluster)
print >> f, 'balance_switch false'
print >> f, 'exit'
f.close()
args.command = ["shell", filename]
ret = run_shell(args)
os.remove(filename)
if ret != 0:
Log.print_critical("balance_switch off for cluster: %s failed!" %
args.cluster);
def rolling_update(args):
if not args.job:
Log.print_critical("You must specify the job name to do rolling update")
if not args.skip_confirm:
deploy_utils.confirm_action(args, "rolling_update")
get_hbase_service_config(args)
job_name = args.job[0]
if job_name != 'regionserver':
args.vacate_rs = False
if args.vacate_rs:
balance_switch(args, False)
Log.print_info("Rolling updating %s" % job_name)
hosts = args.hbase_config.jobs[job_name].hosts
wait_time = 0
for id in hosts.iterkeys():
if not args.skip_confirm:
deploy_utils.confirm_rolling_update(id, wait_time)
if args.vacate_rs:
vacate_region_server(args, hosts[id])
stop_job(args, hosts[id], job_name)
deploy_utils.wait_for_job_stopping("hbase",
args.hbase_config.cluster.name, job_name, hosts[id])
start_job(args, hosts[id], job_name)
deploy_utils.wait_for_job_starting("hbase",
args.hbase_config.cluster.name, job_name, hosts[id])
if args.vacate_rs:
recover_region_server(args, hosts[id])
wait_time = args.time_interval
if args.vacate_rs:
balance_switch(args, True)
Log.print_success("Rolling updating %s success" % job_name)
def rolling_update(args):
if not args.job:
Log.print_critical("You must specify the job name to do rolling update")
get_yarn_service_config(args)
job_name = args.job[0]
if not args.skip_confirm:
deploy_utils.confirm_action(args, "rolling_update")
Log.print_info("Rolling updating %s" % job_name)
hosts = args.yarn_config.jobs[job_name].hosts
wait_time = 0
for id in hosts.iterkeys():
deploy_utils.confirm_rolling_update(id, wait_time)
stop_job(args, hosts[id], job_name)
deploy_utils.wait_for_job_stopping("yarn",
args.yarn_config.cluster.name, job_name, hosts[id])
start_job(args, hosts[id], job_name)
deploy_utils.wait_for_job_starting("yarn",
args.yarn_config.cluster.name, job_name, hosts[id])
wait_time = args.time_interval
Log.print_success("Rolling updating %s success" % job_name)
def get_hbase_service_config(args):
args.hbase_config = deploy_utils.get_service_config(args)
if not args.hbase_config.cluster.zk_cluster:
Log.print_critical(
"hdfs cluster must depends on a zookeeper clusters: %s" %
args.hbase_config.cluster.name)
def get_zk_service_config(args):
args.zookeeper_config = deploy_utils.get_service_config(args)
if args.zookeeper_config.cluster.zk_cluster:
Log.print_critical(
"zookeeper cluster can't depends on other clusters: %s" %
args.zookeeper_config.cluster.name)
def process_command_rolling_update(args):
deploy_utils.check_admin_priviledge(args)
deploy_tool = SERVICE_DEPLOY_TOOL_MAP.get(args.service)
if deploy_tool:
return deploy_tool.rolling_update(args)
Log.print_critical("Not implemented for service: %s", args.service)
def process_command_pack(args):
deploy_tool = SERVICE_DEPLOY_TOOL_MAP.get(args.service)
if deploy_tool:
return deploy_tool.pack(args)
Log.print_critical("Not implemented for service: %s", args.service)
