def restart(args): if not args.skip_confirm: deploy_utils.confirm_restart(args) _get_fds_service_config(args) for job_name in args.job or ALL_JOBS: hosts = args.fds_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range( hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances( host_id, hosts) else instance_id stop_job(args, hosts[host_id].ip, job_name, instance_id) for job_name in args.job or ALL_JOBS: hosts = args.fds_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range( hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances( host_id, hosts) else instance_id deploy_utils.wait_for_job_stopping( "fds", args.fds_config.cluster.name, job_name, hosts[host_id].ip, instance_id) start_job(args, hosts[host_id].ip, job_name, host_id, instance_id)
def bootstrap(args): get_hdfs_service_config(args) cleanup_token = deploy_utils.confirm_bootstrap("hdfs", args.hdfs_config) for job_name in args.job or ALL_JOBS: hosts = args.hdfs_config.jobs[job_name].hosts first = True if job_name == "namenode": while not check_journalnode_all_started(args): Log.print_warning("Wait for journalnode starting") time.sleep(2) args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range( hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances( host_id, hosts) else instance_id if job_name == "namenode" and not first: while not deploy_utils.check_service( hosts[0].ip, args.hdfs_config.jobs["namenode"].base_port): Log.print_warning("Wait for active namenode starting") time.sleep(2) bootstrap_job(args, hosts[host_id].ip, job_name, host_id, instance_id, first, cleanup_token) first = False
def rolling_update(args): if not args.job: Log.print_critical("You must specify the job name to do rolling update") get_hdfs_service_config(args) job_name = args.job[0] if not args.skip_confirm: deploy_utils.confirm_action(args, "rolling_update") Log.print_info("Rolling updating %s" % job_name) hosts = args.hdfs_config.jobs[job_name].hosts wait_time = 0 args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.iterkeys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id deploy_utils.confirm_rolling_update(host_id, instance_id, wait_time) stop_job(args, hosts[host_id].ip, job_name, instance_id) deploy_utils.wait_for_job_stopping("hdfs", args.hdfs_config.cluster.name, job_name, hosts[host_id].ip, instance_id) start_job(args, hosts[host_id].ip, job_name, host_id, instance_id) deploy_utils.wait_for_job_starting("hdfs", args.hdfs_config.cluster.name, job_name, hosts[host_id].ip, instance_id) wait_time = args.time_interval Log.print_success("Rolling updating %s success" % job_name)
def rolling_update(args): if not args.job: Log.print_critical( "You must specify the job name to do rolling update") _get_fds_service_config(args) job_name = args.job[0] if not args.skip_confirm: deploy_utils.confirm_action(args, "rolling_update") Log.print_info("Rolling updating %s" % job_name) hosts = args.fds_config.jobs[job_name].hosts wait_time = 0 args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.iterkeys(): for instance_id in args.task_map.get(host_id) or range( hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances( host_id, hosts) else instance_id deploy_utils.confirm_rolling_update(host_id, instance_id, wait_time) stop_job(args, hosts[host_id].ip, job_name, instance_id) deploy_utils.wait_for_job_stopping("fds", args.fds_config.cluster.name, job_name, hosts[host_id].ip, instance_id) start_job(args, hosts[host_id].ip, job_name, host_id, instance_id) deploy_utils.wait_for_job_starting("fds", args.fds_config.cluster.name, job_name, hosts[host_id].ip, instance_id) wait_time = args.time_interval Log.print_success("Rolling updating %s success" % job_name)
def restart(args): if not args.skip_confirm: deploy_utils.confirm_restart(args) get_zk_service_config(args) hosts = args.zookeeper_config.jobs["zookeeper"].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id stop_job(args, hosts[host_id].ip, "zookeeper", instance_id) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id deploy_utils.wait_for_job_stopping("zookeeper", args.zookeeper_config.cluster.name, "zookeeper", hosts[host_id].ip, instance_id) start_job(args, hosts[host_id].ip, "zookeeper", host_id, instance_id)
def show(args): get_zk_service_config(args) hosts = args.zookeeper_config.jobs["zookeeper"].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id deploy_utils.show_job("zookeeper", args.zookeeper_config, hosts[host_id].ip, "zookeeper", instance_id)
def show(args): get_hbase_service_config(args) for job_name in args.job or ALL_JOBS: hosts = args.hbase_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id deploy_utils.show_job("hbase", args.hbase_config, hosts[host_id].ip, job_name, instance_id)
def bootstrap(args): get_zk_service_config(args) cleanup_token = deploy_utils.confirm_bootstrap("zookeeper", args.zookeeper_config) hosts = args.zookeeper_config.jobs["zookeeper"].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id bootstrap_job(args, hosts[host_id].ip, "zookeeper", host_id, instance_id, cleanup_token)
def show(args): get_yarn_service_config(args) for job_name in args.job or ALL_JOBS: hosts = args.yarn_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id deploy_utils.show_job("yarn", args.yarn_config, hosts[host_id].ip, job_name, instance_id)
def bootstrap(args): get_yarn_service_config(args) cleanup_token = deploy_utils.confirm_bootstrap("yarn", args.yarn_config) for job_name in args.job or ALL_JOBS: hosts = args.yarn_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id bootstrap_job(args, hosts[host_id].ip, job_name, host_id, instance_id, cleanup_token)
def rolling_update(args): if not args.job: Log.print_critical( "You must specify the job name to do rolling update") if not args.skip_confirm: deploy_utils.confirm_action(args, "rolling_update") get_hbase_service_config(args) job_name = args.job[0] if job_name != 'regionserver': args.vacate_rs = False if args.vacate_rs: balance_switch(args, False) Log.print_info("Rolling updating %s" % job_name) hosts = args.hbase_config.jobs[job_name].hosts wait_time = 0 args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.iterkeys(): for instance_id in args.task_map.get(host_id) or range( hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances( host_id, hosts) else instance_id if not args.skip_confirm: deploy_utils.confirm_rolling_update(host_id, instance_id, wait_time) port = deploy_utils.get_base_port( args.hbase_config.jobs[job_name].base_port, instance_id) if args.vacate_rs: vacate_region_server(args, hosts[host_id].ip, port) stop_job(args, hosts[host_id].ip, job_name, instance_id) deploy_utils.wait_for_job_stopping("hbase", args.hbase_config.cluster.name, job_name, hosts[host_id].ip, instance_id) start_job(args, hosts[host_id].ip, job_name, host_id, instance_id) deploy_utils.wait_for_job_starting("hbase", args.hbase_config.cluster.name, job_name, hosts[host_id].ip, instance_id) if args.vacate_rs: recover_region_server(args, hosts[host_id].ip, port) wait_time = args.time_interval if args.vacate_rs: balance_switch(args, True) Log.print_success("Rolling updating %s success" % job_name)
def bootstrap(args): _get_kafka_service_config(args) cleanup_token = deploy_utils.confirm_bootstrap("kafka", args.kafka_config) for job_name in args.job or ALL_JOBS: hosts = args.kafka_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id bootstrap_job(args, hosts[host_id].ip, job_name, host_id, instance_id, cleanup_token)
def cleanup(args): get_zk_service_config(args) cleanup_token = deploy_utils.confirm_cleanup(args, "zookeeper", args.zookeeper_config) hosts = args.zookeeper_config.jobs["zookeeper"].hosts for host_id in hosts.keys(): for instance_id in range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id deploy_utils.cleanup_job("zookeeper", args.zookeeper_config, hosts[host_id].ip, "zookeeper", instance_id, cleanup_token)
def stop(args): if not args.skip_confirm: deploy_utils.confirm_stop(args) get_hdfs_service_config(args) for job_name in args.job or ALL_JOBS: hosts = args.hdfs_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id stop_job(args, hosts[host_id].ip, job_name, instance_id)
def restart(args): if not args.skip_confirm: deploy_utils.confirm_restart(args) _get_kafka_service_config(args) for job_name in args.job or ALL_JOBS: hosts = args.kafka_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id stop_job(args, hosts[host_id].ip, job_name, instance_id) for job_name in args.job or ALL_JOBS: hosts = args.kafka_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id deploy_utils.wait_for_job_stopping("kafka", args.kafka_config.cluster.name, job_name, hosts[host_id].ip, instance_id) start_job(args, hosts[host_id].ip, job_name, host_id, instance_id)
def cleanup(args): get_yarn_service_config(args) cleanup_token = deploy_utils.confirm_cleanup(args, "yarn", args.yarn_config) for job_name in args.job or ALL_JOBS: hosts = args.yarn_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id deploy_utils.cleanup_job("yarn", args.yarn_config, hosts[host_id].ip, job_name, instance_id, cleanup_token)
def stop(args): if not args.skip_confirm: deploy_utils.confirm_stop(args) get_hbase_service_config(args) for job_name in args.job or reversed(ALL_JOBS): hosts = args.hbase_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range( hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances( host_id, hosts) else instance_id stop_job(args, hosts[host_id].ip, job_name, instance_id)
def cleanup(args): _get_fds_service_config(args) cleanup_token = deploy_utils.confirm_cleanup(args, "fds", args.fds_config) for job_name in args.job or ALL_JOBS: hosts = args.fds_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range( hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances( host_id, hosts) else instance_id deploy_utils.cleanup_job("fds", args.fds_config, hosts[host_id].ip, job_name, instance_id, cleanup_token)
def rolling_update(args): if not args.job: Log.print_critical("You must specify the job name to do rolling update") if not args.skip_confirm: deploy_utils.confirm_action(args, "rolling_update") get_hbase_service_config(args) job_name = args.job[0] if job_name != "regionserver": args.vacate_rs = False if args.vacate_rs: balance_switch(args, False) Log.print_info("Rolling updating %s" % job_name) hosts = args.hbase_config.jobs[job_name].hosts wait_time = 0 args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.iterkeys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id if not args.skip_confirm: deploy_utils.confirm_rolling_update(host_id, instance_id, wait_time) port = deploy_utils.get_base_port(args.hbase_config.jobs[job_name].base_port, instance_id) if args.vacate_rs: vacate_region_server(args, hosts[host_id].ip, port) stop_job(args, hosts[host_id].ip, job_name, instance_id) deploy_utils.wait_for_job_stopping( "hbase", args.hbase_config.cluster.name, job_name, hosts[host_id].ip, instance_id ) start_job(args, hosts[host_id].ip, job_name, host_id, instance_id) deploy_utils.wait_for_job_starting( "hbase", args.hbase_config.cluster.name, job_name, hosts[host_id].ip, instance_id ) if args.vacate_rs: recover_region_server(args, hosts[host_id].ip, port) wait_time = args.time_interval if args.vacate_rs: balance_switch(args, True) Log.print_success("Rolling updating %s success" % job_name)
def cleanup(args): get_hdfs_service_config(args) cleanup_token = deploy_utils.confirm_cleanup(args, "hdfs", args.hdfs_config) first = True for job_name in args.job or ALL_JOBS: hosts = args.hdfs_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id cleanup_job(args, hosts[host_id].ip, job_name, host_id, instance_id, first, cleanup_token) if job_name == "zkfc": first = False
def cleanup(args): get_hdfs_service_config(args) cleanup_token = deploy_utils.confirm_cleanup(args, "hdfs", args.hdfs_config) first = True for job_name in args.job or ALL_JOBS: hosts = args.hdfs_config.jobs[job_name].hosts args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range( hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances( host_id, hosts) else instance_id cleanup_job(args, hosts[host_id].ip, job_name, host_id, instance_id, first, cleanup_token) if job_name == "zkfc": first = False
def bootstrap(args): get_hdfs_service_config(args) cleanup_token = deploy_utils.confirm_bootstrap("hdfs", args.hdfs_config) for job_name in args.job or ALL_JOBS: hosts = args.hdfs_config.jobs[job_name].hosts first = True if job_name == "namenode": while not check_journalnode_all_started(args): Log.print_warning("Wait for journalnode starting") time.sleep(2) args.task_map = deploy_utils.parse_args_host_and_task(args, hosts) for host_id in args.task_map.keys() or hosts.keys(): for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num): instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id if job_name == "namenode" and not first: while not deploy_utils.check_service(hosts[0].ip, args.hdfs_config.jobs["namenode"].base_port): Log.print_warning("Wait for active namenode starting") time.sleep(2) bootstrap_job(args, hosts[host_id].ip, job_name, host_id, instance_id, first, cleanup_token) first = False