def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.storm_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "storm", args.storm_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "storm-" + args.storm_config.cluster.version jar_dirs = "$package_dir/*" log_level = deploy_utils.get_service_log_level(args, args.storm_config) params = job.get_arguments(args, args.storm_config.cluster, args.storm_config.jobs, args.storm_config.arguments_dict, job_name, host_id, instance_id) service_env = "export SUPERVISOR_LOG_DIR=%s" % deploy_utils.get_supervisor_client(host, "storm", args.storm_config.cluster.name, 'supervisor', instance_id=instance_id).get_log_dir() script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "service_env": service_env, "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.storm_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "storm", args.storm_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "apache-storm-" + args.storm_config.cluster.version component_dir = "$package_dir" jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir) log_level = deploy_utils.get_service_log_level(args, args.storm_config) params = job.get_arguments(args, args.storm_config.cluster, args.storm_config.jobs, args.storm_config.arguments_dict, job_name, host_id, instance_id) service_env = "export SUPERVISOR_LOG_DIR=%s" % deploy_utils.get_supervisor_client(host, "storm", args.storm_config.cluster.name, 'supervisor', instance_id=instance_id).get_log_dir() script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "service_env": service_env, "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): supervisor_client = deploy_utils.get_supervisor_client( host, "impala", args.impala_config.cluster.name, job_name, instance_id=instance_id ) job = args.impala_config.jobs[job_name] artifact_and_version = "impala-" + args.impala_config.cluster.version log_level = deploy_utils.get_service_log_level(args, args.impala_config) params = job.get_arguments( args, args.impala_config.cluster, args.impala_config.jobs, args.impala_config.arguments_dict, job_name, host_id, instance_id, ) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "run_dir": supervisor_client.get_run_dir(), "ticket_cache": "$run_dir/impala.tc", "log_level": log_level, "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.hdfs_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "hdfs", args.hdfs_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "hadoop-" + args.hdfs_config.cluster.version jar_dirs = "" # must include both [dir]/ and [dir]/* as [dir]/* only import all jars under # this dir but we also need access the webapps under this dir. for component in ["common", "hdfs"]: if jar_dirs: jar_dirs += ":" component_dir = ("$package_dir/share/hadoop/%s" % component) jar_dirs += "%s/:%s/lib/*:%s/*" % ( component_dir, component_dir, component_dir) log_level = deploy_utils.get_service_log_level(args, args.hdfs_config) params = job.get_arguments(args, args.hdfs_config.cluster, args.hdfs_config.jobs, args.hdfs_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": params, } return script_dict
def generate_start_script(args, host, job_name, host_id, instance_id): supervisor_client = deploy_utils.get_supervisor_client(host, "zookeeper", args.zookeeper_config.cluster.name, job_name, instance_id=instance_id) run_dir = supervisor_client.get_run_dir() artifact_and_version = "zookeeper-" + args.zookeeper_config.cluster.version component_dir = "$package_dir" # must include both [dir]/ and [dir]/* as [dir]/* only import all jars under # this dir but we also need access the webapps under this dir. jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir) job = args.zookeeper_config.jobs["zookeeper"] log_level = deploy_utils.get_service_log_level(args, args.zookeeper_config) params = job.get_arguments(args, args.zookeeper_config.cluster, args.zookeeper_config.jobs, args.zookeeper_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": run_dir, "params": params, } return deploy_utils.create_run_script( '%s/start.sh.tmpl' % deploy_utils.get_template_dir(), script_dict)
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): supervisor_client = deploy_utils.get_supervisor_client( host, "impala", args.impala_config.cluster.name, job_name, instance_id=instance_id) job = args.impala_config.jobs[job_name] artifact_and_version = "impala-" + args.impala_config.cluster.version log_level = deploy_utils.get_service_log_level(args, args.impala_config) params = job.get_arguments(args, args.impala_config.cluster, args.impala_config.jobs, args.impala_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "run_dir": supervisor_client.get_run_dir(), "ticket_cache": "$run_dir/impala.tc", "log_level": log_level, "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.fds_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client( host, "fds", args.fds_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "galaxy-fds-" + args.fds_config.cluster.version component_dir = "$package_dir" jar_dirs = "%s/lib/guava-11.0.2.jar:%s/:%s/lib/*" % ( component_dir, component_dir, component_dir) log_level = deploy_utils.get_service_log_level(args, args.fds_config) params = job.get_arguments(args, args.fds_config.cluster, args.fds_config.jobs, args.fds_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.kafka_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client( host, "kafka", args.kafka_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "kafka-" + args.kafka_config.cluster.version jar_dirs = "$package_dir/*" log_level = deploy_utils.get_service_log_level(args, args.kafka_config) params = job.get_arguments(args, args.kafka_config.cluster, args.kafka_config.jobs, args.kafka_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": params, } return script_dict
def generate_metrics_config(args, host, job_name, instance_id=-1): job = args.hbase_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client( host, "hbase", args.hbase_config.cluster.name, job_name, instance_id=instance_id) ganglia_switch = "# " if args.hbase_config.cluster.ganglia_address: ganglia_switch = "" config_dict = { "job_name": job_name, "period": 10, "data_dir": supervisor_client.get_log_dir(), "ganglia_address": args.hbase_config.cluster.ganglia_address, "ganglia_switch": ganglia_switch, } local_path = "%s/hadoop-metrics.properties.tmpl" % deploy_utils.get_template_dir( ) template = deploy_utils.Template(open(local_path, "r").read()) return template.substitute(config_dict)
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.hbase_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client( host, "hbase", args.hbase_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "hbase-" + args.hbase_config.cluster.version component_dir = "$package_dir/" # must include both [dir]/ and [dir]/* as [dir]/* only import all jars under # this dir but we also need access the webapps under this dir. jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir) log_level = deploy_utils.get_service_log_level(args, args.hbase_config) params = job.get_arguments(args, args.hbase_config.cluster, args.hbase_config.jobs, args.hbase_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.yarn_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "yarn", args.yarn_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "hadoop-" + args.yarn_config.cluster.version jar_dirs = "" for component in ["common", "mapreduce", "yarn", "hdfs"]: if jar_dirs: jar_dirs += ":" component_dir = ("$package_dir/share/hadoop/%s" % component) jar_dirs += "%s/:%s/lib/*:%s/*" % ( component_dir, component_dir, component_dir) service_env = "" for component_path in ["HADOOP_COMMON_HOME", "HADOOP_HDFS_HOME", "YARN_HOME"]: service_env += "export %s=$package_dir\n" % (component_path) log_level = deploy_utils.get_service_log_level(args, args.yarn_config) params = job.get_arguments(args, args.yarn_config.cluster, args.yarn_config.jobs, args.yarn_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "service_env": service_env, "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.fds_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "fds", args.fds_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "galaxy-fds-" + args.fds_config.cluster.version component_dir = "$package_dir" jar_dirs = "%s/lib/guava-11.0.2.jar:%s/:%s/lib/*" % ( component_dir, component_dir, component_dir) log_level = deploy_utils.get_service_log_level(args, args.fds_config) params = job.get_arguments(args, args.fds_config.cluster, args.fds_config.jobs, args.fds_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name): job = args.yarn_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client( host, "yarn", args.yarn_config.cluster.name, job_name) artifact_and_version = "hadoop-" + args.yarn_config.cluster.version jar_dirs = "" for component in ["common", "mapreduce", "yarn", "hdfs"]: if jar_dirs: jar_dirs += ":" component_dir = ("$package_dir/share/hadoop/%s" % component) jar_dirs += "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir) service_env = "" for component_path in [ "HADOOP_COMMON_HOME", "HADOOP_HDFS_HOME", "YARN_HOME" ]: service_env += "export %s=$package_dir\n" % (component_path) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "service_env": service_env, "params": '-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn + '-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory + '-XX:MaxPermSize=%dm ' % job.max_perm_size + '-XX:+DisableExplicitGC ' + '-XX:+HeapDumpOnOutOfMemoryError ' + '-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' + '-XX:+UseConcMarkSweepGC ' + '-XX:CMSInitiatingOccupancyFraction=80 ' + '-XX:+UseMembar ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' + '-XX:+PrintGCDateStamps ' + '-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name + '-Dproc_%s ' % job_name + '-Djava.net.preferIPv4Stack=true ' + '-Dyarn.log.dir=$log_dir ' + '-Dyarn.pid=$pid ' + '-Dyarn.cluster=%s ' % args.yarn_config.cluster.name + '-Dhadoop.policy.file=hadoop-policy.xml ' + '-Dhadoop.home.dir=$package_dir ' + '-Dhadoop.id.str=%s ' % args.remote_user + '-Djava.security.krb5.conf=$run_dir/krb5.conf ' + get_job_specific_params(args, job_name) } if deploy_utils.is_security_enabled(args): class_path_root = "$package_dir/share/hadoop" boot_class_path = ("%s/common/lib/hadoop-security-%s.jar" % (class_path_root, args.hdfs_config.cluster.version)) script_dict["params"] += "-Xbootclasspath/p:%s " % boot_class_path script_dict["params"] += JOB_MAIN_CLASS[job_name] return script_dict
def get_data_dir_indexes(args, job_name, host, instance_id): if job_name != "datanode": return "0" else: supervisor_client = deploy_utils.get_supervisor_client(host, "hdfs", args.hdfs_config.cluster.name, job_name, instance_id=instance_id) data_dirs = supervisor_client.get_available_data_dirs() return ",".join([str(i) for i in range(len(data_dirs))])
def get_data_dir_indexes(args, job_name, host): if job_name != "datanode": return "0" else: supervisor_client = deploy_utils.get_supervisor_client( host, "hdfs", args.hdfs_config.cluster.name, job_name) data_dirs = supervisor_client.get_available_data_dirs() return ",".join([str(i) for i in range(len(data_dirs))])
def generate_run_scripts_params(args, host, job_name): job = args.hdfs_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client( host, "hdfs", args.hdfs_config.cluster.name, job_name) artifact_and_version = "hadoop-" + args.hdfs_config.cluster.version jar_dirs = "" # must include both [dir]/ and [dir]/* as [dir]/* only import all jars under # this dir but we also need access the webapps under this dir. for component in ["common", "hdfs"]: if jar_dirs: jar_dirs += ":" component_dir = ("$package_dir/share/hadoop/%s" % component) jar_dirs += "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": '-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn + '-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory + '-XX:MaxPermSize=%dm ' % job.max_perm_size + '-XX:+DisableExplicitGC ' + '-XX:+HeapDumpOnOutOfMemoryError ' + '-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' + '-XX:+UseConcMarkSweepGC ' + '-XX:CMSInitiatingOccupancyFraction=80 ' + '-XX:+UseMembar ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' + '-XX:+PrintGCDateStamps ' + '-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name + '-Dproc_%s ' % job_name + '-Djava.net.preferIPv4Stack=true ' + '-Dhdfs.log.dir=$log_dir ' + '-Dhdfs.pid=$pid ' + '-Dhdfs.cluster=%s ' % args.hdfs_config.cluster.name + '-Dhadoop.policy.file=hadoop-policy.xml ' + '-Dhadoop.home.dir=$package_dir ' + '-Djava.security.krb5.conf=$run_dir/krb5.conf ' + '-Dhadoop.id.str=%s ' % args.remote_user, } # config security-related params if deploy_utils.is_security_enabled(args): class_path_root = "$package_dir/share/hadoop/" boot_class_path = "%s/common/lib/hadoop-security-%s.jar" % ( class_path_root, args.hdfs_config.cluster.version) script_dict["params"] += ('-Xbootclasspath/p:%s ' % boot_class_path + '-Dkerberos.instance=hadoop ') # finally, add the job's main class name script_dict["params"] += (get_job_specific_params(args, job_name) + JOB_MAIN_CLASS[job_name]) return script_dict
def generate_bootstrap_script(args, host, job_name, host_id): supervisor_client = deploy_utils.get_supervisor_client( host, "zookeeper", args.zookeeper_config.cluster.name, job_name ) data_dir = supervisor_client.get_available_data_dirs()[0] myid_file = "%s/%s" % (data_dir, MYID_FILE) script_dict = {"myid_file": myid_file, "host_id": host_id} return deploy_utils.create_run_script("%s/bootstrap_zk.sh.tmpl" % deploy_utils.get_template_dir(), script_dict)
def generate_start_script(args, host, job_name): supervisor_client = deploy_utils.get_supervisor_client(host, "zookeeper", args.zookeeper_config.cluster.name, job_name) run_dir = supervisor_client.get_run_dir() artifact_and_version = "zookeeper-" + args.zookeeper_config.cluster.version component_dir = "$package_dir" # must include both [dir]/ and [dir]/* as [dir]/* only import all jars under # this dir but we also need access the webapps under this dir. jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir) job = args.zookeeper_config.jobs["zookeeper"] script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": run_dir, "params": '-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn + '-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory + '-XX:MaxPermSize=%dm ' % job.max_perm_size + '-XX:+DisableExplicitGC ' + '-XX:+HeapDumpOnOutOfMemoryError ' + '-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' + '-XX:+UseConcMarkSweepGC ' + '-XX:CMSInitiatingOccupancyFraction=80 ' + '-XX:+UseMembar ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' + '-XX:+PrintGCDateStamps ' + '-Xloggc:$run_dir/stdout/zk_gc_${start_time}.log ' + '-Djava.net.preferIPv4Stack=true ' + '-Dzookeeper.log.dir=$log_dir ' + '-Dzookeeper.cluster=%s ' % args.zookeeper_config.cluster.name + '-Dzookeeper.tracelog.dir=$log_dir ', } # Config security if deploy_utils.is_security_enabled(args): script_dict["params"] += '-Dzookeeper.superUser=zk_admin ' script_dict["params"] += '-Djava.security.auth.login.config=$run_dir/jaas.conf ' script_dict["params"] += '-Djava.security.krb5.conf=$run_dir/krb5.conf ' script_dict["params"] += 'org.apache.zookeeper.server.quorum.QuorumPeerMain ' script_dict["params"] += '$run_dir/zookeeper.cfg' return deploy_utils.create_run_script( '%s/start.sh.tmpl' % deploy_utils.get_template_dir(), script_dict)
def generate_bootstrap_script(args, host, job_name, host_id): supervisor_client = deploy_utils.get_supervisor_client(host, "zookeeper", args.zookeeper_config.cluster.name, job_name) data_dir = supervisor_client.get_available_data_dirs()[0] myid_file = "%s/%s" % (data_dir, MYID_FILE) script_dict = { 'myid_file': myid_file, 'host_id': host_id, } return deploy_utils.create_run_script( '%s/bootstrap_zk.sh.tmpl' % deploy_utils.get_template_dir(), script_dict)
def generate_bootstrap_script(args, host, job_name, host_id, instance_id): supervisor_client = deploy_utils.get_supervisor_client(host, "zookeeper", args.zookeeper_config.cluster.name, job_name, instance_id=instance_id) data_dir = supervisor_client.get_available_data_dirs()[0] myid_file = "%s/%s" % (data_dir, MYID_FILE) hosts = args.zookeeper_config.jobs["zookeeper"].hosts task_id = deploy_utils.get_task_id(hosts, host_id, instance_id) script_dict = { 'myid_file': myid_file, 'host_id': task_id, } return deploy_utils.create_run_script( '%s/bootstrap_zk.sh.tmpl' % deploy_utils.get_template_dir(), script_dict)
def generate_run_scripts_params(args, host, job_name): supervisor_client = deploy_utils.get_supervisor_client( host, "impala", args.impala_config.cluster.name, job_name) job = args.impala_config.jobs[job_name] impalad = args.impala_config.jobs["impalad"] statestored = args.impala_config.jobs["statestored"] artifact_and_version = "impala-" + args.impala_config.cluster.version script_dict = { "artifact": artifact_and_version, "job_name": job_name, "run_dir": supervisor_client.get_run_dir(), "ticket_cache": "$run_dir/impala.tc", "params": "-webserver_port=%d " % (job.base_port + 1) + "-be_port=%d " % (impalad.base_port + 2) + "-planservice_port=%d " % (impalad.base_port + 3) + "-state_store_port=%d " % statestored.base_port + "-state_store_subscriber_port=%d " % (statestored.base_port + 1) + "-mem_limit=20% " + # TODO make this configurable "-state_store_host=%s " % statestored.hosts[0] + "-kerberos_reinit_interval=1200 " + # 20hours "-webserver_doc_root=$run_dir/package " + "-webserver_interface=%s " % host + #"-use_statestore=false " + "-log_dir=$run_dir/log " + "-v=2 " + "-logbuflevel=-1 " + "-sasl_path=$run_dir/package/lib/sasl2 ", } if job_name == "impalad": script_dict["params"] += "-beeswax_port=%d " % impalad.base_port script_dict["params"] += "-hs2_port=%d " % (impalad.base_port + 4) if deploy_utils.is_security_enabled(args): script_dict["params"] += "-principal=%s/hadoop@%s " % ( args.impala_config.cluster.kerberos_username or "impala", args.impala_config.cluster.kerberos_realm) script_dict["params"] += "-keytab_file=%s/%s.keytab " % ( deploy_utils.HADOOP_CONF_PATH, args.impala_config.cluster.kerberos_username or "impala") script_dict["params"] += "-tgt_file=$run_dir/impala.tc " return script_dict
def generate_run_scripts_params(args, host, job_name): supervisor_client = deploy_utils.get_supervisor_client(host, "impala", args.impala_config.cluster.name, job_name) job = args.impala_config.jobs[job_name] impalad = args.impala_config.jobs["impalad"] statestored = args.impala_config.jobs["statestored"] artifact_and_version = "impala-" + args.impala_config.cluster.version script_dict = { "artifact": artifact_and_version, "job_name": job_name, "run_dir": supervisor_client.get_run_dir(), "ticket_cache": "$run_dir/impala.tc", "params": "-webserver_port=%d " % (job.base_port + 1) + "-be_port=%d " % (impalad.base_port + 2) + "-planservice_port=%d " % (impalad.base_port + 3) + "-state_store_port=%d " % statestored.base_port + "-state_store_subscriber_port=%d " % (statestored.base_port + 1) + "-mem_limit=20% " + # TODO make this configurable "-state_store_host=%s " % statestored.hosts[0] + "-kerberos_reinit_interval=1200 " + # 20hours "-webserver_doc_root=$run_dir/package " + "-webserver_interface=%s " % host + #"-use_statestore=false " + "-log_dir=$run_dir/log " + "-v=2 " + "-logbuflevel=-1 " + "-sasl_path=$run_dir/package/lib/sasl2 ", } if job_name == "impalad": script_dict["params"] += "-beeswax_port=%d " % impalad.base_port script_dict["params"] += "-hs2_port=%d " % (impalad.base_port + 4) if deploy_utils.is_security_enabled(args): script_dict["params"] += "-principal=%s/hadoop@%s " % ( args.impala_config.cluster.kerberos_username or "impala", args.impala_config.cluster.kerberos_realm) script_dict["params"] += "-keytab_file=%s/%s.keytab " % ( deploy_utils.HADOOP_CONF_PATH, args.impala_config.cluster.kerberos_username or "impala") script_dict["params"] += "-tgt_file=$run_dir/impala.tc " return script_dict
def generate_metrics_config(args, host, job_name, instance_id=-1): job = args.yarn_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "yarn", args.yarn_config.cluster.name, job_name, instance_id=instance_id) ganglia_switch = "# " if args.yarn_config.cluster.ganglia_address: ganglia_switch = "" config_dict = { "job_name": job_name, "period": 10, "data_dir": supervisor_client.get_log_dir(), "ganglia_address": args.yarn_config.cluster.ganglia_address, "ganglia_switch": ganglia_switch, } local_path = "%s/hadoop-metrics2.properties.tmpl" % deploy_utils.get_template_dir() template = deploy_utils.Template(open(local_path, "r").read()) return template.substitute(config_dict)
def generate_run_scripts_params(args, host, job_name, host_id, instance_id): job = args.kafka_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "kafka", args.kafka_config.cluster.name, job_name, instance_id=instance_id) artifact_and_version = "kafka-" + args.kafka_config.cluster.version jar_dirs = "$package_dir/*" log_level = deploy_utils.get_service_log_level(args, args.kafka_config) params = job.get_arguments(args, args.kafka_config.cluster, args.kafka_config.jobs, args.kafka_config.arguments_dict, job_name, host_id, instance_id) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": params, } return script_dict
def generate_run_scripts_params(args, host, job_name): job = args.hbase_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "hbase", args.hbase_config.cluster.name, job_name) artifact_and_version = "hbase-" + args.hbase_config.cluster.version component_dir = "$package_dir/" # must include both [dir]/ and [dir]/* as [dir]/* only import all jars under # this dir but we also need access the webapps under this dir. jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": '-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn + '-Xss256k ' + '-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory + '-XX:MaxPermSize=%dm ' % job.max_perm_size + '-XX:PermSize=%dm ' % job.max_perm_size + '-XX:+HeapDumpOnOutOfMemoryError ' + '-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' + '-XX:+UseConcMarkSweepGC ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' + '-XX:+PrintGCDateStamps ' + '-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name + '-XX:+UseMembar ' + '-XX:SurvivorRatio=1 ' + '-XX:+UseCMSCompactAtFullCollection ' + '-XX:CMSInitiatingOccupancyFraction=75 ' + '-XX:+UseCMSInitiatingOccupancyOnly ' + '-XX:+CMSParallelRemarkEnabled ' + '-XX:+UseNUMA ' + '-XX:+CMSClassUnloadingEnabled ' + '-XX:+PrintSafepointStatistics ' + '-XX:PrintSafepointStatisticsCount=1 ' + '-XX:+PrintHeapAtGC ' + '-XX:+PrintTenuringDistribution ' + '-XX:CMSMaxAbortablePrecleanTime=10000 ' + '-XX:TargetSurvivorRatio=80 ' + '-XX:+UseGCLogFileRotation ' + '-XX:NumberOfGCLogFiles=100 ' + '-XX:GCLogFileSize=128m ' + '-XX:CMSWaitDuration=2000 ' + '-XX:+CMSScavengeBeforeRemark ' + '-XX:+PrintPromotionFailure ' + '-XX:ConcGCThreads=8 ' + '-XX:ParallelGCThreads=8 ' + '-XX:PretenureSizeThreshold=4m ' + '-XX:+CMSConcurrentMTEnabled ' + '-XX:+ExplicitGCInvokesConcurrent ' + '-XX:+SafepointTimeout ' + '-XX:MonitorBound=16384 ' + '-XX:OldPLABSize=16 ' + '-XX:-ResizeOldPLAB ' + '-XX:-UseBiasedLocking ' + '-Dproc_%s ' % job_name + '-Djava.net.preferIPv4Stack=true ' + '-Dhbase.log.dir=$log_dir ' + '-Dhbase.pid=$pid ' + '-Dhbase.cluster=%s ' % args.hbase_config.cluster.name + '-Dhbase.policy.file=hbase-policy.xml ' + '-Dhbase.home.dir=$package_dir ' + '-Djava.security.krb5.conf=$run_dir/krb5.conf ' + '-Dhbase.id.str=%s ' % args.remote_user + get_job_specific_params(args, job_name), } if deploy_utils.is_security_enabled(args): jaas_path = "%s/jaas.conf" % supervisor_client.get_run_dir() script_dict["params"] += "-Djava.security.auth.login.config=%s " % jaas_path boot_class_path = ("$package_dir/lib/hadoop-security-%s.jar" % args.hdfs_config.cluster.version) script_dict["params"] += "-Xbootclasspath/p:%s " % boot_class_path script_dict["params"] += JOB_MAIN_CLASS[job_name] return script_dict
def generate_run_scripts_params(args, host, job_name): job = args.hdfs_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "hdfs", args.hdfs_config.cluster.name, job_name) artifact_and_version = "hadoop-" + args.hdfs_config.cluster.version jar_dirs = "" # must include both [dir]/ and [dir]/* as [dir]/* only import all jars under # this dir but we also need access the webapps under this dir. for component in ["common", "hdfs"]: if jar_dirs: jar_dirs += ":" component_dir = ("$package_dir/share/hadoop/%s" % component) jar_dirs += "%s/:%s/lib/*:%s/*" % ( component_dir, component_dir, component_dir) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": '-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn + '-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory + '-XX:MaxPermSize=%dm ' % job.max_perm_size + '-XX:+DisableExplicitGC ' + '-XX:+HeapDumpOnOutOfMemoryError ' + '-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' + '-XX:+UseConcMarkSweepGC ' + '-XX:CMSInitiatingOccupancyFraction=80 ' + '-XX:+UseMembar ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' + '-XX:+PrintGCDateStamps ' + '-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name + '-Dproc_%s ' % job_name + '-Djava.net.preferIPv4Stack=true ' + '-Dhdfs.log.dir=$log_dir ' + '-Dhdfs.pid=$pid ' + '-Dhdfs.cluster=%s ' % args.hdfs_config.cluster.name + '-Dhadoop.policy.file=hadoop-policy.xml ' + '-Dhadoop.home.dir=$package_dir ' + '-Djava.security.krb5.conf=$run_dir/krb5.conf ' + '-Dhadoop.id.str=%s ' % args.remote_user, } # config security-related params if deploy_utils.is_security_enabled(args): class_path_root = "$package_dir/share/hadoop/" boot_class_path = "%s/common/lib/hadoop-security-%s.jar" % ( class_path_root, args.hdfs_config.cluster.version) script_dict["params"] += ('-Xbootclasspath/p:%s ' % boot_class_path + '-Dkerberos.instance=hadoop ') # finally, add the job's main class name script_dict["params"] += (get_job_specific_params(args, job_name) + JOB_MAIN_CLASS[job_name]) return script_dict
def generate_run_scripts_params(args, host, job_name): job = args.yarn_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client(host, "yarn", args.yarn_config.cluster.name, job_name) artifact_and_version = "hadoop-" + args.yarn_config.cluster.version jar_dirs = "" for component in ["common", "mapreduce", "yarn", "hdfs"]: if jar_dirs: jar_dirs += ":" component_dir = ("$package_dir/share/hadoop/%s" % component) jar_dirs += "%s/:%s/lib/*:%s/*" % ( component_dir, component_dir, component_dir) service_env = "" for component_path in ["HADOOP_COMMON_HOME", "HADOOP_HDFS_HOME", "YARN_HOME"]: service_env += "export %s=$package_dir\n" % (component_path) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "service_env": service_env, "params": '-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn + '-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory + '-XX:MaxPermSize=%dm ' % job.max_perm_size + '-XX:+DisableExplicitGC ' + '-XX:+HeapDumpOnOutOfMemoryError ' + '-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' + '-XX:+UseConcMarkSweepGC ' + '-XX:CMSInitiatingOccupancyFraction=80 ' + '-XX:+UseMembar ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' + '-XX:+PrintGCDateStamps ' + '-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name + '-Dproc_%s ' % job_name + '-Djava.net.preferIPv4Stack=true ' + '-Dyarn.log.dir=$log_dir ' + '-Dyarn.pid=$pid ' + '-Dyarn.cluster=%s ' % args.yarn_config.cluster.name + '-Dhadoop.policy.file=hadoop-policy.xml ' + '-Dhadoop.home.dir=$package_dir ' + '-Dhadoop.id.str=%s ' % args.remote_user + '-Djava.security.krb5.conf=$run_dir/krb5.conf ' + get_job_specific_params(args, job_name) } if deploy_utils.is_security_enabled(args): class_path_root = "$package_dir/share/hadoop" boot_class_path = ("%s/common/lib/hadoop-security-%s.jar" % ( class_path_root, args.hdfs_config.cluster.version)) script_dict["params"] += "-Xbootclasspath/p:%s " % boot_class_path script_dict["params"] += JOB_MAIN_CLASS[job_name] return script_dict
def generate_run_scripts_params(args, host, job_name): job = args.hbase_config.jobs[job_name] supervisor_client = deploy_utils.get_supervisor_client( host, "hbase", args.hbase_config.cluster.name, job_name) artifact_and_version = "hbase-" + args.hbase_config.cluster.version component_dir = "$package_dir/" # must include both [dir]/ and [dir]/* as [dir]/* only import all jars under # this dir but we also need access the webapps under this dir. jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir) script_dict = { "artifact": artifact_and_version, "job_name": job_name, "jar_dirs": jar_dirs, "run_dir": supervisor_client.get_run_dir(), "params": '-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn + '-Xss256k ' + '-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory + '-XX:MaxPermSize=%dm ' % job.max_perm_size + '-XX:PermSize=%dm ' % job.max_perm_size + '-XX:+HeapDumpOnOutOfMemoryError ' + '-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' + '-XX:+UseConcMarkSweepGC ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' + '-XX:+PrintGCDateStamps ' + '-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name + '-XX:+UseMembar ' + '-XX:SurvivorRatio=1 ' + '-XX:+UseCMSCompactAtFullCollection ' + '-XX:CMSInitiatingOccupancyFraction=75 ' + '-XX:+UseCMSInitiatingOccupancyOnly ' + '-XX:+CMSParallelRemarkEnabled ' + '-XX:+UseNUMA ' + '-XX:+CMSClassUnloadingEnabled ' + '-XX:+PrintSafepointStatistics ' + '-XX:PrintSafepointStatisticsCount=1 ' + '-XX:+PrintHeapAtGC ' + '-XX:+PrintTenuringDistribution ' + '-XX:CMSMaxAbortablePrecleanTime=10000 ' + '-XX:TargetSurvivorRatio=80 ' + '-XX:+UseGCLogFileRotation ' + '-XX:NumberOfGCLogFiles=100 ' + '-XX:GCLogFileSize=128m ' + '-XX:CMSWaitDuration=2000 ' + '-XX:+CMSScavengeBeforeRemark ' + '-XX:+PrintPromotionFailure ' + '-XX:ConcGCThreads=8 ' + '-XX:ParallelGCThreads=8 ' + '-XX:PretenureSizeThreshold=4m ' + '-XX:+CMSConcurrentMTEnabled ' + '-XX:+ExplicitGCInvokesConcurrent ' + '-XX:+SafepointTimeout ' + '-XX:MonitorBound=16384 ' + '-XX:OldPLABSize=16 ' + '-XX:-ResizeOldPLAB ' + '-XX:-UseBiasedLocking ' + '-Dproc_%s ' % job_name + '-Djava.net.preferIPv4Stack=true ' + '-Dhbase.log.dir=$log_dir ' + '-Dhbase.pid=$pid ' + '-Dhbase.cluster=%s ' % args.hbase_config.cluster.name + '-Dhbase.policy.file=hbase-policy.xml ' + '-Dhbase.home.dir=$package_dir ' + '-Djava.security.krb5.conf=$run_dir/krb5.conf ' + '-Dhbase.id.str=%s ' % args.remote_user + get_job_specific_params(args, job_name), } if deploy_utils.is_security_enabled(args): jaas_path = "%s/jaas.conf" % supervisor_client.get_run_dir() script_dict[ "params"] += "-Djava.security.auth.login.config=%s " % jaas_path boot_class_path = ("$package_dir/lib/hadoop-security-%s.jar" % args.hdfs_config.cluster.version) script_dict["params"] += "-Xbootclasspath/p:%s " % boot_class_path script_dict["params"] += JOB_MAIN_CLASS[job_name] return script_dict