def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.storm_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"storm", args.storm_config.cluster.name, job_name, instance_id=instance_id)
artifact_and_version = "storm-" + args.storm_config.cluster.version
jar_dirs = "$package_dir/*"
log_level = deploy_utils.get_service_log_level(args, args.storm_config)
params = job.get_arguments(args, args.storm_config.cluster, args.storm_config.jobs,
args.storm_config.arguments_dict, job_name, host_id, instance_id)
service_env = "export SUPERVISOR_LOG_DIR=%s" % deploy_utils.get_supervisor_client(host,
"storm", args.storm_config.cluster.name, 'supervisor', instance_id=instance_id).get_log_dir()
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"service_env": service_env,
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.storm_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"storm", args.storm_config.cluster.name, job_name, instance_id=instance_id)
artifact_and_version = "apache-storm-" + args.storm_config.cluster.version
component_dir = "$package_dir"
jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir)
log_level = deploy_utils.get_service_log_level(args, args.storm_config)
params = job.get_arguments(args, args.storm_config.cluster, args.storm_config.jobs,
args.storm_config.arguments_dict, job_name, host_id, instance_id)
service_env = "export SUPERVISOR_LOG_DIR=%s" % deploy_utils.get_supervisor_client(host,
"storm", args.storm_config.cluster.name, 'supervisor', instance_id=instance_id).get_log_dir()
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"service_env": service_env,
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
supervisor_client = deploy_utils.get_supervisor_client(
host, "impala", args.impala_config.cluster.name, job_name, instance_id=instance_id
)
job = args.impala_config.jobs[job_name]
artifact_and_version = "impala-" + args.impala_config.cluster.version
log_level = deploy_utils.get_service_log_level(args, args.impala_config)
params = job.get_arguments(
args,
args.impala_config.cluster,
args.impala_config.jobs,
args.impala_config.arguments_dict,
job_name,
host_id,
instance_id,
)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"run_dir": supervisor_client.get_run_dir(),
"ticket_cache": "$run_dir/impala.tc",
"log_level": log_level,
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.hdfs_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"hdfs", args.hdfs_config.cluster.name, job_name, instance_id=instance_id)
artifact_and_version = "hadoop-" + args.hdfs_config.cluster.version
jar_dirs = ""
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
for component in ["common", "hdfs"]:
if jar_dirs: jar_dirs += ":"
component_dir = ("$package_dir/share/hadoop/%s" % component)
jar_dirs += "%s/:%s/lib/*:%s/*" % (
component_dir, component_dir, component_dir)
log_level = deploy_utils.get_service_log_level(args, args.hdfs_config)
params = job.get_arguments(args, args.hdfs_config.cluster, args.hdfs_config.jobs,
args.hdfs_config.arguments_dict, job_name, host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"params": params,
}
return script_dict
def generate_start_script(args, host, job_name, host_id, instance_id):
supervisor_client = deploy_utils.get_supervisor_client(host,
"zookeeper", args.zookeeper_config.cluster.name, job_name, instance_id=instance_id)
run_dir = supervisor_client.get_run_dir()
artifact_and_version = "zookeeper-" + args.zookeeper_config.cluster.version
component_dir = "$package_dir"
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir)
job = args.zookeeper_config.jobs["zookeeper"]
log_level = deploy_utils.get_service_log_level(args, args.zookeeper_config)
params = job.get_arguments(args, args.zookeeper_config.cluster, args.zookeeper_config.jobs,
args.zookeeper_config.arguments_dict, job_name, host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": run_dir,
"params": params,
}
return deploy_utils.create_run_script(
'%s/start.sh.tmpl' % deploy_utils.get_template_dir(),
script_dict)
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
supervisor_client = deploy_utils.get_supervisor_client(
host,
"impala",
args.impala_config.cluster.name,
job_name,
instance_id=instance_id)
job = args.impala_config.jobs[job_name]
artifact_and_version = "impala-" + args.impala_config.cluster.version
log_level = deploy_utils.get_service_log_level(args, args.impala_config)
params = job.get_arguments(args, args.impala_config.cluster,
args.impala_config.jobs,
args.impala_config.arguments_dict, job_name,
host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"run_dir": supervisor_client.get_run_dir(),
"ticket_cache": "$run_dir/impala.tc",
"log_level": log_level,
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.fds_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(
host,
"fds",
args.fds_config.cluster.name,
job_name,
instance_id=instance_id)
artifact_and_version = "galaxy-fds-" + args.fds_config.cluster.version
component_dir = "$package_dir"
jar_dirs = "%s/lib/guava-11.0.2.jar:%s/:%s/lib/*" % (
component_dir, component_dir, component_dir)
log_level = deploy_utils.get_service_log_level(args, args.fds_config)
params = job.get_arguments(args, args.fds_config.cluster,
args.fds_config.jobs,
args.fds_config.arguments_dict, job_name,
host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.kafka_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(
host,
"kafka",
args.kafka_config.cluster.name,
job_name,
instance_id=instance_id)
artifact_and_version = "kafka-" + args.kafka_config.cluster.version
jar_dirs = "$package_dir/*"
log_level = deploy_utils.get_service_log_level(args, args.kafka_config)
params = job.get_arguments(args, args.kafka_config.cluster,
args.kafka_config.jobs,
args.kafka_config.arguments_dict, job_name,
host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"params": params,
}
return script_dict
def generate_metrics_config(args, host, job_name, instance_id=-1):
job = args.hbase_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(
host,
"hbase",
args.hbase_config.cluster.name,
job_name,
instance_id=instance_id)
ganglia_switch = "# "
if args.hbase_config.cluster.ganglia_address:
ganglia_switch = ""
config_dict = {
"job_name": job_name,
"period": 10,
"data_dir": supervisor_client.get_log_dir(),
"ganglia_address": args.hbase_config.cluster.ganglia_address,
"ganglia_switch": ganglia_switch,
}
local_path = "%s/hadoop-metrics.properties.tmpl" % deploy_utils.get_template_dir(
)
template = deploy_utils.Template(open(local_path, "r").read())
return template.substitute(config_dict)
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.hbase_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(
host,
"hbase",
args.hbase_config.cluster.name,
job_name,
instance_id=instance_id)
artifact_and_version = "hbase-" + args.hbase_config.cluster.version
component_dir = "$package_dir/"
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir,
component_dir)
log_level = deploy_utils.get_service_log_level(args, args.hbase_config)
params = job.get_arguments(args, args.hbase_config.cluster,
args.hbase_config.jobs,
args.hbase_config.arguments_dict, job_name,
host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.yarn_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"yarn", args.yarn_config.cluster.name, job_name, instance_id=instance_id)
artifact_and_version = "hadoop-" + args.yarn_config.cluster.version
jar_dirs = ""
for component in ["common", "mapreduce", "yarn", "hdfs"]:
if jar_dirs: jar_dirs += ":"
component_dir = ("$package_dir/share/hadoop/%s" % component)
jar_dirs += "%s/:%s/lib/*:%s/*" % (
component_dir, component_dir, component_dir)
service_env = ""
for component_path in ["HADOOP_COMMON_HOME", "HADOOP_HDFS_HOME", "YARN_HOME"]:
service_env += "export %s=$package_dir\n" % (component_path)
log_level = deploy_utils.get_service_log_level(args, args.yarn_config)
params = job.get_arguments(args, args.yarn_config.cluster, args.yarn_config.jobs,
args.yarn_config.arguments_dict, job_name, host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"service_env": service_env,
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.fds_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"fds", args.fds_config.cluster.name, job_name, instance_id=instance_id)
artifact_and_version = "galaxy-fds-" + args.fds_config.cluster.version
component_dir = "$package_dir"
jar_dirs = "%s/lib/guava-11.0.2.jar:%s/:%s/lib/*" % (
component_dir, component_dir, component_dir)
log_level = deploy_utils.get_service_log_level(args, args.fds_config)
params = job.get_arguments(args, args.fds_config.cluster, args.fds_config.jobs,
args.fds_config.arguments_dict, job_name, host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.yarn_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"yarn", args.yarn_config.cluster.name, job_name, instance_id=instance_id)
artifact_and_version = "hadoop-" + args.yarn_config.cluster.version
jar_dirs = ""
for component in ["common", "mapreduce", "yarn", "hdfs"]:
if jar_dirs: jar_dirs += ":"
component_dir = ("$package_dir/share/hadoop/%s" % component)
jar_dirs += "%s/:%s/lib/*:%s/*" % (
component_dir, component_dir, component_dir)
service_env = ""
for component_path in ["HADOOP_COMMON_HOME", "HADOOP_HDFS_HOME", "YARN_HOME"]:
service_env += "export %s=$package_dir\n" % (component_path)
log_level = deploy_utils.get_service_log_level(args, args.yarn_config)
params = job.get_arguments(args, args.yarn_config.cluster, args.yarn_config.jobs,
args.yarn_config.arguments_dict, job_name, host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"service_env": service_env,
"params": params,
}
return script_dict
def generate_start_script(args, host, job_name, host_id, instance_id):
supervisor_client = deploy_utils.get_supervisor_client(host,
"zookeeper", args.zookeeper_config.cluster.name, job_name, instance_id=instance_id)
run_dir = supervisor_client.get_run_dir()
artifact_and_version = "zookeeper-" + args.zookeeper_config.cluster.version
component_dir = "$package_dir"
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir)
job = args.zookeeper_config.jobs["zookeeper"]
log_level = deploy_utils.get_service_log_level(args, args.zookeeper_config)
params = job.get_arguments(args, args.zookeeper_config.cluster, args.zookeeper_config.jobs,
args.zookeeper_config.arguments_dict, job_name, host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": run_dir,
"params": params,
}
return deploy_utils.create_run_script(
'%s/start.sh.tmpl' % deploy_utils.get_template_dir(),
script_dict)
def generate_run_scripts_params(args, host, job_name):
job = args.yarn_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(
host, "yarn", args.yarn_config.cluster.name, job_name)
artifact_and_version = "hadoop-" + args.yarn_config.cluster.version
jar_dirs = ""
for component in ["common", "mapreduce", "yarn", "hdfs"]:
if jar_dirs: jar_dirs += ":"
component_dir = ("$package_dir/share/hadoop/%s" % component)
jar_dirs += "%s/:%s/lib/*:%s/*" % (component_dir, component_dir,
component_dir)
service_env = ""
for component_path in [
"HADOOP_COMMON_HOME", "HADOOP_HDFS_HOME", "YARN_HOME"
]:
service_env += "export %s=$package_dir\n" % (component_path)
script_dict = {
"artifact":
artifact_and_version,
"job_name":
job_name,
"jar_dirs":
jar_dirs,
"run_dir":
supervisor_client.get_run_dir(),
"service_env":
service_env,
"params":
'-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn +
'-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory +
'-XX:MaxPermSize=%dm ' % job.max_perm_size +
'-XX:+DisableExplicitGC ' + '-XX:+HeapDumpOnOutOfMemoryError ' +
'-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' +
'-XX:+UseConcMarkSweepGC ' + '-XX:CMSInitiatingOccupancyFraction=80 ' +
'-XX:+UseMembar ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' +
'-XX:+PrintGCDateStamps ' +
'-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name +
'-Dproc_%s ' % job_name + '-Djava.net.preferIPv4Stack=true ' +
'-Dyarn.log.dir=$log_dir ' + '-Dyarn.pid=$pid ' +
'-Dyarn.cluster=%s ' % args.yarn_config.cluster.name +
'-Dhadoop.policy.file=hadoop-policy.xml ' +
'-Dhadoop.home.dir=$package_dir ' +
'-Dhadoop.id.str=%s ' % args.remote_user +
'-Djava.security.krb5.conf=$run_dir/krb5.conf ' +
get_job_specific_params(args, job_name)
}
if deploy_utils.is_security_enabled(args):
class_path_root = "$package_dir/share/hadoop"
boot_class_path = ("%s/common/lib/hadoop-security-%s.jar" %
(class_path_root, args.hdfs_config.cluster.version))
script_dict["params"] += "-Xbootclasspath/p:%s " % boot_class_path
script_dict["params"] += JOB_MAIN_CLASS[job_name]
return script_dict
def get_data_dir_indexes(args, job_name, host, instance_id):
if job_name != "datanode":
return "0"
else:
supervisor_client = deploy_utils.get_supervisor_client(host,
"hdfs", args.hdfs_config.cluster.name, job_name, instance_id=instance_id)
data_dirs = supervisor_client.get_available_data_dirs()
return ",".join([str(i) for i in range(len(data_dirs))])
def get_data_dir_indexes(args, job_name, host):
if job_name != "datanode":
return "0"
else:
supervisor_client = deploy_utils.get_supervisor_client(
host, "hdfs", args.hdfs_config.cluster.name, job_name)
data_dirs = supervisor_client.get_available_data_dirs()
return ",".join([str(i) for i in range(len(data_dirs))])
def generate_run_scripts_params(args, host, job_name):
job = args.hdfs_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(
host, "hdfs", args.hdfs_config.cluster.name, job_name)
artifact_and_version = "hadoop-" + args.hdfs_config.cluster.version
jar_dirs = ""
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
for component in ["common", "hdfs"]:
if jar_dirs: jar_dirs += ":"
component_dir = ("$package_dir/share/hadoop/%s" % component)
jar_dirs += "%s/:%s/lib/*:%s/*" % (component_dir, component_dir,
component_dir)
script_dict = {
"artifact":
artifact_and_version,
"job_name":
job_name,
"jar_dirs":
jar_dirs,
"run_dir":
supervisor_client.get_run_dir(),
"params":
'-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn +
'-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory +
'-XX:MaxPermSize=%dm ' % job.max_perm_size +
'-XX:+DisableExplicitGC ' + '-XX:+HeapDumpOnOutOfMemoryError ' +
'-XX:HeapDumpPath=$log_dir ' + '-XX:+PrintGCApplicationStoppedTime ' +
'-XX:+UseConcMarkSweepGC ' + '-XX:CMSInitiatingOccupancyFraction=80 ' +
'-XX:+UseMembar ' + '-verbose:gc ' + '-XX:+PrintGCDetails ' +
'-XX:+PrintGCDateStamps ' +
'-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name +
'-Dproc_%s ' % job_name + '-Djava.net.preferIPv4Stack=true ' +
'-Dhdfs.log.dir=$log_dir ' + '-Dhdfs.pid=$pid ' +
'-Dhdfs.cluster=%s ' % args.hdfs_config.cluster.name +
'-Dhadoop.policy.file=hadoop-policy.xml ' +
'-Dhadoop.home.dir=$package_dir ' +
'-Djava.security.krb5.conf=$run_dir/krb5.conf ' +
'-Dhadoop.id.str=%s ' % args.remote_user,
}
# config security-related params
if deploy_utils.is_security_enabled(args):
class_path_root = "$package_dir/share/hadoop/"
boot_class_path = "%s/common/lib/hadoop-security-%s.jar" % (
class_path_root, args.hdfs_config.cluster.version)
script_dict["params"] += ('-Xbootclasspath/p:%s ' % boot_class_path +
'-Dkerberos.instance=hadoop ')
# finally, add the job's main class name
script_dict["params"] += (get_job_specific_params(args, job_name) +
JOB_MAIN_CLASS[job_name])
return script_dict
def generate_bootstrap_script(args, host, job_name, host_id):
supervisor_client = deploy_utils.get_supervisor_client(
host, "zookeeper", args.zookeeper_config.cluster.name, job_name
)
data_dir = supervisor_client.get_available_data_dirs()[0]
myid_file = "%s/%s" % (data_dir, MYID_FILE)
script_dict = {"myid_file": myid_file, "host_id": host_id}
return deploy_utils.create_run_script("%s/bootstrap_zk.sh.tmpl" % deploy_utils.get_template_dir(), script_dict)
def generate_start_script(args, host, job_name):
supervisor_client = deploy_utils.get_supervisor_client(host,
"zookeeper", args.zookeeper_config.cluster.name, job_name)
run_dir = supervisor_client.get_run_dir()
artifact_and_version = "zookeeper-" + args.zookeeper_config.cluster.version
component_dir = "$package_dir"
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir)
job = args.zookeeper_config.jobs["zookeeper"]
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": run_dir,
"params":
'-Xmx%dm ' % job.xmx +
'-Xms%dm ' % job.xms +
'-Xmn%dm ' % job.xmn +
'-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory +
'-XX:MaxPermSize=%dm ' % job.max_perm_size +
'-XX:+DisableExplicitGC ' +
'-XX:+HeapDumpOnOutOfMemoryError ' +
'-XX:HeapDumpPath=$log_dir ' +
'-XX:+PrintGCApplicationStoppedTime ' +
'-XX:+UseConcMarkSweepGC ' +
'-XX:CMSInitiatingOccupancyFraction=80 ' +
'-XX:+UseMembar ' +
'-verbose:gc ' +
'-XX:+PrintGCDetails ' +
'-XX:+PrintGCDateStamps ' +
'-Xloggc:$run_dir/stdout/zk_gc_${start_time}.log ' +
'-Djava.net.preferIPv4Stack=true ' +
'-Dzookeeper.log.dir=$log_dir ' +
'-Dzookeeper.cluster=%s ' % args.zookeeper_config.cluster.name +
'-Dzookeeper.tracelog.dir=$log_dir ',
}
# Config security
if deploy_utils.is_security_enabled(args):
script_dict["params"] += '-Dzookeeper.superUser=zk_admin '
script_dict["params"] += '-Djava.security.auth.login.config=$run_dir/jaas.conf '
script_dict["params"] += '-Djava.security.krb5.conf=$run_dir/krb5.conf '
script_dict["params"] += 'org.apache.zookeeper.server.quorum.QuorumPeerMain '
script_dict["params"] += '$run_dir/zookeeper.cfg'
return deploy_utils.create_run_script(
'%s/start.sh.tmpl' % deploy_utils.get_template_dir(),
script_dict)
def generate_start_script(args, host, job_name):
supervisor_client = deploy_utils.get_supervisor_client(host,
"zookeeper", args.zookeeper_config.cluster.name, job_name)
run_dir = supervisor_client.get_run_dir()
artifact_and_version = "zookeeper-" + args.zookeeper_config.cluster.version
component_dir = "$package_dir"
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir)
job = args.zookeeper_config.jobs["zookeeper"]
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": run_dir,
"params":
'-Xmx%dm ' % job.xmx +
'-Xms%dm ' % job.xms +
'-Xmn%dm ' % job.xmn +
'-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory +
'-XX:MaxPermSize=%dm ' % job.max_perm_size +
'-XX:+DisableExplicitGC ' +
'-XX:+HeapDumpOnOutOfMemoryError ' +
'-XX:HeapDumpPath=$log_dir ' +
'-XX:+PrintGCApplicationStoppedTime ' +
'-XX:+UseConcMarkSweepGC ' +
'-XX:CMSInitiatingOccupancyFraction=80 ' +
'-XX:+UseMembar ' +
'-verbose:gc ' +
'-XX:+PrintGCDetails ' +
'-XX:+PrintGCDateStamps ' +
'-Xloggc:$run_dir/stdout/zk_gc_${start_time}.log ' +
'-Djava.net.preferIPv4Stack=true ' +
'-Dzookeeper.log.dir=$log_dir ' +
'-Dzookeeper.cluster=%s ' % args.zookeeper_config.cluster.name +
'-Dzookeeper.tracelog.dir=$log_dir ',
}
# Config security
if deploy_utils.is_security_enabled(args):
script_dict["params"] += '-Dzookeeper.superUser=zk_admin '
script_dict["params"] += '-Djava.security.auth.login.config=$run_dir/jaas.conf '
script_dict["params"] += '-Djava.security.krb5.conf=$run_dir/krb5.conf '
script_dict["params"] += 'org.apache.zookeeper.server.quorum.QuorumPeerMain '
script_dict["params"] += '$run_dir/zookeeper.cfg'
return deploy_utils.create_run_script(
'%s/start.sh.tmpl' % deploy_utils.get_template_dir(),
script_dict)
def generate_bootstrap_script(args, host, job_name, host_id):
supervisor_client = deploy_utils.get_supervisor_client(host,
"zookeeper", args.zookeeper_config.cluster.name, job_name)
data_dir = supervisor_client.get_available_data_dirs()[0]
myid_file = "%s/%s" % (data_dir, MYID_FILE)
script_dict = {
'myid_file': myid_file,
'host_id': host_id,
}
return deploy_utils.create_run_script(
'%s/bootstrap_zk.sh.tmpl' % deploy_utils.get_template_dir(),
script_dict)
def generate_bootstrap_script(args, host, job_name, host_id, instance_id):
supervisor_client = deploy_utils.get_supervisor_client(host,
"zookeeper", args.zookeeper_config.cluster.name, job_name, instance_id=instance_id)
data_dir = supervisor_client.get_available_data_dirs()[0]
myid_file = "%s/%s" % (data_dir, MYID_FILE)
hosts = args.zookeeper_config.jobs["zookeeper"].hosts
task_id = deploy_utils.get_task_id(hosts, host_id, instance_id)
script_dict = {
'myid_file': myid_file,
'host_id': task_id,
}
return deploy_utils.create_run_script(
'%s/bootstrap_zk.sh.tmpl' % deploy_utils.get_template_dir(),
script_dict)
def generate_run_scripts_params(args, host, job_name):
supervisor_client = deploy_utils.get_supervisor_client(
host, "impala", args.impala_config.cluster.name, job_name)
job = args.impala_config.jobs[job_name]
impalad = args.impala_config.jobs["impalad"]
statestored = args.impala_config.jobs["statestored"]
artifact_and_version = "impala-" + args.impala_config.cluster.version
script_dict = {
"artifact":
artifact_and_version,
"job_name":
job_name,
"run_dir":
supervisor_client.get_run_dir(),
"ticket_cache":
"$run_dir/impala.tc",
"params":
"-webserver_port=%d " % (job.base_port + 1) + "-be_port=%d " %
(impalad.base_port + 2) + "-planservice_port=%d " %
(impalad.base_port + 3) +
"-state_store_port=%d " % statestored.base_port +
"-state_store_subscriber_port=%d " % (statestored.base_port + 1) +
"-mem_limit=20% " + # TODO make this configurable
"-state_store_host=%s " % statestored.hosts[0] +
"-kerberos_reinit_interval=1200 " + # 20hours
"-webserver_doc_root=$run_dir/package " +
"-webserver_interface=%s " % host +
#"-use_statestore=false " +
"-log_dir=$run_dir/log " + "-v=2 " + "-logbuflevel=-1 " +
"-sasl_path=$run_dir/package/lib/sasl2 ",
}
if job_name == "impalad":
script_dict["params"] += "-beeswax_port=%d " % impalad.base_port
script_dict["params"] += "-hs2_port=%d " % (impalad.base_port + 4)
if deploy_utils.is_security_enabled(args):
script_dict["params"] += "-principal=%s/hadoop@%s " % (
args.impala_config.cluster.kerberos_username
or "impala", args.impala_config.cluster.kerberos_realm)
script_dict["params"] += "-keytab_file=%s/%s.keytab " % (
deploy_utils.HADOOP_CONF_PATH,
args.impala_config.cluster.kerberos_username or "impala")
script_dict["params"] += "-tgt_file=$run_dir/impala.tc "
return script_dict
def generate_run_scripts_params(args, host, job_name):
supervisor_client = deploy_utils.get_supervisor_client(host,
"impala", args.impala_config.cluster.name, job_name)
job = args.impala_config.jobs[job_name]
impalad = args.impala_config.jobs["impalad"]
statestored = args.impala_config.jobs["statestored"]
artifact_and_version = "impala-" + args.impala_config.cluster.version
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"run_dir": supervisor_client.get_run_dir(),
"ticket_cache": "$run_dir/impala.tc",
"params":
"-webserver_port=%d " % (job.base_port + 1) +
"-be_port=%d " % (impalad.base_port + 2) +
"-planservice_port=%d " % (impalad.base_port + 3) +
"-state_store_port=%d " % statestored.base_port +
"-state_store_subscriber_port=%d " % (statestored.base_port + 1) +
"-mem_limit=20% " + # TODO make this configurable
"-state_store_host=%s " % statestored.hosts[0] +
"-kerberos_reinit_interval=1200 " + # 20hours
"-webserver_doc_root=$run_dir/package " +
"-webserver_interface=%s " % host +
#"-use_statestore=false " +
"-log_dir=$run_dir/log " +
"-v=2 " +
"-logbuflevel=-1 " +
"-sasl_path=$run_dir/package/lib/sasl2 ",
}
if job_name == "impalad":
script_dict["params"] += "-beeswax_port=%d " % impalad.base_port
script_dict["params"] += "-hs2_port=%d " % (impalad.base_port + 4)
if deploy_utils.is_security_enabled(args):
script_dict["params"] += "-principal=%s/hadoop@%s " % (
args.impala_config.cluster.kerberos_username or "impala",
args.impala_config.cluster.kerberos_realm)
script_dict["params"] += "-keytab_file=%s/%s.keytab " % (
deploy_utils.HADOOP_CONF_PATH,
args.impala_config.cluster.kerberos_username or "impala")
script_dict["params"] += "-tgt_file=$run_dir/impala.tc "
return script_dict
def generate_metrics_config(args, host, job_name, instance_id=-1):
job = args.yarn_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host, "yarn",
args.yarn_config.cluster.name, job_name, instance_id=instance_id)
ganglia_switch = "# "
if args.yarn_config.cluster.ganglia_address:
ganglia_switch = ""
config_dict = {
"job_name": job_name,
"period": 10,
"data_dir": supervisor_client.get_log_dir(),
"ganglia_address": args.yarn_config.cluster.ganglia_address,
"ganglia_switch": ganglia_switch,
}
local_path = "%s/hadoop-metrics2.properties.tmpl" % deploy_utils.get_template_dir()
template = deploy_utils.Template(open(local_path, "r").read())
return template.substitute(config_dict)
def generate_run_scripts_params(args, host, job_name, host_id, instance_id):
job = args.kafka_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"kafka", args.kafka_config.cluster.name, job_name, instance_id=instance_id)
artifact_and_version = "kafka-" + args.kafka_config.cluster.version
jar_dirs = "$package_dir/*"
log_level = deploy_utils.get_service_log_level(args, args.kafka_config)
params = job.get_arguments(args, args.kafka_config.cluster, args.kafka_config.jobs,
args.kafka_config.arguments_dict, job_name, host_id, instance_id)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"params": params,
}
return script_dict
def generate_run_scripts_params(args, host, job_name):
job = args.hbase_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"hbase", args.hbase_config.cluster.name, job_name)
artifact_and_version = "hbase-" + args.hbase_config.cluster.version
component_dir = "$package_dir/"
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir, component_dir)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"params":
'-Xmx%dm ' % job.xmx +
'-Xms%dm ' % job.xms +
'-Xmn%dm ' % job.xmn +
'-Xss256k ' +
'-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory +
'-XX:MaxPermSize=%dm ' % job.max_perm_size +
'-XX:PermSize=%dm ' % job.max_perm_size +
'-XX:+HeapDumpOnOutOfMemoryError ' +
'-XX:HeapDumpPath=$log_dir ' +
'-XX:+PrintGCApplicationStoppedTime ' +
'-XX:+UseConcMarkSweepGC ' +
'-verbose:gc ' +
'-XX:+PrintGCDetails ' +
'-XX:+PrintGCDateStamps ' +
'-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name +
'-XX:+UseMembar ' +
'-XX:SurvivorRatio=1 ' +
'-XX:+UseCMSCompactAtFullCollection ' +
'-XX:CMSInitiatingOccupancyFraction=75 ' +
'-XX:+UseCMSInitiatingOccupancyOnly ' +
'-XX:+CMSParallelRemarkEnabled ' +
'-XX:+UseNUMA ' +
'-XX:+CMSClassUnloadingEnabled ' +
'-XX:+PrintSafepointStatistics ' +
'-XX:PrintSafepointStatisticsCount=1 ' +
'-XX:+PrintHeapAtGC ' +
'-XX:+PrintTenuringDistribution ' +
'-XX:CMSMaxAbortablePrecleanTime=10000 ' +
'-XX:TargetSurvivorRatio=80 ' +
'-XX:+UseGCLogFileRotation ' +
'-XX:NumberOfGCLogFiles=100 ' +
'-XX:GCLogFileSize=128m ' +
'-XX:CMSWaitDuration=2000 ' +
'-XX:+CMSScavengeBeforeRemark ' +
'-XX:+PrintPromotionFailure ' +
'-XX:ConcGCThreads=8 ' +
'-XX:ParallelGCThreads=8 ' +
'-XX:PretenureSizeThreshold=4m ' +
'-XX:+CMSConcurrentMTEnabled ' +
'-XX:+ExplicitGCInvokesConcurrent ' +
'-XX:+SafepointTimeout ' +
'-XX:MonitorBound=16384 ' +
'-XX:OldPLABSize=16 ' +
'-XX:-ResizeOldPLAB ' +
'-XX:-UseBiasedLocking ' +
'-Dproc_%s ' % job_name +
'-Djava.net.preferIPv4Stack=true ' +
'-Dhbase.log.dir=$log_dir ' +
'-Dhbase.pid=$pid ' +
'-Dhbase.cluster=%s ' % args.hbase_config.cluster.name +
'-Dhbase.policy.file=hbase-policy.xml ' +
'-Dhbase.home.dir=$package_dir ' +
'-Djava.security.krb5.conf=$run_dir/krb5.conf ' +
'-Dhbase.id.str=%s ' % args.remote_user +
get_job_specific_params(args, job_name),
}
if deploy_utils.is_security_enabled(args):
jaas_path = "%s/jaas.conf" % supervisor_client.get_run_dir()
script_dict["params"] += "-Djava.security.auth.login.config=%s " % jaas_path
boot_class_path = ("$package_dir/lib/hadoop-security-%s.jar" %
args.hdfs_config.cluster.version)
script_dict["params"] += "-Xbootclasspath/p:%s " % boot_class_path
script_dict["params"] += JOB_MAIN_CLASS[job_name]
return script_dict
def generate_run_scripts_params(args, host, job_name):
job = args.hdfs_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"hdfs", args.hdfs_config.cluster.name, job_name)
artifact_and_version = "hadoop-" + args.hdfs_config.cluster.version
jar_dirs = ""
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
for component in ["common", "hdfs"]:
if jar_dirs: jar_dirs += ":"
component_dir = ("$package_dir/share/hadoop/%s" % component)
jar_dirs += "%s/:%s/lib/*:%s/*" % (
component_dir, component_dir, component_dir)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"params":
'-Xmx%dm ' % job.xmx +
'-Xms%dm ' % job.xms +
'-Xmn%dm ' % job.xmn +
'-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory +
'-XX:MaxPermSize=%dm ' % job.max_perm_size +
'-XX:+DisableExplicitGC ' +
'-XX:+HeapDumpOnOutOfMemoryError ' +
'-XX:HeapDumpPath=$log_dir ' +
'-XX:+PrintGCApplicationStoppedTime ' +
'-XX:+UseConcMarkSweepGC ' +
'-XX:CMSInitiatingOccupancyFraction=80 ' +
'-XX:+UseMembar ' +
'-verbose:gc ' +
'-XX:+PrintGCDetails ' +
'-XX:+PrintGCDateStamps ' +
'-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name +
'-Dproc_%s ' % job_name +
'-Djava.net.preferIPv4Stack=true ' +
'-Dhdfs.log.dir=$log_dir ' +
'-Dhdfs.pid=$pid ' +
'-Dhdfs.cluster=%s ' % args.hdfs_config.cluster.name +
'-Dhadoop.policy.file=hadoop-policy.xml ' +
'-Dhadoop.home.dir=$package_dir ' +
'-Djava.security.krb5.conf=$run_dir/krb5.conf ' +
'-Dhadoop.id.str=%s ' % args.remote_user,
}
# config security-related params
if deploy_utils.is_security_enabled(args):
class_path_root = "$package_dir/share/hadoop/"
boot_class_path = "%s/common/lib/hadoop-security-%s.jar" % (
class_path_root, args.hdfs_config.cluster.version)
script_dict["params"] += ('-Xbootclasspath/p:%s ' % boot_class_path +
'-Dkerberos.instance=hadoop ')
# finally, add the job's main class name
script_dict["params"] += (get_job_specific_params(args, job_name) +
JOB_MAIN_CLASS[job_name])
return script_dict
def generate_run_scripts_params(args, host, job_name):
job = args.yarn_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(host,
"yarn", args.yarn_config.cluster.name, job_name)
artifact_and_version = "hadoop-" + args.yarn_config.cluster.version
jar_dirs = ""
for component in ["common", "mapreduce", "yarn", "hdfs"]:
if jar_dirs: jar_dirs += ":"
component_dir = ("$package_dir/share/hadoop/%s" % component)
jar_dirs += "%s/:%s/lib/*:%s/*" % (
component_dir, component_dir, component_dir)
service_env = ""
for component_path in ["HADOOP_COMMON_HOME", "HADOOP_HDFS_HOME", "YARN_HOME"]:
service_env += "export %s=$package_dir\n" % (component_path)
script_dict = {
"artifact": artifact_and_version,
"job_name": job_name,
"jar_dirs": jar_dirs,
"run_dir": supervisor_client.get_run_dir(),
"service_env": service_env,
"params":
'-Xmx%dm ' % job.xmx +
'-Xms%dm ' % job.xms +
'-Xmn%dm ' % job.xmn +
'-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory +
'-XX:MaxPermSize=%dm ' % job.max_perm_size +
'-XX:+DisableExplicitGC ' +
'-XX:+HeapDumpOnOutOfMemoryError ' +
'-XX:HeapDumpPath=$log_dir ' +
'-XX:+PrintGCApplicationStoppedTime ' +
'-XX:+UseConcMarkSweepGC ' +
'-XX:CMSInitiatingOccupancyFraction=80 ' +
'-XX:+UseMembar ' +
'-verbose:gc ' +
'-XX:+PrintGCDetails ' +
'-XX:+PrintGCDateStamps ' +
'-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name +
'-Dproc_%s ' % job_name +
'-Djava.net.preferIPv4Stack=true ' +
'-Dyarn.log.dir=$log_dir ' +
'-Dyarn.pid=$pid ' +
'-Dyarn.cluster=%s ' % args.yarn_config.cluster.name +
'-Dhadoop.policy.file=hadoop-policy.xml ' +
'-Dhadoop.home.dir=$package_dir ' +
'-Dhadoop.id.str=%s ' % args.remote_user +
'-Djava.security.krb5.conf=$run_dir/krb5.conf ' +
get_job_specific_params(args, job_name)
}
if deploy_utils.is_security_enabled(args):
class_path_root = "$package_dir/share/hadoop"
boot_class_path = ("%s/common/lib/hadoop-security-%s.jar" % (
class_path_root, args.hdfs_config.cluster.version))
script_dict["params"] += "-Xbootclasspath/p:%s " % boot_class_path
script_dict["params"] += JOB_MAIN_CLASS[job_name]
return script_dict
def generate_run_scripts_params(args, host, job_name):
job = args.hbase_config.jobs[job_name]
supervisor_client = deploy_utils.get_supervisor_client(
host, "hbase", args.hbase_config.cluster.name, job_name)
artifact_and_version = "hbase-" + args.hbase_config.cluster.version
component_dir = "$package_dir/"
# must include both [dir]/ and [dir]/* as [dir]/* only import all jars under
# this dir but we also need access the webapps under this dir.
jar_dirs = "%s/:%s/lib/*:%s/*" % (component_dir, component_dir,
component_dir)
script_dict = {
"artifact":
artifact_and_version,
"job_name":
job_name,
"jar_dirs":
jar_dirs,
"run_dir":
supervisor_client.get_run_dir(),
"params":
'-Xmx%dm ' % job.xmx + '-Xms%dm ' % job.xms + '-Xmn%dm ' % job.xmn +
'-Xss256k ' + '-XX:MaxDirectMemorySize=%dm ' % job.max_direct_memory +
'-XX:MaxPermSize=%dm ' % job.max_perm_size +
'-XX:PermSize=%dm ' % job.max_perm_size +
'-XX:+HeapDumpOnOutOfMemoryError ' + '-XX:HeapDumpPath=$log_dir ' +
'-XX:+PrintGCApplicationStoppedTime ' + '-XX:+UseConcMarkSweepGC ' +
'-verbose:gc ' + '-XX:+PrintGCDetails ' + '-XX:+PrintGCDateStamps ' +
'-Xloggc:$run_dir/stdout/%s_gc_${start_time}.log ' % job_name +
'-XX:+UseMembar ' + '-XX:SurvivorRatio=1 ' +
'-XX:+UseCMSCompactAtFullCollection ' +
'-XX:CMSInitiatingOccupancyFraction=75 ' +
'-XX:+UseCMSInitiatingOccupancyOnly ' +
'-XX:+CMSParallelRemarkEnabled ' + '-XX:+UseNUMA ' +
'-XX:+CMSClassUnloadingEnabled ' + '-XX:+PrintSafepointStatistics ' +
'-XX:PrintSafepointStatisticsCount=1 ' + '-XX:+PrintHeapAtGC ' +
'-XX:+PrintTenuringDistribution ' +
'-XX:CMSMaxAbortablePrecleanTime=10000 ' +
'-XX:TargetSurvivorRatio=80 ' + '-XX:+UseGCLogFileRotation ' +
'-XX:NumberOfGCLogFiles=100 ' + '-XX:GCLogFileSize=128m ' +
'-XX:CMSWaitDuration=2000 ' + '-XX:+CMSScavengeBeforeRemark ' +
'-XX:+PrintPromotionFailure ' + '-XX:ConcGCThreads=8 ' +
'-XX:ParallelGCThreads=8 ' + '-XX:PretenureSizeThreshold=4m ' +
'-XX:+CMSConcurrentMTEnabled ' + '-XX:+ExplicitGCInvokesConcurrent ' +
'-XX:+SafepointTimeout ' + '-XX:MonitorBound=16384 ' +
'-XX:OldPLABSize=16 ' + '-XX:-ResizeOldPLAB ' +
'-XX:-UseBiasedLocking ' + '-Dproc_%s ' % job_name +
'-Djava.net.preferIPv4Stack=true ' + '-Dhbase.log.dir=$log_dir ' +
'-Dhbase.pid=$pid ' +
'-Dhbase.cluster=%s ' % args.hbase_config.cluster.name +
'-Dhbase.policy.file=hbase-policy.xml ' +
'-Dhbase.home.dir=$package_dir ' +
'-Djava.security.krb5.conf=$run_dir/krb5.conf ' +
'-Dhbase.id.str=%s ' % args.remote_user +
get_job_specific_params(args, job_name),
}
if deploy_utils.is_security_enabled(args):
jaas_path = "%s/jaas.conf" % supervisor_client.get_run_dir()
script_dict[
"params"] += "-Djava.security.auth.login.config=%s " % jaas_path
boot_class_path = ("$package_dir/lib/hadoop-security-%s.jar" %
args.hdfs_config.cluster.version)
script_dict["params"] += "-Xbootclasspath/p:%s " % boot_class_path
script_dict["params"] += JOB_MAIN_CLASS[job_name]
return script_dict
