def yarn(name = None):
import params
XmlConfig("mapred-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['mapred-site'],
owner=params.yarn_user,
mode='f'
)
XmlConfig("yarn-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['yarn-site'],
owner=params.yarn_user,
mode='f',
configuration_attributes=params.config['configuration_attributes']['yarn-site']
)
XmlConfig("capacity-scheduler.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
owner=params.yarn_user,
mode='f'
)
if params.service_map.has_key(name):
service_name = params.service_map[name]
ServiceConfig(service_name,
action="change_user",
username = params.yarn_user,
password = Script.get_password(params.yarn_user))
def zookeeper(type=None, upgrade_type=None):
import params
configFile("zoo.cfg", template_name="zoo.cfg.j2", mode="f")
configFile("configuration.xsl",
template_name="configuration.xsl.j2",
mode="f")
ServiceConfig(params.zookeeper_win_service_name,
action="change_user",
username=params.zk_user,
password=Script.get_password(params.zk_user))
Directory(params.zk_data_dir,
owner=params.zk_user,
mode="(OI)(CI)F",
create_parents=True)
if (params.log4j_props != None):
File(os.path.join(params.config_dir, "log4j.properties"),
mode="f",
owner=params.zk_user,
content=params.log4j_props)
elif (os.path.exists(os.path.join(params.config_dir, "log4j.properties"))):
File(os.path.join(params.config_dir, "log4j.properties"),
mode="f",
owner=params.zk_user)
if type == 'server':
myid = str(sorted(params.zookeeper_hosts).index(params.hostname) + 1)
File(os.path.join(params.zk_data_dir, "myid"),
owner=params.zk_user,
mode="f",
content=myid)
def knox():
import params
XmlConfig(
"gateway-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']['gateway-site'],
configuration_attributes=params.config['configuration_attributes']
['gateway-site'],
owner=params.knox_user)
# Manually overriding service logon user & password set by the installation package
ServiceConfig(params.knox_gateway_win_service_name,
action="change_user",
username=params.knox_user,
password=Script.get_password(params.knox_user))
File(os.path.join(params.knox_conf_dir, "gateway-log4j.properties"),
owner=params.knox_user,
content=params.gateway_log4j)
File(os.path.join(params.knox_conf_dir, "topologies", "default.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.topology_template))
if params.admin_topology_template:
File(os.path.join(params.knox_conf_dir, "topologies", "admin.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.admin_topology_template))
if params.version_formatted and check_stack_feature(
StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted):
knoxsso_topology_template_content = get_config("knoxsso-topology")
if knoxsso_topology_template_content:
File(os.path.join(params.knox_conf_dir, "topologies",
"knoxsso.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.knoxsso_topology_template))
if params.security_enabled:
TemplateConfig(os.path.join(params.knox_conf_dir,
"krb5JAASLogin.conf"),
owner=params.knox_user,
template_tag=None)
if not os.path.isfile(params.knox_master_secret_path):
cmd = format(
'cmd /C {knox_client_bin} create-master --master {knox_master_secret!p}'
)
Execute(cmd)
cmd = format(
'cmd /C {knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}'
)
Execute(cmd)
def flume(action=None):
import params
from service_mapping import flume_win_service_name
if action == 'config':
ServiceConfig(flume_win_service_name,
action="configure",
start_type="manual")
ServiceConfig(flume_win_service_name,
action="change_user",
username=params.flume_user,
password=Script.get_password(params.flume_user))
# remove previously defined meta's
for n in find_expected_agent_names(params.flume_conf_dir):
os.unlink(
os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'))
flume_agents = {}
if params.flume_conf_content is not None:
flume_agents = build_flume_topology(params.flume_conf_content)
for agent in flume_agents.keys():
flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent)
flume_agent_conf_file = os.path.join(flume_agent_conf_dir,
'flume.conf')
flume_agent_meta_file = os.path.join(flume_agent_conf_dir,
'ambari-meta.json')
flume_agent_log4j_file = os.path.join(flume_agent_conf_dir,
'log4j.properties')
flume_agent_env_file = os.path.join(flume_agent_conf_dir,
'flume-env.ps1')
Directory(flume_agent_conf_dir)
PropertiesFile(flume_agent_conf_file,
properties=flume_agents[agent])
File(flume_agent_log4j_file,
content=InlineTemplate(params.flume_log4j_content,
agent_name=agent)),
File(flume_agent_meta_file,
content=json.dumps(ambari_meta(agent, flume_agents[agent])))
File(flume_agent_env_file,
owner=params.flume_user,
content=InlineTemplate(params.flume_env_sh_template))
if params.has_metric_collector:
File(os.path.join(flume_agent_conf_dir,
"flume-metrics2.properties"),
owner=params.flume_user,
content=Template("flume-metrics2.properties.j2"))
def webhcat():
import params
XmlConfig("webhcat-site.xml",
conf_dir=params.hcat_config_dir,
configurations=params.config['configurations']['webhcat-site'])
# Manually overriding service logon user & password set by the installation package
ServiceConfig(params.webhcat_server_win_service_name,
action="change_user",
username=params.webhcat_user,
password=Script.get_password(params.webhcat_user))
def ldap():
import params
# Manually overriding service logon user & password set by the installation package
ServiceConfig(params.knox_ldap_win_service_name,
action="change_user",
username=params.knox_user,
password=Script.get_password(params.knox_user))
_ldap_common()
def hive(name=None):
import params
XmlConfig("hive-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['hive-site'],
owner=params.hive_user,
configuration_attributes=params.config['configuration_attributes']['hive-site']
)
if name in ["hiveserver2","metastore"]:
# Manually overriding service logon user & password set by the installation package
service_name = params.service_map[name]
ServiceConfig(service_name,
action="change_user",
username = params.hive_user,
password = Script.get_password(params.hive_user))
Execute(format("cmd /c hadoop fs -mkdir -p {hive_warehouse_dir}"), logoutput=True, user=params.hadoop_user)
if name == 'metastore':
if params.init_metastore_schema:
check_schema_created_cmd = format('cmd /c "{hive_bin}\\hive.cmd --service schematool -info '
'-dbType {hive_metastore_db_type} '
'-userName {hive_metastore_user_name} '
'-passWord {hive_metastore_user_passwd!p}'
'&set EXITCODE=%ERRORLEVEL%&exit /B %EXITCODE%"', #cmd "feature", propagate the process exit code manually
hive_bin=params.hive_bin,
hive_metastore_db_type=params.hive_metastore_db_type,
hive_metastore_user_name=params.hive_metastore_user_name,
hive_metastore_user_passwd=params.hive_metastore_user_passwd)
try:
Execute(check_schema_created_cmd)
except Fail:
create_schema_cmd = format('cmd /c {hive_bin}\\hive.cmd --service schematool -initSchema '
'-dbType {hive_metastore_db_type} '
'-userName {hive_metastore_user_name} '
'-passWord {hive_metastore_user_passwd!p}',
hive_bin=params.hive_bin,
hive_metastore_db_type=params.hive_metastore_db_type,
hive_metastore_user_name=params.hive_metastore_user_name,
hive_metastore_user_passwd=params.hive_metastore_user_passwd)
Execute(create_schema_cmd,
user = params.hive_user,
logoutput=True
)
if name == "hiveserver2":
if params.hive_execution_engine == "tez":
# Init the tez app dir in hadoop
script_file = __file__.replace('/', os.sep)
cmd_file = os.path.normpath(os.path.join(os.path.dirname(script_file), "..", "files", "hiveTezSetup.cmd"))
Execute("cmd /c " + cmd_file, logoutput=True, user=params.hadoop_user)
def hive(name=None):
import params
XmlConfig("hive-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['hive-site'],
owner=params.hive_user,
configuration_attributes=params.config['configuration_attributes']['hive-site']
)
if name in ["hiveserver2","metastore"]:
# Manually overriding service logon user & password set by the installation package
service_name = params.service_map[name]
ServiceConfig(service_name,
action="change_user",
username = params.hive_user,
password = Script.get_password(params.hive_user))
Execute(format("cmd /c hadoop fs -mkdir -p {hive_warehouse_dir}"), logoutput=True, user=params.hadoop_user)
if name == 'metastore':
if params.init_metastore_schema:
check_schema_created_cmd = format('cmd /c "{hive_bin}\\hive.cmd --service schematool -info '
'-dbType {hive_metastore_db_type} '
'-userName {hive_metastore_user_name} '
'-passWord {hive_metastore_user_passwd!p}'
'&set EXITCODE=%ERRORLEVEL%&exit /B %EXITCODE%"', #cmd "feature", propagate the process exit code manually
hive_bin=params.hive_bin,
hive_metastore_db_type=params.hive_metastore_db_type,
hive_metastore_user_name=params.hive_metastore_user_name,
hive_metastore_user_passwd=params.hive_metastore_user_passwd)
try:
Execute(check_schema_created_cmd)
except Fail:
create_schema_cmd = format('cmd /c {hive_bin}\\hive.cmd --service schematool -initSchema '
'-dbType {hive_metastore_db_type} '
'-userName {hive_metastore_user_name} '
'-passWord {hive_metastore_user_passwd!p}',
hive_bin=params.hive_bin,
hive_metastore_db_type=params.hive_metastore_db_type,
hive_metastore_user_name=params.hive_metastore_user_name,
hive_metastore_user_passwd=params.hive_metastore_user_passwd)
Execute(create_schema_cmd,
user = params.hive_user,
logoutput=True
)
if name == "hiveserver2":
if params.hive_execution_engine == "tez":
# Init the tez app dir in hadoop
script_file = __file__.replace('/', os.sep)
cmd_file = os.path.normpath(os.path.join(os.path.dirname(script_file), "..", "files", "hiveTezSetup.cmd"))
Execute("cmd /c " + cmd_file, logoutput=True, user=params.hadoop_user)
def oozie(is_server=False):
import params
from status_params import oozie_server_win_service_name
XmlConfig(
"oozie-site.xml",
conf_dir=params.oozie_conf_dir,
configurations=params.config['configurations']['oozie-site'],
owner=params.oozie_user,
mode='f',
configuration_attributes=params.config['configuration_attributes']
['oozie-site'])
File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_cmd_template))
Directory(
params.oozie_tmp_dir,
owner=params.oozie_user,
recursive=True,
)
if is_server:
# Manually overriding service logon user & password set by the installation package
ServiceConfig(oozie_server_win_service_name,
action="change_user",
username=params.oozie_user,
password=Script.get_password(params.oozie_user))
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar"))
webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server",
"webapps", "oozie", "WEB-INF", "lib",
"sqljdbc4.jar")
if os.path.isfile(webapps_sqljdbc_path):
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"), webapps_sqljdbc_path)
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_home, "share", "lib", "oozie",
"sqljdbc4.jar"))
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_home, "temp", "WEB-INF", "lib",
"sqljdbc4.jar"))
def storm(name=None):
import params
yaml_config("storm.yaml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['storm-site'],
owner=params.storm_user)
if params.service_map.has_key(name):
service_name = params.service_map[name]
ServiceConfig(service_name,
action="change_user",
username=params.storm_user,
password=Script.get_password(params.storm_user))
def storm(name=None):
import params
yaml_config("storm.yaml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['storm-site'],
owner=params.storm_user
)
if params.service_map.has_key(name):
service_name = params.service_map[name]
ServiceConfig(service_name,
action="change_user",
username = params.storm_user,
password = Script.get_password(params.storm_user))
def hbase(name=None):
import params
XmlConfig("hbase-site.xml",
conf_dir = params.hbase_conf_dir,
configurations = params.config['configurations']['hbase-site'],
configuration_attributes=params.config['configuration_attributes']['hbase-site']
)
if params.service_map.has_key(name):
# Manually overriding service logon user & password set by the installation package
service_name = params.service_map[name]
ServiceConfig(service_name,
action="change_user",
username = params.hbase_user,
password = Script.get_password(params.hbase_user))
def hdfs(component=None):
import params
if component == "namenode":
directories = params.dfs_name_dir.split(",")
Directory(directories,
owner=params.hdfs_user,
mode="(OI)(CI)F",
create_parents=True)
File(
params.exclude_file_path,
content=Template("exclude_hosts_list.j2"),
owner=params.hdfs_user,
mode="f",
)
if params.hdfs_include_file:
File(
params.include_file_path,
content=Template("include_hosts_list.j2"),
owner=params.hdfs_user,
mode="f",
)
pass
if params.service_map.has_key(component):
service_name = params.service_map[component]
ServiceConfig(service_name,
action="change_user",
username=params.hdfs_user,
password=Script.get_password(params.hdfs_user))
if "hadoop-policy" in params.config['configurations']:
XmlConfig(
"hadoop-policy.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hadoop-policy'],
owner=params.hdfs_user,
mode="f",
configuration_attributes=params.config['configuration_attributes']
['hadoop-policy'])
XmlConfig(
"hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
owner=params.hdfs_user,
mode="f",
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'])
def rebalancehdfs(self, env):
from ambari_commons.os_windows import UserHelper, run_os_command_impersonated
import params
env.set_params(params)
hdfs_username, hdfs_domain = UserHelper.parse_user_name(
params.hdfs_user, ".")
name_node_parameters = json.loads(params.name_node_params)
threshold = name_node_parameters['threshold']
_print("Starting balancer with threshold = %s\n" % threshold)
def calculateCompletePercent(first, current):
return 1.0 - current.bytesLeftToMove / first.bytesLeftToMove
def startRebalancingProcess(threshold):
rebalanceCommand = 'hdfs balancer -threshold %s' % threshold
return ['cmd', '/C', rebalanceCommand]
command = startRebalancingProcess(threshold)
basedir = os.path.join(env.config.basedir, 'scripts')
_print("Executing command %s\n" % command)
parser = hdfs_rebalance.HdfsParser()
returncode, stdout, err = run_os_command_impersonated(
' '.join(command), hdfs_username,
Script.get_password(params.hdfs_user), hdfs_domain)
for line in stdout.split('\n'):
_print('[balancer] %s %s' % (str(datetime.now()), line))
pl = parser.parseLine(line)
if pl:
res = pl.toJson()
res['completePercent'] = calculateCompletePercent(
parser.initialLine, pl)
self.put_structured_out(res)
elif parser.state == 'PROCESS_FINISED':
_print('[balancer] %s %s' %
(str(datetime.now()), 'Process is finished'))
self.put_structured_out({'completePercent': 1})
break
if returncode != None and returncode != 0:
raise Fail(
'Hdfs rebalance process exited with error. See the log output')
def oozie(is_server=False):
import params
from status_params import oozie_server_win_service_name
XmlConfig("oozie-site.xml",
conf_dir=params.oozie_conf_dir,
configurations=params.config['configurations']['oozie-site'],
owner=params.oozie_user,
mode='f',
configuration_attributes=params.config['configuration_attributes']['oozie-site']
)
File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_cmd_template)
)
Directory(params.oozie_tmp_dir,
owner=params.oozie_user,
recursive = True,
)
if is_server:
# Manually overriding service logon user & password set by the installation package
ServiceConfig(oozie_server_win_service_name,
action="change_user",
username = params.oozie_user,
password = Script.get_password(params.oozie_user))
download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"),
os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar")
)
webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server", "webapps", "oozie", "WEB-INF", "lib", "sqljdbc4.jar")
if os.path.isfile(webapps_sqljdbc_path):
download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"),
webapps_sqljdbc_path
)
download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"),
os.path.join(params.oozie_home, "share", "lib", "oozie", "sqljdbc4.jar")
)
download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"),
os.path.join(params.oozie_home, "temp", "WEB-INF", "lib", "sqljdbc4.jar")
)
def knox():
import params
XmlConfig("gateway-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']['gateway-site'],
configuration_attributes=params.config['configuration_attributes']['gateway-site'],
owner=params.knox_user
)
# Manually overriding service logon user & password set by the installation package
ServiceConfig(params.knox_gateway_win_service_name,
action="change_user",
username = params.knox_user,
password = Script.get_password(params.knox_user))
File(os.path.join(params.knox_conf_dir, "gateway-log4j.properties"),
owner=params.knox_user,
content=params.gateway_log4j
)
File(os.path.join(params.knox_conf_dir, "topologies", "default.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.topology_template)
)
if params.security_enabled:
TemplateConfig( os.path.join(params.knox_conf_dir, "krb5JAASLogin.conf"),
owner = params.knox_user,
template_tag = None
)
if not os.path.isfile(params.knox_master_secret_path):
cmd = format('cmd /C {knox_client_bin} create-master --master {knox_master_secret!p}')
Execute(cmd)
cmd = format('cmd /C {knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}')
Execute(cmd)
