def yarn(name = None): import params XmlConfig("mapred-site.xml", conf_dir=params.config_dir, configurations=params.config['configurations']['mapred-site'], owner=params.yarn_user, mode='f' ) XmlConfig("yarn-site.xml", conf_dir=params.config_dir, configurations=params.config['configurations']['yarn-site'], owner=params.yarn_user, mode='f', configuration_attributes=params.config['configuration_attributes']['yarn-site'] ) XmlConfig("capacity-scheduler.xml", conf_dir=params.config_dir, configurations=params.config['configurations']['capacity-scheduler'], owner=params.yarn_user, mode='f' ) if params.service_map.has_key(name): service_name = params.service_map[name] ServiceConfig(service_name, action="change_user", username = params.yarn_user, password = Script.get_password(params.yarn_user))
def zookeeper(type=None, upgrade_type=None): import params configFile("zoo.cfg", template_name="zoo.cfg.j2", mode="f") configFile("configuration.xsl", template_name="configuration.xsl.j2", mode="f") ServiceConfig(params.zookeeper_win_service_name, action="change_user", username=params.zk_user, password=Script.get_password(params.zk_user)) Directory(params.zk_data_dir, owner=params.zk_user, mode="(OI)(CI)F", create_parents=True) if (params.log4j_props != None): File(os.path.join(params.config_dir, "log4j.properties"), mode="f", owner=params.zk_user, content=params.log4j_props) elif (os.path.exists(os.path.join(params.config_dir, "log4j.properties"))): File(os.path.join(params.config_dir, "log4j.properties"), mode="f", owner=params.zk_user) if type == 'server': myid = str(sorted(params.zookeeper_hosts).index(params.hostname) + 1) File(os.path.join(params.zk_data_dir, "myid"), owner=params.zk_user, mode="f", content=myid)
def knox(): import params XmlConfig( "gateway-site.xml", conf_dir=params.knox_conf_dir, configurations=params.config['configurations']['gateway-site'], configuration_attributes=params.config['configuration_attributes'] ['gateway-site'], owner=params.knox_user) # Manually overriding service logon user & password set by the installation package ServiceConfig(params.knox_gateway_win_service_name, action="change_user", username=params.knox_user, password=Script.get_password(params.knox_user)) File(os.path.join(params.knox_conf_dir, "gateway-log4j.properties"), owner=params.knox_user, content=params.gateway_log4j) File(os.path.join(params.knox_conf_dir, "topologies", "default.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.topology_template)) if params.admin_topology_template: File(os.path.join(params.knox_conf_dir, "topologies", "admin.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.admin_topology_template)) if params.version_formatted and check_stack_feature( StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted): knoxsso_topology_template_content = get_config("knoxsso-topology") if knoxsso_topology_template_content: File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.knoxsso_topology_template)) if params.security_enabled: TemplateConfig(os.path.join(params.knox_conf_dir, "krb5JAASLogin.conf"), owner=params.knox_user, template_tag=None) if not os.path.isfile(params.knox_master_secret_path): cmd = format( 'cmd /C {knox_client_bin} create-master --master {knox_master_secret!p}' ) Execute(cmd) cmd = format( 'cmd /C {knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}' ) Execute(cmd)
def flume(action=None): import params from service_mapping import flume_win_service_name if action == 'config': ServiceConfig(flume_win_service_name, action="configure", start_type="manual") ServiceConfig(flume_win_service_name, action="change_user", username=params.flume_user, password=Script.get_password(params.flume_user)) # remove previously defined meta's for n in find_expected_agent_names(params.flume_conf_dir): os.unlink( os.path.join(params.flume_conf_dir, n, 'ambari-meta.json')) flume_agents = {} if params.flume_conf_content is not None: flume_agents = build_flume_topology(params.flume_conf_content) for agent in flume_agents.keys(): flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent) flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf') flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json') flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties') flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.ps1') Directory(flume_agent_conf_dir) PropertiesFile(flume_agent_conf_file, properties=flume_agents[agent]) File(flume_agent_log4j_file, content=InlineTemplate(params.flume_log4j_content, agent_name=agent)), File(flume_agent_meta_file, content=json.dumps(ambari_meta(agent, flume_agents[agent]))) File(flume_agent_env_file, owner=params.flume_user, content=InlineTemplate(params.flume_env_sh_template)) if params.has_metric_collector: File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"), owner=params.flume_user, content=Template("flume-metrics2.properties.j2"))
def webhcat(): import params XmlConfig("webhcat-site.xml", conf_dir=params.hcat_config_dir, configurations=params.config['configurations']['webhcat-site']) # Manually overriding service logon user & password set by the installation package ServiceConfig(params.webhcat_server_win_service_name, action="change_user", username=params.webhcat_user, password=Script.get_password(params.webhcat_user))
def ldap(): import params # Manually overriding service logon user & password set by the installation package ServiceConfig(params.knox_ldap_win_service_name, action="change_user", username=params.knox_user, password=Script.get_password(params.knox_user)) _ldap_common()
def hive(name=None): import params XmlConfig("hive-site.xml", conf_dir = params.hive_conf_dir, configurations = params.config['configurations']['hive-site'], owner=params.hive_user, configuration_attributes=params.config['configuration_attributes']['hive-site'] ) if name in ["hiveserver2","metastore"]: # Manually overriding service logon user & password set by the installation package service_name = params.service_map[name] ServiceConfig(service_name, action="change_user", username = params.hive_user, password = Script.get_password(params.hive_user)) Execute(format("cmd /c hadoop fs -mkdir -p {hive_warehouse_dir}"), logoutput=True, user=params.hadoop_user) if name == 'metastore': if params.init_metastore_schema: check_schema_created_cmd = format('cmd /c "{hive_bin}\\hive.cmd --service schematool -info ' '-dbType {hive_metastore_db_type} ' '-userName {hive_metastore_user_name} ' '-passWord {hive_metastore_user_passwd!p}' '&set EXITCODE=%ERRORLEVEL%&exit /B %EXITCODE%"', #cmd "feature", propagate the process exit code manually hive_bin=params.hive_bin, hive_metastore_db_type=params.hive_metastore_db_type, hive_metastore_user_name=params.hive_metastore_user_name, hive_metastore_user_passwd=params.hive_metastore_user_passwd) try: Execute(check_schema_created_cmd) except Fail: create_schema_cmd = format('cmd /c {hive_bin}\\hive.cmd --service schematool -initSchema ' '-dbType {hive_metastore_db_type} ' '-userName {hive_metastore_user_name} ' '-passWord {hive_metastore_user_passwd!p}', hive_bin=params.hive_bin, hive_metastore_db_type=params.hive_metastore_db_type, hive_metastore_user_name=params.hive_metastore_user_name, hive_metastore_user_passwd=params.hive_metastore_user_passwd) Execute(create_schema_cmd, user = params.hive_user, logoutput=True ) if name == "hiveserver2": if params.hive_execution_engine == "tez": # Init the tez app dir in hadoop script_file = __file__.replace('/', os.sep) cmd_file = os.path.normpath(os.path.join(os.path.dirname(script_file), "..", "files", "hiveTezSetup.cmd")) Execute("cmd /c " + cmd_file, logoutput=True, user=params.hadoop_user)
def oozie(is_server=False): import params from status_params import oozie_server_win_service_name XmlConfig( "oozie-site.xml", conf_dir=params.oozie_conf_dir, configurations=params.config['configurations']['oozie-site'], owner=params.oozie_user, mode='f', configuration_attributes=params.config['configuration_attributes'] ['oozie-site']) File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"), owner=params.oozie_user, content=InlineTemplate(params.oozie_env_cmd_template)) Directory( params.oozie_tmp_dir, owner=params.oozie_user, recursive=True, ) if is_server: # Manually overriding service logon user & password set by the installation package ServiceConfig(oozie_server_win_service_name, action="change_user", username=params.oozie_user, password=Script.get_password(params.oozie_user)) download_file( os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar")) webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server", "webapps", "oozie", "WEB-INF", "lib", "sqljdbc4.jar") if os.path.isfile(webapps_sqljdbc_path): download_file( os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), webapps_sqljdbc_path) download_file( os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), os.path.join(params.oozie_home, "share", "lib", "oozie", "sqljdbc4.jar")) download_file( os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), os.path.join(params.oozie_home, "temp", "WEB-INF", "lib", "sqljdbc4.jar"))
def storm(name=None): import params yaml_config("storm.yaml", conf_dir=params.conf_dir, configurations=params.config['configurations']['storm-site'], owner=params.storm_user) if params.service_map.has_key(name): service_name = params.service_map[name] ServiceConfig(service_name, action="change_user", username=params.storm_user, password=Script.get_password(params.storm_user))
def storm(name=None): import params yaml_config("storm.yaml", conf_dir=params.conf_dir, configurations=params.config['configurations']['storm-site'], owner=params.storm_user ) if params.service_map.has_key(name): service_name = params.service_map[name] ServiceConfig(service_name, action="change_user", username = params.storm_user, password = Script.get_password(params.storm_user))
def hbase(name=None): import params XmlConfig("hbase-site.xml", conf_dir = params.hbase_conf_dir, configurations = params.config['configurations']['hbase-site'], configuration_attributes=params.config['configuration_attributes']['hbase-site'] ) if params.service_map.has_key(name): # Manually overriding service logon user & password set by the installation package service_name = params.service_map[name] ServiceConfig(service_name, action="change_user", username = params.hbase_user, password = Script.get_password(params.hbase_user))
def hdfs(component=None): import params if component == "namenode": directories = params.dfs_name_dir.split(",") Directory(directories, owner=params.hdfs_user, mode="(OI)(CI)F", create_parents=True) File( params.exclude_file_path, content=Template("exclude_hosts_list.j2"), owner=params.hdfs_user, mode="f", ) if params.hdfs_include_file: File( params.include_file_path, content=Template("include_hosts_list.j2"), owner=params.hdfs_user, mode="f", ) pass if params.service_map.has_key(component): service_name = params.service_map[component] ServiceConfig(service_name, action="change_user", username=params.hdfs_user, password=Script.get_password(params.hdfs_user)) if "hadoop-policy" in params.config['configurations']: XmlConfig( "hadoop-policy.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['hadoop-policy'], owner=params.hdfs_user, mode="f", configuration_attributes=params.config['configuration_attributes'] ['hadoop-policy']) XmlConfig( "hdfs-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['hdfs-site'], owner=params.hdfs_user, mode="f", configuration_attributes=params.config['configuration_attributes'] ['hdfs-site'])
def rebalancehdfs(self, env): from ambari_commons.os_windows import UserHelper, run_os_command_impersonated import params env.set_params(params) hdfs_username, hdfs_domain = UserHelper.parse_user_name( params.hdfs_user, ".") name_node_parameters = json.loads(params.name_node_params) threshold = name_node_parameters['threshold'] _print("Starting balancer with threshold = %s\n" % threshold) def calculateCompletePercent(first, current): return 1.0 - current.bytesLeftToMove / first.bytesLeftToMove def startRebalancingProcess(threshold): rebalanceCommand = 'hdfs balancer -threshold %s' % threshold return ['cmd', '/C', rebalanceCommand] command = startRebalancingProcess(threshold) basedir = os.path.join(env.config.basedir, 'scripts') _print("Executing command %s\n" % command) parser = hdfs_rebalance.HdfsParser() returncode, stdout, err = run_os_command_impersonated( ' '.join(command), hdfs_username, Script.get_password(params.hdfs_user), hdfs_domain) for line in stdout.split('\n'): _print('[balancer] %s %s' % (str(datetime.now()), line)) pl = parser.parseLine(line) if pl: res = pl.toJson() res['completePercent'] = calculateCompletePercent( parser.initialLine, pl) self.put_structured_out(res) elif parser.state == 'PROCESS_FINISED': _print('[balancer] %s %s' % (str(datetime.now()), 'Process is finished')) self.put_structured_out({'completePercent': 1}) break if returncode != None and returncode != 0: raise Fail( 'Hdfs rebalance process exited with error. See the log output')
def oozie(is_server=False): import params from status_params import oozie_server_win_service_name XmlConfig("oozie-site.xml", conf_dir=params.oozie_conf_dir, configurations=params.config['configurations']['oozie-site'], owner=params.oozie_user, mode='f', configuration_attributes=params.config['configuration_attributes']['oozie-site'] ) File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"), owner=params.oozie_user, content=InlineTemplate(params.oozie_env_cmd_template) ) Directory(params.oozie_tmp_dir, owner=params.oozie_user, recursive = True, ) if is_server: # Manually overriding service logon user & password set by the installation package ServiceConfig(oozie_server_win_service_name, action="change_user", username = params.oozie_user, password = Script.get_password(params.oozie_user)) download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar") ) webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server", "webapps", "oozie", "WEB-INF", "lib", "sqljdbc4.jar") if os.path.isfile(webapps_sqljdbc_path): download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), webapps_sqljdbc_path ) download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), os.path.join(params.oozie_home, "share", "lib", "oozie", "sqljdbc4.jar") ) download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), os.path.join(params.oozie_home, "temp", "WEB-INF", "lib", "sqljdbc4.jar") )
def knox(): import params XmlConfig("gateway-site.xml", conf_dir=params.knox_conf_dir, configurations=params.config['configurations']['gateway-site'], configuration_attributes=params.config['configuration_attributes']['gateway-site'], owner=params.knox_user ) # Manually overriding service logon user & password set by the installation package ServiceConfig(params.knox_gateway_win_service_name, action="change_user", username = params.knox_user, password = Script.get_password(params.knox_user)) File(os.path.join(params.knox_conf_dir, "gateway-log4j.properties"), owner=params.knox_user, content=params.gateway_log4j ) File(os.path.join(params.knox_conf_dir, "topologies", "default.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.topology_template) ) if params.security_enabled: TemplateConfig( os.path.join(params.knox_conf_dir, "krb5JAASLogin.conf"), owner = params.knox_user, template_tag = None ) if not os.path.isfile(params.knox_master_secret_path): cmd = format('cmd /C {knox_client_bin} create-master --master {knox_master_secret!p}') Execute(cmd) cmd = format('cmd /C {knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}') Execute(cmd)