def main(argv=None): from ambari_agent.ActionQueue import ActionQueue from ambari_agent.AmbariConfig import AmbariConfig from ambari_agent.Controller import Controller cfg = AmbariConfig() if os.path.exists(AmbariConfig.getConfigFile()): cfg.read(AmbariConfig.getConfigFile()) else: raise Exception("No config found, use default") ctl = Controller(cfg) actionQueue = ActionQueue(cfg, ctl) heartbeat = Heartbeat(actionQueue) print json.dumps(heartbeat.build('3',3))
def dump_command_to_json(self, command, retry=False): """ Converts command to json file and returns file path """ # Perform few modifications to stay compatible with the way in which public_fqdn = self.public_fqdn command['public_hostname'] = public_fqdn # Add cache dir to make it visible for commands command["hostLevelParams"]["agentCacheDir"] = self.config.get('agent', 'cache_dir') # Now, dump the json file command_type = command['commandType'] from ActionQueue import ActionQueue # To avoid cyclic dependency if command_type == ActionQueue.STATUS_COMMAND: # These files are frequently created, that's why we don't # store them all, but only the latest one file_path = os.path.join(self.tmp_dir, "status_command.json") else: task_id = command['taskId'] if 'clusterHostInfo' in command and command['clusterHostInfo'] and not retry: command['clusterHostInfo'] = self.decompressClusterHostInfo(command['clusterHostInfo']) file_path = os.path.join(self.tmp_dir, "command-{0}.json".format(task_id)) if command_type == ActionQueue.AUTO_EXECUTION_COMMAND: file_path = os.path.join(self.tmp_dir, "auto_command-{0}.json".format(task_id)) # Json may contain passwords, that's why we need proper permissions if os.path.isfile(file_path): os.unlink(file_path) with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT, 0600), 'w') as f: content = json.dumps(command, sort_keys = False, indent = 4) f.write(content)
def check_ranger_service(): import params ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url) ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}") response_code = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url) if response_code is not None and response_code == 200: user_resp_code = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password) if user_resp_code is not None and user_resp_code == 200: get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger) if not get_repo_flag: create_repo_flag = create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger) if create_repo_flag: return True else: return False else: return True else: Logger.error('Ambari admin user creation failed') return False else: Logger.error('Ranger service is not reachable host') return False
def create_repository_urllib2(self, data, usernamepassword, policy_user): """ :param data: repository dict :param usernamepassword: user credentials using which repository needs to be created :param policy_user: use this policy user for policies that will be used during repository creation :return Returns created repository response else None """ try: searchRepoURL = self.urlReposPub base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(searchRepoURL, data, headers) request.add_header("Authorization", "Basic {0}".format(base64string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200: Logger.info('Repository created Successfully') # Get Policies repoData = json.loads(data) repoName = repoData['name'] typeOfPolicy = repoData['repositoryType'] # Get Policies by repo name policyList = self.get_policy_by_repo_name(name=repoName, component=typeOfPolicy, status="true", usernamepassword=usernamepassword) if policyList is not None and (len(policyList)) > 0: policiesUpdateCount = 0 for policy in policyList: updatedPolicyObj = self.get_policy_params(typeOfPolicy, policy, policy_user) policyResCode = self.update_ranger_policy(updatedPolicyObj['id'], json.dumps(updatedPolicyObj), usernamepassword) if policyResCode == 200: policiesUpdateCount = policiesUpdateCount + 1 else: Logger.info('Policy Update failed') # Check for count of updated policies if len(policyList) == policiesUpdateCount: Logger.info( "Ranger Repository created successfully and policies updated successfully providing ambari-qa user all permissions") return response else: return None else: Logger.info("Policies not found for the newly created Repository") return None else: Logger.info('Repository creation failed') return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error creating repository. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Error creating repository. Reason - {0}.".format(e.reason))
def set_current(options): server_status, pid = is_server_runing() if not server_status: err = 'Ambari Server is not running.' raise FatalException(1, err) finalize_options = SetCurrentVersionOptions(options) if finalize_options.no_finalize_options_set(): err = 'Must specify --cluster-name and --version-display-name. Please invoke ambari-server.py --help to print the options.' raise FatalException(1, err) admin_login = get_validated_string_input(prompt="Enter Ambari Admin login: "******"Enter Ambari Admin password: "******"Failed to read properties file.") base_url = get_ambari_server_api_base(properties) url = base_url + "clusters/{0}/stack_versions".format(finalize_options.cluster_name) admin_auth = base64.encodestring('%s:%s' % (admin_login, admin_password)).replace('\n', '') request = urllib2.Request(url) request.add_header('Authorization', 'Basic %s' % admin_auth) request.add_header('X-Requested-By', 'ambari') data = { "ClusterStackVersions": { "repository_version": finalize_options.desired_repo_version, "state": "CURRENT", "force": "true" if finalize_options.force_repo_version else "false" } } if get_verbose(): sys.stdout.write('\nCalling API ' + url + ' : ' + str(data) + '\n') request.add_data(json.dumps(data)) request.get_method = lambda: 'PUT' try: response = urllib2.urlopen(request) except urllib2.HTTPError, e: code = e.getcode() content = e.read() err = 'Error during setting current version. Http status code - {0}. \n {1}'.format( code, content) raise FatalException(1, err)
def flume(action = None): import params from service_mapping import flume_win_service_name if action == 'config': ServiceConfig(flume_win_service_name, action="configure", start_type="manual") ServiceConfig(flume_win_service_name, action="change_user", username=params.flume_user, password = Script.get_password(params.flume_user)) # remove previously defined meta's for n in find_expected_agent_names(params.flume_conf_dir): os.unlink(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json')) flume_agents = {} if params.flume_conf_content is not None: flume_agents = build_flume_topology(params.flume_conf_content) for agent in flume_agents.keys(): flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent) flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf') flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json') flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties') flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.ps1') Directory(flume_agent_conf_dir ) PropertiesFile(flume_agent_conf_file, properties=flume_agents[agent]) File(flume_agent_log4j_file, content=Template('log4j.properties.j2', agent_name = agent)) File(flume_agent_meta_file, content = json.dumps(ambari_meta(agent, flume_agents[agent]))) File(flume_agent_env_file, owner=params.flume_user, content=InlineTemplate(params.flume_env_sh_template) ) if params.has_metric_collector: File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"), owner=params.flume_user, content=Template("flume-metrics2.properties.j2") )
def createDB(self, env): import params env.set_params(params) if params.db_name and params.db_pass: user_json = {'user': params.db_user, 'pwd': params.db_pass, 'roles': ['readWrite']} create_user_cmd = ( "mongo {db} --eval 'db.getUser(\"{user}\")' | grep -q -w {db}\\.{user} || " "mongo {db} --eval 'db.createUser({json});'" ) Execute(create_user_cmd.format( db=params.db_name, user=params.db_user, json=json.dumps(user_json)))
def create_ranger_repository(self, component, repo_name, repo_properties, ambari_ranger_admin, ambari_ranger_password, admin_uname, admin_password, policy_user): """ :param component: name of the component, from which it will get or create repository :param repo_name: name of the repository to be get or create :param repo_properties: dict of repository to be create if not exist :param ambari_ranger_admin: ambari admin user creation username :param ambari_ranger_password: ambari admin user creation password :param admin_uname: ranger admin username :param admin_password: ranger admin password :param policy_user: use this policy user for policies that will be used during repository creation """ response_code = self.check_ranger_login_urllib2(self.baseUrl) repo_data = json.dumps(repo_properties) ambari_ranger_password = unicode(ambari_ranger_password) admin_password = unicode(admin_password) ambari_username_password_for_ranger = format('{ambari_ranger_admin}:{ambari_ranger_password}') if response_code is not None and response_code == 200: user_resp_code = self.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}")) if user_resp_code is not None and user_resp_code == 200: retryCount = 0 while retryCount <= 5: repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger) if repo is not None: Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name'])) break else: response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger, policy_user) if response is not None: Logger.info('{0} Repository created in Ranger admin'.format(component.title())) break else: if retryCount < 5: Logger.info("Retry Repository Creation is being called") time.sleep(15) # delay for 15 seconds retryCount += 1 else: Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) break else: Logger.error('Ambari admin user creation failed') elif not self.skip_if_rangeradmin_down: Logger.error("Connection to Ranger Admin failed !")
def run_metainfo_upgrade(keyValueMap=None): jdk_path = get_java_exe_path() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) retcode = 1 if keyValueMap: command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(), 'updateMetaInfo', "'" + json.dumps(keyValueMap) + "'") (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode)) if retcode > 0: print_error_msg("Error executing metainfo upgrade, please check the " "server logs.") return retcode
def action_execute(self, main_resource): env = Environment.get_instance() # Check required parameters main_resource.assert_parameter_is_set('user') if not 'hdfs_files' in env.config or not env.config['hdfs_files']: Logger.info("No resources to create. 'create_on_execute' or 'delete_on_execute' wasn't triggered before this 'execute' action.") return hadoop_bin_dir = main_resource.resource.hadoop_bin_dir hadoop_conf_dir = main_resource.resource.hadoop_conf_dir user = main_resource.resource.user security_enabled = main_resource.resource.security_enabled keytab_file = main_resource.resource.keytab kinit_path = main_resource.resource.kinit_path_local logoutput = main_resource.resource.logoutput principal_name = main_resource.resource.principal_name jar_path=JAR_PATH timestamp = time.time() json_path=format(JSON_PATH) if security_enabled: main_resource.kinit() # Write json file to disk File(json_path, owner = user, content = json.dumps(env.config['hdfs_files']) ) # Execute jar to create/delete resources in hadoop Execute(format("hadoop --config {hadoop_conf_dir} jar {jar_path} {json_path}"), user=user, path=[hadoop_bin_dir], logoutput=logoutput, ) # Clean env.config['hdfs_files'] = []
def run_command(self, target, operation, method='POST', assertable_result=True, file_to_put=None, ignore_status_codes=[], **kwargs): """ assertable_result - some POST requests return '{"boolean":false}' or '{"boolean":true}' depending on if query was successful or not, we can assert this for them """ target = HdfsResourceProvider.parse_path(target) url = format("{address}/webhdfs/v1{target}?op={operation}&user.name={run_user}", address=self.address, run_user=self.run_user) for k,v in kwargs.iteritems(): url = format("{url}&{k}={v}") if file_to_put and not os.path.exists(file_to_put): raise Fail(format("File {file_to_put} is not found.")) cmd = ["curl", "-sS","-L", "-w", "%{http_code}", "-X", method] if file_to_put: cmd += ["-T", file_to_put] if self.security_enabled: cmd += ["--negotiate", "-u", ":"] if self.is_https_enabled: cmd += ["-k"] cmd.append(url) _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False) status_code = out[-3:] out = out[:-3] # remove last line from output which is status code try: result_dict = json.loads(out) except ValueError: result_dict = out if status_code not in WebHDFSUtil.valid_status_codes+ignore_status_codes or assertable_result and result_dict and not result_dict['boolean']: formatted_output = json.dumps(result_dict, indent=2) if isinstance(result_dict, dict) else result_dict formatted_output = err + "\n" + formatted_output err_msg = "Execution of '%s' returned status_code=%s. %s" % (shell.string_cmd_from_args_list(cmd), status_code, formatted_output) raise Fail(err_msg) return result_dict
def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os): jdk_path = get_java_exe_path() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 stackId = {} stackId[stackName] = stackVersion if repo_url is not None: stackId['repo_url'] = repo_url if repo_url_os is not None: stackId['repo_url_os'] = repo_url_os command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(), "updateStackId", "'" + json.dumps(stackId) + "'") (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode)) if retcode > 0: print_error_msg("Error executing stack upgrade, please check the server logs.") return retcode
def reqSignCrt(self): sign_crt_req_url = self.server_url + '/certs/' + hostname.hostname( self.config) agent_crt_req_f = open(self.getAgentCrtReqName()) agent_crt_req_content = agent_crt_req_f.read() passphrase_env_var = self.config.get('security', 'passphrase_env_var_name') passphrase = os.environ[passphrase_env_var] register_data = {'csr': agent_crt_req_content, 'passphrase': passphrase} data = json.dumps(register_data) proxy_handler = urllib2.ProxyHandler({}) opener = urllib2.build_opener(proxy_handler) urllib2.install_opener(opener) req = urllib2.Request(sign_crt_req_url, data, {'Content-Type': 'application/json'}) f = urllib2.urlopen(req) response = f.read() f.close() try: data = json.loads(response) logger.debug("Sign response from Server: \n" + pprint.pformat(data)) except Exception: logger.warn("Malformed response! data: " + str(data)) data = {'result': 'ERROR'} result = data['result'] if result == 'OK': agentCrtContent = data['signedCa'] agentCrtF = open(self.getAgentCrtName(), "w") agentCrtF.write(agentCrtContent) else: # Possible exception is catched higher at Controller logger.error('Certificate signing failed.' '\nIn order to receive a new agent' ' certificate, remove existing certificate file from keys ' 'directory. As a workaround you can turn off two-way SSL ' 'authentication in server configuration(ambari.properties) ' '\nExiting..') raise ssl.SSLError
def on_background_command_complete_callback(self, process_condensed_result, handle): logger.debug('Start callback: %s' % process_condensed_result) logger.debug('The handle is: %s' % handle) status = self.COMPLETED_STATUS if handle.exitCode == 0 else self.FAILED_STATUS aborted_postfix = self.customServiceOrchestrator.command_canceled_reason(handle.command['taskId']) if aborted_postfix: status = self.FAILED_STATUS logger.debug('Set status to: %s , reason = %s' % (status, aborted_postfix)) else: aborted_postfix = '' roleResult = self.commandStatuses.generate_report_template(handle.command) roleResult.update({ 'stdout': process_condensed_result['stdout'] + aborted_postfix, 'stderr': process_condensed_result['stderr'] + aborted_postfix, 'exitCode': process_condensed_result['exitcode'], 'structuredOut': str(json.dumps(process_condensed_result['structuredOut'])) if 'structuredOut' in process_condensed_result else '', 'status': status, }) self.commandStatuses.put_command_status(handle.command, roleResult)
def create_ranger_repository(self, component, repo_name, repo_properties, ambari_ranger_admin, ambari_ranger_password, admin_uname, admin_password, policy_user): response_code = self.check_ranger_login_urllib2(self.base_url) repo_data = json.dumps(repo_properties) ambari_ranger_password = unicode(ambari_ranger_password) admin_password = unicode(admin_password) ambari_username_password_for_ranger = format('{ambari_ranger_admin}:{ambari_ranger_password}') if response_code is not None and response_code == 200: user_resp_code = self.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}")) if user_resp_code is not None and user_resp_code == 200: retryCount = 0 while retryCount <= 5: repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger) if repo is not None: Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name'])) break else: response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger) if response is not None: Logger.info('{0} Repository created in Ranger admin'.format(component.title())) break else: if retryCount < 5: Logger.info("Retry Repository Creation is being called") time.sleep(30) # delay for 30 seconds retryCount += 1 else: Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) break else: Logger.error('Ambari admin user creation failed') elif not self.skip_if_rangeradmin_down: Logger.error("Connection failed to Ranger Admin !")
def execute_command(self, command): ''' Executes commands of type EXECUTION_COMMAND ''' clusterName = command['clusterName'] commandId = command['commandId'] isCommandBackground = command['commandType'] == self.BACKGROUND_EXECUTION_COMMAND isAutoExecuteCommand = command['commandType'] == self.AUTO_EXECUTION_COMMAND message = "Executing command with id = {commandId} for role = {role} of " \ "cluster {cluster}.".format( commandId = str(commandId), role=command['role'], cluster=clusterName) logger.info(message) taskId = command['taskId'] # Preparing 'IN_PROGRESS' report in_progress_status = self.commandStatuses.generate_report_template(command) # The path of the files that contain the output log and error log use a prefix that the agent advertises to the # server. The prefix is defined in agent-config.ini if not isAutoExecuteCommand: in_progress_status.update({ 'tmpout': self.tmpdir + os.sep + 'output-' + str(taskId) + '.txt', 'tmperr': self.tmpdir + os.sep + 'errors-' + str(taskId) + '.txt', 'structuredOut' : self.tmpdir + os.sep + 'structured-out-' + str(taskId) + '.json', 'status': self.IN_PROGRESS_STATUS }) else: in_progress_status.update({ 'tmpout': self.tmpdir + os.sep + 'auto_output-' + str(taskId) + '.txt', 'tmperr': self.tmpdir + os.sep + 'auto_errors-' + str(taskId) + '.txt', 'structuredOut' : self.tmpdir + os.sep + 'auto_structured-out-' + str(taskId) + '.json', 'status': self.IN_PROGRESS_STATUS }) self.commandStatuses.put_command_status(command, in_progress_status) numAttempts = 0 retryDuration = 0 # even with 0 allow one attempt retryAble = False delay = 1 if 'commandParams' in command: if 'max_duration_for_retries' in command['commandParams']: retryDuration = int(command['commandParams']['max_duration_for_retries']) if 'command_retry_enabled' in command['commandParams']: retryAble = command['commandParams']['command_retry_enabled'] == "true" if isAutoExecuteCommand: retryAble = False logger.debug("Command execution metadata - retry enabled = {retryAble}, max retry duration (sec) = {retryDuration}". format(retryAble=retryAble, retryDuration=retryDuration)) while retryDuration >= 0: numAttempts += 1 start = 0 if retryAble: start = int(time.time()) # running command commandresult = self.customServiceOrchestrator.runCommand(command, in_progress_status['tmpout'], in_progress_status['tmperr'], override_output_files=numAttempts == 1, retry=numAttempts > 1) end = 1 if retryAble: end = int(time.time()) retryDuration -= (end - start) # dumping results if isCommandBackground: return else: if commandresult['exitcode'] == 0: status = self.COMPLETED_STATUS else: status = self.FAILED_STATUS if status != self.COMPLETED_STATUS and retryAble == True and retryDuration > 0: delay = self.get_retry_delay(delay) if delay > retryDuration: delay = retryDuration retryDuration -= delay # allow one last attempt logger.info("Retrying command id {cid} after a wait of {delay}".format(cid=taskId, delay=delay)) time.sleep(delay) continue else: break roleResult = self.commandStatuses.generate_report_template(command) roleResult.update({ 'stdout': commandresult['stdout'], 'stderr': commandresult['stderr'], 'exitCode': commandresult['exitcode'], 'status': status, }) if roleResult['stdout'] == '': roleResult['stdout'] = 'None' if roleResult['stderr'] == '': roleResult['stderr'] = 'None' # let ambari know name of custom command if command['hostLevelParams'].has_key('custom_command'): roleResult['customCommand'] = command['hostLevelParams']['custom_command'] if 'structuredOut' in commandresult: roleResult['structuredOut'] = str(json.dumps(commandresult['structuredOut'])) else: roleResult['structuredOut'] = '' # let recovery manager know the current state if status == self.COMPLETED_STATUS: if self.controller.recovery_manager.enabled() and command.has_key('roleCommand') \ and self.controller.recovery_manager.configured_for_recovery(command['role']): if command['roleCommand'] == self.ROLE_COMMAND_START: self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.LIVE_STATUS) self.controller.recovery_manager.update_config_staleness(command['role'], False) logger.info("After EXECUTION_COMMAND (START), current state of " + command['role'] + " to " + self.controller.recovery_manager.get_current_status(command['role']) ) elif command['roleCommand'] == self.ROLE_COMMAND_STOP or command['roleCommand'] == self.ROLE_COMMAND_INSTALL: self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.DEAD_STATUS) logger.info("After EXECUTION_COMMAND (STOP/INSTALL), current state of " + command['role'] + " to " + self.controller.recovery_manager.get_current_status(command['role']) ) elif command['roleCommand'] == self.ROLE_COMMAND_CUSTOM_COMMAND: if command['hostLevelParams'].has_key('custom_command') and \ command['hostLevelParams']['custom_command'] == self.CUSTOM_COMMAND_RESTART: self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.LIVE_STATUS) self.controller.recovery_manager.update_config_staleness(command['role'], False) logger.info("After EXECUTION_COMMAND (RESTART), current state of " + command['role'] + " to " + self.controller.recovery_manager.get_current_status(command['role']) ) pass # let ambari know that configuration tags were applied configHandler = ActualConfigHandler(self.config, self.configTags) #update if command.has_key('forceRefreshConfigTags') and len(command['forceRefreshConfigTags']) > 0 : forceRefreshConfigTags = command['forceRefreshConfigTags'] logger.info("Got refresh additional component tags command") for configTag in forceRefreshConfigTags : configHandler.update_component_tag(command['role'], configTag, command['configurationTags'][configTag]) roleResult['customCommand'] = self.CUSTOM_COMMAND_RESTART # force restart for component to evict stale_config on server side command['configurationTags'] = configHandler.read_actual_component(command['role']) if command.has_key('configurationTags'): configHandler.write_actual(command['configurationTags']) roleResult['configurationTags'] = command['configurationTags'] component = {'serviceName':command['serviceName'],'componentName':command['role']} if command.has_key('roleCommand') and \ (command['roleCommand'] == self.ROLE_COMMAND_START or \ (command['roleCommand'] == self.ROLE_COMMAND_INSTALL \ and component in LiveStatus.CLIENT_COMPONENTS) or \ (command['roleCommand'] == self.ROLE_COMMAND_CUSTOM_COMMAND and \ command['hostLevelParams'].has_key('custom_command') and \ command['hostLevelParams']['custom_command'] == self.CUSTOM_COMMAND_RESTART)): configHandler.write_actual_component(command['role'], command['configurationTags']) if command['hostLevelParams'].has_key('clientsToUpdateConfigs') and \ command['hostLevelParams']['clientsToUpdateConfigs']: configHandler.write_client_components(command['serviceName'], command['configurationTags'], command['hostLevelParams']['clientsToUpdateConfigs']) roleResult['configurationTags'] = configHandler.read_actual_component(command['role']) self.commandStatuses.put_command_status(command, roleResult)
def sync_ldap(options): if not is_root(): err = 'Ambari-server sync-ldap should be run with ' \ 'root-level privileges' raise FatalException(4, err) server_status, pid = is_server_runing() if not server_status: err = 'Ambari Server is not running.' raise FatalException(1, err) properties = get_ambari_properties() if properties == -1: raise FatalException(1, "Failed to read properties file.") ldap_configured = properties.get_property(IS_LDAP_CONFIGURED) if ldap_configured != 'true': err = "LDAP is not configured. Run 'ambari-server setup-ldap' first." raise FatalException(1, err) # set ldap sync options ldap_sync_options = LdapSyncOptions(options) if ldap_sync_options.no_ldap_sync_options_set(): err = 'Must specify a sync option (all, existing, users or groups). Please invoke ambari-server.py --help to print the options.' raise FatalException(1, err) admin_login = get_validated_string_input(prompt="Enter Ambari Admin login: "******"Enter Ambari Admin password: "******"Event":{"specs":[{"principal_type":"users","sync_type":"all"},{"principal_type":"groups","sync_type":"all"}]}}] elif ldap_sync_options.ldap_sync_existing: sys.stdout.write('Syncing existing.') bodies = [{"Event":{"specs":[{"principal_type":"users","sync_type":"existing"},{"principal_type":"groups","sync_type":"existing"}]}}] else: sys.stdout.write('Syncing specified users and groups.') bodies = [{"Event":{"specs":[]}}] body = bodies[0] events = body['Event'] specs = events['specs'] if ldap_sync_options.ldap_sync_users is not None: new_specs = [{"principal_type":"users","sync_type":"specific","names":""}] get_ldap_event_spec_names(ldap_sync_options.ldap_sync_users, specs, new_specs) if ldap_sync_options.ldap_sync_groups is not None: new_specs = [{"principal_type":"groups","sync_type":"specific","names":""}] get_ldap_event_spec_names(ldap_sync_options.ldap_sync_groups, specs, new_specs) if get_verbose(): sys.stdout.write('\nCalling API ' + url + ' : ' + str(bodies) + '\n') request.add_data(json.dumps(bodies)) request.get_method = lambda: 'POST' try: response = urllib2.urlopen(request) except Exception as e: err = 'Sync event creation failed. Error details: %s' % e raise FatalException(1, err) response_status_code = response.getcode() if response_status_code != 201: err = 'Error during syncing. Http status code - ' + str(response_status_code) raise FatalException(1, err) response_body = json.loads(response.read()) url = response_body['resources'][0]['href'] request = urllib2.Request(url) request.add_header('Authorization', 'Basic %s' % admin_auth) request.add_header('X-Requested-By', 'ambari') body = [{"LDAP":{"synced_groups":"*","synced_users":"*"}}] request.add_data(json.dumps(body)) request.get_method = lambda: 'GET' request_in_progress = True while request_in_progress: sys.stdout.write('.') sys.stdout.flush() try: response = urllib2.urlopen(request) except Exception as e: request_in_progress = False err = 'Sync event check failed. Error details: %s' % e raise FatalException(1, err) response_status_code = response.getcode() if response_status_code != 200: err = 'Error during syncing. Http status code - ' + str(response_status_code) raise FatalException(1, err) response_body = json.loads(response.read()) sync_info = response_body['Event'] if sync_info['status'] == 'ERROR': raise FatalException(1, str(sync_info['status_detail'])) elif sync_info['status'] == 'COMPLETE': print '\n\nCompleted LDAP Sync.' print 'Summary:' for principal_type, summary in sync_info['summary'].iteritems(): print ' {0}:'.format(principal_type) for action, amount in summary.iteritems(): print ' {0} = {1!s}'.format(action, amount) request_in_progress = False else: time.sleep(1) sys.stdout.write('\n') sys.stdout.flush()
def flume(action = None): import params if action == 'config': # remove previously defined meta's for n in find_expected_agent_names(params.flume_conf_dir): File(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'), action = "delete", ) Directory(params.flume_run_dir, ) Directory(params.flume_conf_dir, create_parents = True, owner=params.flume_user, ) Directory(params.flume_log_dir, owner=params.flume_user, cd_access="a", mode=0755, ) flume_agents = {} if params.flume_conf_content is not None: flume_agents = build_flume_topology(params.flume_conf_content) for agent in flume_agents.keys(): flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent) flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf') flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json') flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties') flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.sh') Directory(flume_agent_conf_dir, owner=params.flume_user, ) PropertiesFile(flume_agent_conf_file, properties=flume_agents[agent], owner=params.flume_user, mode = 0644) File(flume_agent_log4j_file, content=Template('log4j.properties.j2', agent_name = agent), owner=params.flume_user, mode = 0644) File(flume_agent_meta_file, content = json.dumps(ambari_meta(agent, flume_agents[agent])), owner=params.flume_user, mode = 0644) File(flume_agent_env_file, owner=params.flume_user, content=InlineTemplate(params.flume_env_sh_template) ) if params.has_metric_collector: File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"), owner=params.flume_user, content=Template("flume-metrics2.properties.j2") ) elif action == 'start': # desired state for service should be STARTED if len(params.flume_command_targets) == 0: _set_desired_state('STARTED') # It is important to run this command as a background process. flume_base = as_user(format("{flume_bin} agent --name {{0}} --conf {{1}} --conf-file {{2}} {{3}} > {flume_log_dir}/{{4}}.out 2>&1"), params.flume_user, env={'JAVA_HOME': params.java_home}) + " &" for agent in cmd_target_names(): flume_agent_conf_dir = params.flume_conf_dir + os.sep + agent flume_agent_conf_file = flume_agent_conf_dir + os.sep + "flume.conf" flume_agent_pid_file = params.flume_run_dir + os.sep + agent + ".pid" if not os.path.isfile(flume_agent_conf_file): continue if not is_flume_process_live(flume_agent_pid_file): # TODO someday make the ganglia ports configurable extra_args = '' if params.ganglia_server_host is not None: extra_args = '-Dflume.monitoring.type=ganglia -Dflume.monitoring.hosts={0}:{1}' extra_args = extra_args.format(params.ganglia_server_host, '8655') if params.has_metric_collector: extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \ '-Dflume.monitoring.node={0}:{1}' extra_args = extra_args.format(params.metric_collector_host, params.metric_collector_port) flume_cmd = flume_base.format(agent, flume_agent_conf_dir, flume_agent_conf_file, extra_args, agent) Execute(flume_cmd, wait_for_finish=False, environment={'JAVA_HOME': params.java_home} ) # sometimes startup spawns a couple of threads - so only the first line may count pid_cmd = as_sudo(('pgrep', '-o', '-u', params.flume_user, '-f', format('^{java_home}.*{agent}.*'))) + \ " | " + as_sudo(('tee', flume_agent_pid_file)) + " && test ${PIPESTATUS[0]} -eq 0" Execute(pid_cmd, logoutput=True, tries=20, try_sleep=10) pass elif action == 'stop': # desired state for service should be INSTALLED if len(params.flume_command_targets) == 0: _set_desired_state('INSTALLED') pid_files = glob.glob(params.flume_run_dir + os.sep + "*.pid") if 0 == len(pid_files): return agent_names = cmd_target_names() for agent in agent_names: pid_file = format("{flume_run_dir}/{agent}.pid") if is_flume_process_live(pid_file): pid = shell.checked_call(("cat", pid_file), sudo=True)[1].strip() Execute(("kill", "-15", pid), sudo=True) # kill command has to be a tuple if not await_flume_process_termination(pid_file): raise Fail("Can't stop flume agent: {0}".format(agent)) File(pid_file, action = 'delete')
ranger_hive_url = format("{hive_url}/default;principal={hive_principal}") if security_enabled else hive_url if stack_supports_ranger_hive_jdbc_url_change: ranger_hive_url = format("jdbc:hive2://{hive_zookeeper_quorum}/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace={hive_server2_zookeeper_namespace}") hive_ranger_plugin_config = { 'username': repo_config_username, 'password': repo_config_password, 'jdbc.driverClassName': jdbc_driver_class_name, 'jdbc.url': ranger_hive_url, 'commonNameForCertificate': common_name_for_certificate } hive_ranger_plugin_repo = { 'isActive': 'true', 'config': json.dumps(hive_ranger_plugin_config), 'description': 'hive repo', 'name': repo_name, 'repositoryType': 'hive', 'assetType': '3' } custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) if len(custom_ranger_service_config) > 0: hive_ranger_plugin_config.update(custom_ranger_service_config) if stack_supports_ranger_kerberos and security_enabled: hive_ranger_plugin_config['policy.download.auth.users'] = hive_user hive_ranger_plugin_config['tag.download.auth.users'] = hive_user hive_ranger_plugin_config['policy.grantrevoke.auth.users'] = hive_user
def create_ambari_admin_user(self, ambari_admin_username, ambari_admin_password, usernamepassword): """ :param ambari_admin_username: username of user to be created :param ambari_admin_password: user password of user to be created :param usernamepassword: user credentials using which repository needs to be searched. :return: Returns user credentials if user exist otherwise rerutns credentials of created user. """ flag_ambari_admin_present = False match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password) if match is None: raise Fail('Invalid password given for Ranger Admin user for Ambari') try: url = self.url_users + '?name=' + str(ambari_admin_username) request = urllib2.Request(url) base_64_string = base64.encodestring(usernamepassword).replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response['vXUsers']) >= 0: for vxuser in response['vXUsers']: if vxuser['name'] == ambari_admin_username: flag_ambari_admin_present = True break else: flag_ambari_admin_present = False if flag_ambari_admin_present: Logger.info(ambari_admin_username + ' user already exists.') return response_code else: Logger.info(ambari_admin_username + ' user is not present, creating user using given configurations') url = self.url_sec_users admin_user = dict() admin_user['status'] = 1 admin_user['userRoleList'] = ['ROLE_SYS_ADMIN'] admin_user['name'] = ambari_admin_username admin_user['password'] = ambari_admin_password admin_user['description'] = ambari_admin_username admin_user['firstName'] = ambari_admin_username data = json.dumps(admin_user) base_64_string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(url, data, headers) request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200 and response is not None: Logger.info('Ambari admin user creation successful.') return response_code else: Logger.error('Ambari admin user creation failed.') return None else: return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error creating ambari admin user. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Error creating ambari admin user. Reason - {0}.".format(e.reason))
hbase_ranger_plugin_config = { 'username': repo_config_username, 'password': repo_config_password, 'hadoop.security.authentication': hadoop_security_authentication, 'hbase.security.authentication': hbase_security_authentication, 'hbase.zookeeper.property.clientPort': hbase_zookeeper_property_clientPort, 'hbase.zookeeper.quorum': hbase_zookeeper_quorum, 'zookeeper.znode.parent': zookeeper_znode_parent, 'commonNameForCertificate': common_name_for_certificate, 'hbase.master.kerberos.principal': master_jaas_princ if security_enabled else '' } hbase_ranger_plugin_repo = { 'isActive': 'true', 'config': json.dumps(hbase_ranger_plugin_config), 'description': 'hbase repo', 'name': repo_name, 'repositoryType': 'hbase', 'assetType': '2' } ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None xa_audit_hdfs_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla':
downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") driver_curl_target = format("/usr/hdp/current/knox-server/ext/{jdbc_jar_name}") knox_ranger_plugin_config = { 'username': repo_config_username, 'password': repo_config_password, 'knox.url': format("https://{knox_host_name}:{knox_host_port}/gateway/admin/api/v1/topologies"), 'commonNameForCertificate': common_name_for_certificate } knox_ranger_plugin_repo = { 'isActive': 'true', 'config': json.dumps(knox_ranger_plugin_config), 'description': 'knox repo', 'name': repo_name, 'repositoryType': 'knox', 'assetType': '5', } ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None xa_audit_hdfs_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla':
def create_ambari_admin_user(self, ambari_admin_username, ambari_admin_password, usernamepassword): """ :param ambari_admin_username: username of user to be created :param ambari_admin_password: user password of user to be created :param usernamepassword: user credentials using which repository needs to be searched. :return: Returns user credentials if user exist otherwise rerutns credentials of created user. """ flag_ambari_admin_present = False match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password) if match is None: raise Fail( 'Invalid password given for Ranger Admin user for Ambari') try: url = self.url_users + '?name=' + str(ambari_admin_username) request = urllib2.Request(url) base_64_string = base64.encodestring(usernamepassword).replace( '\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response['vXUsers']) >= 0: for vxuser in response['vXUsers']: if vxuser['name'] == ambari_admin_username: flag_ambari_admin_present = True break else: flag_ambari_admin_present = False if flag_ambari_admin_present: Logger.info(ambari_admin_username + ' user already exists.') return response_code else: Logger.info( ambari_admin_username + ' user is not present, creating user using given configurations' ) url = self.url_sec_users admin_user = dict() admin_user['status'] = 1 admin_user['userRoleList'] = ['ROLE_SYS_ADMIN'] admin_user['name'] = ambari_admin_username admin_user['password'] = ambari_admin_password admin_user['description'] = ambari_admin_username admin_user['firstName'] = ambari_admin_username data = json.dumps(admin_user) base_64_string = base64.encodestring( '{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(url, data, headers) request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode( result.read())) if response_code == 200 and response is not None: Logger.info('Ambari admin user creation successful.') return response_code else: Logger.error('Ambari admin user creation failed.') return None else: return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail( "Error creating ambari admin user. Http status code - {0}. \n {1}" .format(e.code, e.read())) else: raise Fail( "Error creating ambari admin user. Reason - {0}.".format( e.reason))
def execute_command(self, command): ''' Executes commands of type EXECUTION_COMMAND ''' clusterName = command['clusterName'] commandId = command['commandId'] isCommandBackground = command['commandType'] == self.BACKGROUND_EXECUTION_COMMAND isAutoExecuteCommand = command['commandType'] == self.AUTO_EXECUTION_COMMAND message = "Executing command with id = {commandId}, taskId = {taskId} for role = {role} of " \ "cluster {cluster}.".format( commandId = str(commandId), taskId = str(command['taskId']), role=command['role'], cluster=clusterName) logger.info(message) taskId = command['taskId'] # Preparing 'IN_PROGRESS' report in_progress_status = self.commandStatuses.generate_report_template(command) # The path of the files that contain the output log and error log use a prefix that the agent advertises to the # server. The prefix is defined in agent-config.ini if not isAutoExecuteCommand: in_progress_status.update({ 'tmpout': self.tmpdir + os.sep + 'output-' + str(taskId) + '.txt', 'tmperr': self.tmpdir + os.sep + 'errors-' + str(taskId) + '.txt', 'structuredOut' : self.tmpdir + os.sep + 'structured-out-' + str(taskId) + '.json', 'status': self.IN_PROGRESS_STATUS }) else: in_progress_status.update({ 'tmpout': self.tmpdir + os.sep + 'auto_output-' + str(taskId) + '.txt', 'tmperr': self.tmpdir + os.sep + 'auto_errors-' + str(taskId) + '.txt', 'structuredOut' : self.tmpdir + os.sep + 'auto_structured-out-' + str(taskId) + '.json', 'status': self.IN_PROGRESS_STATUS }) self.commandStatuses.put_command_status(command, in_progress_status) numAttempts = 0 retryDuration = 0 # even with 0 allow one attempt retryAble = False delay = 1 log_command_output = True if 'commandParams' in command and 'log_output' in command['commandParams'] and "false" == command['commandParams']['log_output']: log_command_output = False if 'commandParams' in command: if 'max_duration_for_retries' in command['commandParams']: retryDuration = int(command['commandParams']['max_duration_for_retries']) if 'command_retry_enabled' in command['commandParams']: retryAble = command['commandParams']['command_retry_enabled'] == "true" if isAutoExecuteCommand: retryAble = False logger.info("Command execution metadata - taskId = {taskId}, retry enabled = {retryAble}, max retry duration (sec) = {retryDuration}, log_output = {log_command_output}". format(taskId=taskId, retryAble=retryAble, retryDuration=retryDuration, log_command_output=log_command_output)) while retryDuration >= 0: numAttempts += 1 start = 0 if retryAble: start = int(time.time()) # running command commandresult = self.customServiceOrchestrator.runCommand(command, in_progress_status['tmpout'], in_progress_status['tmperr'], override_output_files=numAttempts == 1, retry=numAttempts > 1) end = 1 if retryAble: end = int(time.time()) retryDuration -= (end - start) # dumping results if isCommandBackground: logger.info("Command is background command, quit retrying. Exit code: {exitCode}, retryAble: {retryAble}, retryDuration (sec): {retryDuration}, last delay (sec): {delay}" .format(cid=taskId, exitCode=commandresult['exitcode'], retryAble=retryAble, retryDuration=retryDuration, delay=delay)) return else: if commandresult['exitcode'] == 0: status = self.COMPLETED_STATUS else: status = self.FAILED_STATUS if (commandresult['exitcode'] == -signal.SIGTERM) or (commandresult['exitcode'] == -signal.SIGKILL): logger.info('Command {cid} was canceled!'.format(cid=taskId)) break if status != self.COMPLETED_STATUS and retryAble and retryDuration > 0: delay = self.get_retry_delay(delay) if delay > retryDuration: delay = retryDuration retryDuration -= delay # allow one last attempt commandresult['stderr'] += "\n\nCommand failed. Retrying command execution ...\n\n" logger.info("Retrying command id {cid} after a wait of {delay}".format(cid=taskId, delay=delay)) time.sleep(delay) continue else: logger.info("Quit retrying for command id {cid}. Status: {status}, retryAble: {retryAble}, retryDuration (sec): {retryDuration}, last delay (sec): {delay}" .format(cid=taskId, status=status, retryAble=retryAble, retryDuration=retryDuration, delay=delay)) break # final result to stdout commandresult['stdout'] += '\n\nCommand completed successfully!\n' if status == self.COMPLETED_STATUS else '\n\nCommand failed after ' + str(numAttempts) + ' tries\n' logger.info('Command {cid} completed successfully!'.format(cid=taskId) if status == self.COMPLETED_STATUS else 'Command {cid} failed after {attempts} tries'.format(cid=taskId, attempts=numAttempts)) roleResult = self.commandStatuses.generate_report_template(command) roleResult.update({ 'stdout': commandresult['stdout'], 'stderr': commandresult['stderr'], 'exitCode': commandresult['exitcode'], 'status': status, }) if self.config.has_option("logging","log_command_executes") \ and int(self.config.get("logging", "log_command_executes")) == 1 \ and log_command_output: if roleResult['stdout'] != '': logger.info("Begin command output log for command with id = " + str(command['taskId']) + ", role = " + command['role'] + ", roleCommand = " + command['roleCommand']) self.log_command_output(roleResult['stdout'], str(command['taskId'])) logger.info("End command output log for command with id = " + str(command['taskId']) + ", role = " + command['role'] + ", roleCommand = " + command['roleCommand']) if roleResult['stderr'] != '': logger.info("Begin command stderr log for command with id = " + str(command['taskId']) + ", role = " + command['role'] + ", roleCommand = " + command['roleCommand']) self.log_command_output(roleResult['stderr'], str(command['taskId'])) logger.info("End command stderr log for command with id = " + str(command['taskId']) + ", role = " + command['role'] + ", roleCommand = " + command['roleCommand']) if roleResult['stdout'] == '': roleResult['stdout'] = 'None' if roleResult['stderr'] == '': roleResult['stderr'] = 'None' # let ambari know name of custom command if command['hostLevelParams'].has_key('custom_command'): roleResult['customCommand'] = command['hostLevelParams']['custom_command'] if 'structuredOut' in commandresult: roleResult['structuredOut'] = str(json.dumps(commandresult['structuredOut'])) else: roleResult['structuredOut'] = '' # let recovery manager know the current state if status == self.COMPLETED_STATUS: if self.controller.recovery_manager.enabled() and command.has_key('roleCommand') \ and self.controller.recovery_manager.configured_for_recovery(command['role']): if command['roleCommand'] == self.ROLE_COMMAND_START: self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.LIVE_STATUS) self.controller.recovery_manager.update_config_staleness(command['role'], False) logger.info("After EXECUTION_COMMAND (START), with taskId=" + str(command['taskId']) + ", current state of " + command['role'] + " to " + self.controller.recovery_manager.get_current_status(command['role']) ) elif command['roleCommand'] == self.ROLE_COMMAND_STOP or command['roleCommand'] == self.ROLE_COMMAND_INSTALL: self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.DEAD_STATUS) logger.info("After EXECUTION_COMMAND (STOP/INSTALL), with taskId=" + str(command['taskId']) + ", current state of " + command['role'] + " to " + self.controller.recovery_manager.get_current_status(command['role']) ) elif command['roleCommand'] == self.ROLE_COMMAND_CUSTOM_COMMAND: if command['hostLevelParams'].has_key('custom_command') and \ command['hostLevelParams']['custom_command'] == self.CUSTOM_COMMAND_RESTART: self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.LIVE_STATUS) self.controller.recovery_manager.update_config_staleness(command['role'], False) logger.info("After EXECUTION_COMMAND (RESTART), current state of " + command['role'] + " to " + self.controller.recovery_manager.get_current_status(command['role']) ) pass # let ambari know that configuration tags were applied configHandler = ActualConfigHandler(self.config, self.configTags) if command.has_key('configurationTags'): configHandler.write_actual(command['configurationTags']) roleResult['configurationTags'] = command['configurationTags'] component = {'serviceName':command['serviceName'],'componentName':command['role']} if 'roleCommand' in command and \ (command['roleCommand'] == self.ROLE_COMMAND_START or (command['roleCommand'] == self.ROLE_COMMAND_INSTALL and component in LiveStatus.CLIENT_COMPONENTS) or (command['roleCommand'] == self.ROLE_COMMAND_CUSTOM_COMMAND and 'custom_command' in command['hostLevelParams'] and command['hostLevelParams']['custom_command'] == self.CUSTOM_COMMAND_RESTART)): configHandler.write_actual_component(command['role'], command['configurationTags']) if 'clientsToUpdateConfigs' in command['hostLevelParams'] and command['hostLevelParams']['clientsToUpdateConfigs']: configHandler.write_client_components(command['serviceName'], command['configurationTags'], command['hostLevelParams']['clientsToUpdateConfigs']) roleResult['configurationTags'] = configHandler.read_actual_component( command['role']) elif status == self.FAILED_STATUS: if self.controller.recovery_manager.enabled() and command.has_key('roleCommand') \ and self.controller.recovery_manager.configured_for_recovery(command['role']): if command['roleCommand'] == self.ROLE_COMMAND_INSTALL: self.controller.recovery_manager.update_current_status(command['role'], self.controller.recovery_manager.INSTALL_FAILED) logger.info("After EXECUTION_COMMAND (INSTALL), with taskId=" + str(command['taskId']) + ", current state of " + command['role'] + " to " + self.controller.recovery_manager.get_current_status(command['role'])) self.commandStatuses.put_command_status(command, roleResult)
def create_ranger_repository(self, component, repo_name, repo_properties, ambari_ranger_admin, ambari_ranger_password, admin_uname, admin_password, policy_user, is_security_enabled = False, is_stack_supports_ranger_kerberos = False, component_user = None, component_user_principal = None, component_user_keytab = None): if not is_stack_supports_ranger_kerberos or not is_security_enabled: response_code = self.check_ranger_login_urllib2(self.base_url) repo_data = json.dumps(repo_properties) ambari_ranger_password = unicode(ambari_ranger_password) admin_password = unicode(admin_password) ambari_username_password_for_ranger = format('{ambari_ranger_admin}:{ambari_ranger_password}') if response_code is not None and response_code == 200: user_resp_code = self.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}")) if user_resp_code is not None and user_resp_code == 200: retryCount = 0 while retryCount <= 5: repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger) if repo is not None: Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name'])) break else: response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger) if response is not None: Logger.info('{0} Repository created in Ranger admin'.format(component.title())) break else: if retryCount < 5: Logger.info("Retry Repository Creation is being called") time.sleep(30) # delay for 30 seconds retryCount += 1 else: Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) break else: Logger.error('Ambari admin user creation failed') elif not self.skip_if_rangeradmin_down: Logger.error("Connection failed to Ranger Admin !") elif is_stack_supports_ranger_kerberos and is_security_enabled: response = self.check_ranger_login_curl(component_user,component_user_keytab,component_user_principal,self.url_login,True) if response and response[0] == 200: retryCount = 0 repo_data = json.dumps(repo_properties) while retryCount <= 5: response = self.get_repository_by_name_curl(component_user,component_user_keytab,component_user_principal,repo_name, component, 'true') if response is not None: Logger.info('{0} Repository {1} exist'.format(component.title(), (response['name']))) break else: response = self.create_repository_curl(component_user,component_user_keytab,component_user_principal,repo_name, repo_data,policy_user) if response and len(response) > 0: Logger.info('{0} Repository created in Ranger admin'.format(component.title())) break else: if retryCount < 5: time.sleep(30) # delay for 30 seconds retryCount += 1 else: Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) break else: Logger.error("Connection failed to Ranger Admin !")
'username': repo_config_username, 'password': repo_config_password, 'hadoop.security.authentication': hadoop_security_authentication, 'hadoop.security.authorization': hadoop_security_authorization, 'fs.default.name': fs_default_name, 'hadoop.security.auth_to_local': hadoop_security_auth_to_local, 'hadoop.rpc.protection': hadoop_rpc_protection, 'commonNameForCertificate': common_name_for_certificate, 'dfs.datanode.kerberos.principal': dn_principal_name if security_enabled else '', 'dfs.namenode.kerberos.principal': nn_principal_name if security_enabled else '', 'dfs.secondary.namenode.kerberos.principal': sn_principal_name if security_enabled else '' } hdfs_ranger_plugin_repo = { 'isActive': 'true', 'config': json.dumps(hdfs_ranger_plugin_config), 'description': 'hdfs repo', 'name': repo_name, 'repositoryType': 'hdfs', 'assetType': '1' } ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla':
ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}") ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_symlink_name}") ranger_driver_curl_target = format("{java_share_dir}/{ranger_jdbc_jar_name}") hive_ranger_plugin_config = { "username": repo_config_username, "password": repo_config_password, "jdbc.driverClassName": jdbc_driver_class_name, "jdbc.url": format("{hive_url}/default;principal={hive_principal}") if security_enabled else hive_url, "commonNameForCertificate": common_name_for_certificate, } hive_ranger_plugin_repo = { "isActive": "true", "config": json.dumps(hive_ranger_plugin_config), "description": "hive repo", "name": repo_name, "repositoryType": "hive", "assetType": "3", } xa_audit_db_password = unicode(config["configurations"]["admin-properties"]["audit_db_password"]) ranger_audit_solr_urls = config["configurations"]["ranger-admin-site"]["ranger.audit.solr.urls"] xa_audit_db_is_enabled = ( config["configurations"]["ranger-hive-audit"]["xasecure.audit.destination.db"] if xml_configurations_supported else None ) ssl_keystore_password = ( unicode(config["configurations"]["ranger-hive-policymgr-ssl"]["xasecure.policymgr.clientssl.keystore.password"])
def main(argv=None): for service in SERVICES: livestatus = LiveStatus('', service) print json.dumps(livestatus.build())
downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") driver_curl_target = format("{storm_component_home_dir}/lib/{jdbc_jar_name}") storm_ranger_plugin_config = { 'username': repo_config_username, 'password': repo_config_password, 'nimbus.url': 'http://' + storm_ui_host[0].lower() + ':' + str(storm_ui_port), 'commonNameForCertificate': common_name_for_certificate } storm_ranger_plugin_repo = { 'isActive': 'true', 'config': json.dumps(storm_ranger_plugin_config), 'description': 'storm repo', 'name': repo_name, 'repositoryType': 'storm', 'assetType': '6' } ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-storm-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None xa_audit_hdfs_is_enabled = config['configurations']['ranger-storm-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla':
def flume(action = None): import params if action == 'config': # remove previously defined meta's for n in find_expected_agent_names(params.flume_conf_dir): File(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'), action = "delete", ) Directory(params.flume_run_dir, ) Directory(params.flume_conf_dir, recursive=True, owner=params.flume_user, ) Directory(params.flume_log_dir, owner=params.flume_user) flume_agents = {} if params.flume_conf_content is not None: flume_agents = build_flume_topology(params.flume_conf_content) for agent in flume_agents.keys(): flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent) flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf') flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json') flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties') flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.sh') Directory(flume_agent_conf_dir, owner=params.flume_user, ) PropertiesFile(flume_agent_conf_file, properties=flume_agents[agent], owner=params.flume_user, mode = 0644) File(flume_agent_log4j_file, content=Template('log4j.properties.j2', agent_name = agent), owner=params.flume_user, mode = 0644) File(flume_agent_meta_file, content = json.dumps(ambari_meta(agent, flume_agents[agent])), owner=params.flume_user, mode = 0644) File(flume_agent_env_file, owner=params.flume_user, content=InlineTemplate(params.flume_env_sh_template) ) if params.has_metric_collector: File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"), owner=params.flume_user, content=Template("flume-metrics2.properties.j2") ) elif action == 'start': # desired state for service should be STARTED if len(params.flume_command_targets) == 0: _set_desired_state('STARTED') # It is important to run this command as a background process. flume_base = as_user(format("{flume_bin} agent --name {{0}} --conf {{1}} --conf-file {{2}} {{3}} > {flume_log_dir}/{{4}}.out 2>&1"), params.flume_user, env={'JAVA_HOME': params.java_home}) + " &" for agent in cmd_target_names(): flume_agent_conf_dir = params.flume_conf_dir + os.sep + agent flume_agent_conf_file = flume_agent_conf_dir + os.sep + "flume.conf" flume_agent_pid_file = params.flume_run_dir + os.sep + agent + ".pid" if not os.path.isfile(flume_agent_conf_file): continue if not is_flume_process_live(flume_agent_pid_file): # TODO someday make the ganglia ports configurable extra_args = '' if params.ganglia_server_host is not None: extra_args = '-Dflume.monitoring.type=ganglia -Dflume.monitoring.hosts={0}:{1}' extra_args = extra_args.format(params.ganglia_server_host, '8655') if params.has_metric_collector: extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \ '-Dflume.monitoring.node={0}:{1}' extra_args = extra_args.format(params.metric_collector_host, params.metric_collector_port) flume_cmd = flume_base.format(agent, flume_agent_conf_dir, flume_agent_conf_file, extra_args, agent) Execute(flume_cmd, wait_for_finish=False, environment={'JAVA_HOME': params.java_home} ) # sometimes startup spawns a couple of threads - so only the first line may count pid_cmd = as_sudo(('pgrep', '-o', '-u', params.flume_user, '-f', format('^{java_home}.*{agent}.*'))) + \ " | " + as_sudo(('tee', flume_agent_pid_file)) + " && test ${PIPESTATUS[0]} -eq 0" Execute(pid_cmd, logoutput=True, tries=20, try_sleep=10) pass elif action == 'stop': # desired state for service should be INSTALLED if len(params.flume_command_targets) == 0: _set_desired_state('INSTALLED') pid_files = glob.glob(params.flume_run_dir + os.sep + "*.pid") if 0 == len(pid_files): return agent_names = cmd_target_names() for agent in agent_names: pid_file = format("{flume_run_dir}/{agent}.pid") if is_flume_process_live(pid_file): pid = shell.checked_call(("cat", pid_file), sudo=True)[1].strip() Execute(("kill", "-15", pid), sudo=True) # kill command has to be a tuple if not await_flume_process_termination(pid_file): raise Fail("Can't stop flume agent: {0}".format(agent)) File(pid_file, action = 'delete')
def enable_kms_plugin(): import params if params.has_ranger_admin: ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url) response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url + '/login.jsp', 'test:test') if response_code is not None and response_code == 200: ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password) ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password else: raise Fail('Ranger service is not started on given host') if ambari_ranger_admin != '' and ambari_ranger_password != '': get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger) if not get_repo_flag: create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger) else: raise Fail('Ambari admin username and password not available') current_datetime = datetime.now() File(format('{kms_conf_dir}/ranger-security.xml'), owner = params.kms_user, group = params.kms_group, mode = 0644, content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')) ) Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')], owner = params.kms_user, group = params.kms_group, mode=0775, recursive = True ) File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')), owner = params.kms_user, group = params.kms_group, mode = 0644 ) XmlConfig("ranger-kms-audit.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-audit'], configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig("ranger-kms-security.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-security'], configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig("ranger-policymgr-ssl.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-policymgr-ssl'], configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'], owner=params.kms_user, group=params.kms_group, mode=0744) if params.xa_audit_db_is_enabled: cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.credential_file, owner = params.kms_user, group = params.kms_group, mode = 0640 )
def create_ranger_repository(self, component, repo_name, repo_properties, ambari_ranger_admin, ambari_ranger_password, admin_uname, admin_password, policy_user, is_security_enabled, component_user, component_user_principal, component_user_keytab): if not is_security_enabled: response_code = self.check_ranger_login_urllib2(self.base_url) repo_data = json.dumps(repo_properties) ambari_ranger_password = unicode(ambari_ranger_password) admin_password = unicode(admin_password) ambari_username_password_for_ranger = format( '{ambari_ranger_admin}:{ambari_ranger_password}') if response_code is not None and response_code == 200: user_resp_code = self.create_ambari_admin_user( ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}")) if user_resp_code is not None and user_resp_code == 200: retryCount = 0 while retryCount <= 5: repo = self.get_repository_by_name_urllib2( repo_name, component, 'true', ambari_username_password_for_ranger) if repo is not None: Logger.info('{0} Repository {1} exist'.format( component.title(), repo['name'])) break else: response = self.create_repository_urllib2( repo_data, ambari_username_password_for_ranger) if response is not None: Logger.info( '{0} Repository created in Ranger admin'. format(component.title())) break else: if retryCount < 5: Logger.info( "Retry Repository Creation is being called" ) time.sleep(30) # delay for 30 seconds retryCount += 1 else: Logger.error( '{0} Repository creation failed in Ranger admin' .format(component.title())) break else: Logger.error('Ambari admin user creation failed') elif not self.skip_if_rangeradmin_down: Logger.error("Connection failed to Ranger Admin !") else: response = self.check_ranger_login_curl(component_user, component_user_keytab, component_user_principal, self.base_url, True) if response and response[0] == 200: retryCount = 0 repo_data = json.dumps(repo_properties) while retryCount <= 5: response = self.get_repository_by_name_curl( component_user, component_user_keytab, component_user_principal, repo_name, component, 'true') if response is not None: Logger.info('{0} Repository {1} exist'.format( component.title(), (response['name']))) break else: response = self.create_repository_curl( component_user, component_user_keytab, component_user_principal, repo_name, repo_data, policy_user) if response and len(response) > 0: Logger.info( '{0} Repository created in Ranger admin'. format(component.title())) break else: if retryCount < 5: time.sleep(30) # delay for 30 seconds retryCount += 1 else: Logger.error( '{0} Repository creation failed in Ranger admin' .format(component.title())) break else: Logger.error("Connection failed to Ranger Admin !")