def main(argv=None):
from ambari_agent.ActionQueue import ActionQueue
from ambari_agent.AmbariConfig import AmbariConfig
from ambari_agent.Controller import Controller
cfg = AmbariConfig()
if os.path.exists(AmbariConfig.getConfigFile()):
cfg.read(AmbariConfig.getConfigFile())
else:
raise Exception("No config found, use default")
ctl = Controller(cfg)
actionQueue = ActionQueue(cfg, ctl)
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build('3',3))
def dump_command_to_json(self, command, retry=False):
"""
Converts command to json file and returns file path
"""
# Perform few modifications to stay compatible with the way in which
public_fqdn = self.public_fqdn
command['public_hostname'] = public_fqdn
# Add cache dir to make it visible for commands
command["hostLevelParams"]["agentCacheDir"] = self.config.get('agent', 'cache_dir')
# Now, dump the json file
command_type = command['commandType']
from ActionQueue import ActionQueue # To avoid cyclic dependency
if command_type == ActionQueue.STATUS_COMMAND:
# These files are frequently created, that's why we don't
# store them all, but only the latest one
file_path = os.path.join(self.tmp_dir, "status_command.json")
else:
task_id = command['taskId']
if 'clusterHostInfo' in command and command['clusterHostInfo'] and not retry:
command['clusterHostInfo'] = self.decompressClusterHostInfo(command['clusterHostInfo'])
file_path = os.path.join(self.tmp_dir, "command-{0}.json".format(task_id))
if command_type == ActionQueue.AUTO_EXECUTION_COMMAND:
file_path = os.path.join(self.tmp_dir, "auto_command-{0}.json".format(task_id))
# Json may contain passwords, that's why we need proper permissions
if os.path.isfile(file_path):
os.unlink(file_path)
with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
0600), 'w') as f:
content = json.dumps(command, sort_keys = False, indent = 4)
f.write(content)
def check_ranger_service():
import params
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}")
response_code = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url)
if response_code is not None and response_code == 200:
user_resp_code = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
if user_resp_code is not None and user_resp_code == 200:
get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo_flag = create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
if create_repo_flag:
return True
else:
return False
else:
return True
else:
Logger.error('Ambari admin user creation failed')
return False
else:
Logger.error('Ranger service is not reachable host')
return False
def create_repository_urllib2(self, data, usernamepassword, policy_user):
"""
:param data: repository dict
:param usernamepassword: user credentials using which repository needs to be created
:param policy_user: use this policy user for policies that will be used during repository creation
:return Returns created repository response else None
"""
try:
searchRepoURL = self.urlReposPub
base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(searchRepoURL, data, headers)
request.add_header("Authorization", "Basic {0}".format(base64string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Repository created Successfully')
# Get Policies
repoData = json.loads(data)
repoName = repoData['name']
typeOfPolicy = repoData['repositoryType']
# Get Policies by repo name
policyList = self.get_policy_by_repo_name(name=repoName, component=typeOfPolicy, status="true",
usernamepassword=usernamepassword)
if policyList is not None and (len(policyList)) > 0:
policiesUpdateCount = 0
for policy in policyList:
updatedPolicyObj = self.get_policy_params(typeOfPolicy, policy, policy_user)
policyResCode = self.update_ranger_policy(updatedPolicyObj['id'], json.dumps(updatedPolicyObj), usernamepassword)
if policyResCode == 200:
policiesUpdateCount = policiesUpdateCount + 1
else:
Logger.info('Policy Update failed')
# Check for count of updated policies
if len(policyList) == policiesUpdateCount:
Logger.info(
"Ranger Repository created successfully and policies updated successfully providing ambari-qa user all permissions")
return response
else:
return None
else:
Logger.info("Policies not found for the newly created Repository")
return None
else:
Logger.info('Repository creation failed')
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail("Error creating repository. Http status code - {0}. \n {1}".format(e.code, e.read()))
else:
raise Fail("Error creating repository. Reason - {0}.".format(e.reason))
def set_current(options):
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
finalize_options = SetCurrentVersionOptions(options)
if finalize_options.no_finalize_options_set():
err = 'Must specify --cluster-name and --version-display-name. Please invoke ambari-server.py --help to print the options.'
raise FatalException(1, err)
admin_login = get_validated_string_input(prompt="Enter Ambari Admin login: "******"Enter Ambari Admin password: "******"Failed to read properties file.")
base_url = get_ambari_server_api_base(properties)
url = base_url + "clusters/{0}/stack_versions".format(finalize_options.cluster_name)
admin_auth = base64.encodestring('%s:%s' % (admin_login, admin_password)).replace('\n', '')
request = urllib2.Request(url)
request.add_header('Authorization', 'Basic %s' % admin_auth)
request.add_header('X-Requested-By', 'ambari')
data = {
"ClusterStackVersions": {
"repository_version": finalize_options.desired_repo_version,
"state": "CURRENT",
"force": "true" if finalize_options.force_repo_version else "false"
}
}
if get_verbose():
sys.stdout.write('\nCalling API ' + url + ' : ' + str(data) + '\n')
request.add_data(json.dumps(data))
request.get_method = lambda: 'PUT'
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError, e:
code = e.getcode()
content = e.read()
err = 'Error during setting current version. Http status code - {0}. \n {1}'.format(
code, content)
raise FatalException(1, err)
def flume(action = None):
import params
from service_mapping import flume_win_service_name
if action == 'config':
ServiceConfig(flume_win_service_name,
action="configure",
start_type="manual")
ServiceConfig(flume_win_service_name,
action="change_user",
username=params.flume_user,
password = Script.get_password(params.flume_user))
# remove previously defined meta's
for n in find_expected_agent_names(params.flume_conf_dir):
os.unlink(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'))
flume_agents = {}
if params.flume_conf_content is not None:
flume_agents = build_flume_topology(params.flume_conf_content)
for agent in flume_agents.keys():
flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent)
flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf')
flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json')
flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties')
flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.ps1')
Directory(flume_agent_conf_dir
)
PropertiesFile(flume_agent_conf_file,
properties=flume_agents[agent])
File(flume_agent_log4j_file,
content=Template('log4j.properties.j2', agent_name = agent))
File(flume_agent_meta_file,
content = json.dumps(ambari_meta(agent, flume_agents[agent])))
File(flume_agent_env_file,
owner=params.flume_user,
content=InlineTemplate(params.flume_env_sh_template)
)
if params.has_metric_collector:
File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"),
owner=params.flume_user,
content=Template("flume-metrics2.properties.j2")
)
def createDB(self, env):
import params
env.set_params(params)
if params.db_name and params.db_pass:
user_json = {'user': params.db_user, 'pwd': params.db_pass, 'roles': ['readWrite']}
create_user_cmd = (
"mongo {db} --eval 'db.getUser(\"{user}\")' | grep -q -w {db}\\.{user} || "
"mongo {db} --eval 'db.createUser({json});'"
)
Execute(create_user_cmd.format(
db=params.db_name,
user=params.db_user,
json=json.dumps(user_json)))
def create_ranger_repository(self, component, repo_name, repo_properties,
ambari_ranger_admin, ambari_ranger_password,
admin_uname, admin_password, policy_user):
"""
:param component: name of the component, from which it will get or create repository
:param repo_name: name of the repository to be get or create
:param repo_properties: dict of repository to be create if not exist
:param ambari_ranger_admin: ambari admin user creation username
:param ambari_ranger_password: ambari admin user creation password
:param admin_uname: ranger admin username
:param admin_password: ranger admin password
:param policy_user: use this policy user for policies that will be used during repository creation
"""
response_code = self.check_ranger_login_urllib2(self.baseUrl)
repo_data = json.dumps(repo_properties)
ambari_ranger_password = unicode(ambari_ranger_password)
admin_password = unicode(admin_password)
ambari_username_password_for_ranger = format('{ambari_ranger_admin}:{ambari_ranger_password}')
if response_code is not None and response_code == 200:
user_resp_code = self.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}"))
if user_resp_code is not None and user_resp_code == 200:
retryCount = 0
while retryCount <= 5:
repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger)
if repo is not None:
Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name']))
break
else:
response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger, policy_user)
if response is not None:
Logger.info('{0} Repository created in Ranger admin'.format(component.title()))
break
else:
if retryCount < 5:
Logger.info("Retry Repository Creation is being called")
time.sleep(15) # delay for 15 seconds
retryCount += 1
else:
Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title()))
break
else:
Logger.error('Ambari admin user creation failed')
elif not self.skip_if_rangeradmin_down:
Logger.error("Connection to Ranger Admin failed !")
def run_metainfo_upgrade(keyValueMap=None):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
retcode = 1
if keyValueMap:
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
'updateMetaInfo',
"'" + json.dumps(keyValueMap) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing metainfo upgrade, please check the "
"server logs.")
return retcode
def action_execute(self, main_resource):
env = Environment.get_instance()
# Check required parameters
main_resource.assert_parameter_is_set('user')
if not 'hdfs_files' in env.config or not env.config['hdfs_files']:
Logger.info("No resources to create. 'create_on_execute' or 'delete_on_execute' wasn't triggered before this 'execute' action.")
return
hadoop_bin_dir = main_resource.resource.hadoop_bin_dir
hadoop_conf_dir = main_resource.resource.hadoop_conf_dir
user = main_resource.resource.user
security_enabled = main_resource.resource.security_enabled
keytab_file = main_resource.resource.keytab
kinit_path = main_resource.resource.kinit_path_local
logoutput = main_resource.resource.logoutput
principal_name = main_resource.resource.principal_name
jar_path=JAR_PATH
timestamp = time.time()
json_path=format(JSON_PATH)
if security_enabled:
main_resource.kinit()
# Write json file to disk
File(json_path,
owner = user,
content = json.dumps(env.config['hdfs_files'])
)
# Execute jar to create/delete resources in hadoop
Execute(format("hadoop --config {hadoop_conf_dir} jar {jar_path} {json_path}"),
user=user,
path=[hadoop_bin_dir],
logoutput=logoutput,
)
# Clean
env.config['hdfs_files'] = []
def run_command(self, target, operation, method='POST', assertable_result=True, file_to_put=None, ignore_status_codes=[], **kwargs):
"""
assertable_result - some POST requests return '{"boolean":false}' or '{"boolean":true}'
depending on if query was successful or not, we can assert this for them
"""
target = HdfsResourceProvider.parse_path(target)
url = format("{address}/webhdfs/v1{target}?op={operation}&user.name={run_user}", address=self.address, run_user=self.run_user)
for k,v in kwargs.iteritems():
url = format("{url}&{k}={v}")
if file_to_put and not os.path.exists(file_to_put):
raise Fail(format("File {file_to_put} is not found."))
cmd = ["curl", "-sS","-L", "-w", "%{http_code}", "-X", method]
if file_to_put:
cmd += ["-T", file_to_put]
if self.security_enabled:
cmd += ["--negotiate", "-u", ":"]
if self.is_https_enabled:
cmd += ["-k"]
cmd.append(url)
_, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)
status_code = out[-3:]
out = out[:-3] # remove last line from output which is status code
try:
result_dict = json.loads(out)
except ValueError:
result_dict = out
if status_code not in WebHDFSUtil.valid_status_codes+ignore_status_codes or assertable_result and result_dict and not result_dict['boolean']:
formatted_output = json.dumps(result_dict, indent=2) if isinstance(result_dict, dict) else result_dict
formatted_output = err + "\n" + formatted_output
err_msg = "Execution of '%s' returned status_code=%s. %s" % (shell.string_cmd_from_args_list(cmd), status_code, formatted_output)
raise Fail(err_msg)
return result_dict
def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
stackId = {}
stackId[stackName] = stackVersion
if repo_url is not None:
stackId['repo_url'] = repo_url
if repo_url_os is not None:
stackId['repo_url_os'] = repo_url_os
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
"updateStackId",
"'" + json.dumps(stackId) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing stack upgrade, please check the server logs.")
return retcode
def reqSignCrt(self):
sign_crt_req_url = self.server_url + '/certs/' + hostname.hostname(
self.config)
agent_crt_req_f = open(self.getAgentCrtReqName())
agent_crt_req_content = agent_crt_req_f.read()
passphrase_env_var = self.config.get('security', 'passphrase_env_var_name')
passphrase = os.environ[passphrase_env_var]
register_data = {'csr': agent_crt_req_content,
'passphrase': passphrase}
data = json.dumps(register_data)
proxy_handler = urllib2.ProxyHandler({})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
req = urllib2.Request(sign_crt_req_url, data,
{'Content-Type': 'application/json'})
f = urllib2.urlopen(req)
response = f.read()
f.close()
try:
data = json.loads(response)
logger.debug("Sign response from Server: \n" + pprint.pformat(data))
except Exception:
logger.warn("Malformed response! data: " + str(data))
data = {'result': 'ERROR'}
result = data['result']
if result == 'OK':
agentCrtContent = data['signedCa']
agentCrtF = open(self.getAgentCrtName(), "w")
agentCrtF.write(agentCrtContent)
else:
# Possible exception is catched higher at Controller
logger.error('Certificate signing failed.'
'\nIn order to receive a new agent'
' certificate, remove existing certificate file from keys '
'directory. As a workaround you can turn off two-way SSL '
'authentication in server configuration(ambari.properties) '
'\nExiting..')
raise ssl.SSLError
def on_background_command_complete_callback(self, process_condensed_result, handle):
logger.debug('Start callback: %s' % process_condensed_result)
logger.debug('The handle is: %s' % handle)
status = self.COMPLETED_STATUS if handle.exitCode == 0 else self.FAILED_STATUS
aborted_postfix = self.customServiceOrchestrator.command_canceled_reason(handle.command['taskId'])
if aborted_postfix:
status = self.FAILED_STATUS
logger.debug('Set status to: %s , reason = %s' % (status, aborted_postfix))
else:
aborted_postfix = ''
roleResult = self.commandStatuses.generate_report_template(handle.command)
roleResult.update({
'stdout': process_condensed_result['stdout'] + aborted_postfix,
'stderr': process_condensed_result['stderr'] + aborted_postfix,
'exitCode': process_condensed_result['exitcode'],
'structuredOut': str(json.dumps(process_condensed_result['structuredOut'])) if 'structuredOut' in process_condensed_result else '',
'status': status,
})
self.commandStatuses.put_command_status(handle.command, roleResult)
def create_ranger_repository(self, component, repo_name, repo_properties,
ambari_ranger_admin, ambari_ranger_password,
admin_uname, admin_password, policy_user):
response_code = self.check_ranger_login_urllib2(self.base_url)
repo_data = json.dumps(repo_properties)
ambari_ranger_password = unicode(ambari_ranger_password)
admin_password = unicode(admin_password)
ambari_username_password_for_ranger = format('{ambari_ranger_admin}:{ambari_ranger_password}')
if response_code is not None and response_code == 200:
user_resp_code = self.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}"))
if user_resp_code is not None and user_resp_code == 200:
retryCount = 0
while retryCount <= 5:
repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger)
if repo is not None:
Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name']))
break
else:
response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger)
if response is not None:
Logger.info('{0} Repository created in Ranger admin'.format(component.title()))
break
else:
if retryCount < 5:
Logger.info("Retry Repository Creation is being called")
time.sleep(30) # delay for 30 seconds
retryCount += 1
else:
Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title()))
break
else:
Logger.error('Ambari admin user creation failed')
elif not self.skip_if_rangeradmin_down:
Logger.error("Connection failed to Ranger Admin !")
def execute_command(self, command):
'''
Executes commands of type EXECUTION_COMMAND
'''
clusterName = command['clusterName']
commandId = command['commandId']
isCommandBackground = command['commandType'] == self.BACKGROUND_EXECUTION_COMMAND
isAutoExecuteCommand = command['commandType'] == self.AUTO_EXECUTION_COMMAND
message = "Executing command with id = {commandId} for role = {role} of " \
"cluster {cluster}.".format(
commandId = str(commandId), role=command['role'],
cluster=clusterName)
logger.info(message)
taskId = command['taskId']
# Preparing 'IN_PROGRESS' report
in_progress_status = self.commandStatuses.generate_report_template(command)
# The path of the files that contain the output log and error log use a prefix that the agent advertises to the
# server. The prefix is defined in agent-config.ini
if not isAutoExecuteCommand:
in_progress_status.update({
'tmpout': self.tmpdir + os.sep + 'output-' + str(taskId) + '.txt',
'tmperr': self.tmpdir + os.sep + 'errors-' + str(taskId) + '.txt',
'structuredOut' : self.tmpdir + os.sep + 'structured-out-' + str(taskId) + '.json',
'status': self.IN_PROGRESS_STATUS
})
else:
in_progress_status.update({
'tmpout': self.tmpdir + os.sep + 'auto_output-' + str(taskId) + '.txt',
'tmperr': self.tmpdir + os.sep + 'auto_errors-' + str(taskId) + '.txt',
'structuredOut' : self.tmpdir + os.sep + 'auto_structured-out-' + str(taskId) + '.json',
'status': self.IN_PROGRESS_STATUS
})
self.commandStatuses.put_command_status(command, in_progress_status)
numAttempts = 0
retryDuration = 0 # even with 0 allow one attempt
retryAble = False
delay = 1
if 'commandParams' in command:
if 'max_duration_for_retries' in command['commandParams']:
retryDuration = int(command['commandParams']['max_duration_for_retries'])
if 'command_retry_enabled' in command['commandParams']:
retryAble = command['commandParams']['command_retry_enabled'] == "true"
if isAutoExecuteCommand:
retryAble = False
logger.debug("Command execution metadata - retry enabled = {retryAble}, max retry duration (sec) = {retryDuration}".
format(retryAble=retryAble, retryDuration=retryDuration))
while retryDuration >= 0:
numAttempts += 1
start = 0
if retryAble:
start = int(time.time())
# running command
commandresult = self.customServiceOrchestrator.runCommand(command,
in_progress_status['tmpout'],
in_progress_status['tmperr'],
override_output_files=numAttempts == 1,
retry=numAttempts > 1)
end = 1
if retryAble:
end = int(time.time())
retryDuration -= (end - start)
# dumping results
if isCommandBackground:
return
else:
if commandresult['exitcode'] == 0:
status = self.COMPLETED_STATUS
else:
status = self.FAILED_STATUS
if status != self.COMPLETED_STATUS and retryAble == True and retryDuration > 0:
delay = self.get_retry_delay(delay)
if delay > retryDuration:
delay = retryDuration
retryDuration -= delay # allow one last attempt
logger.info("Retrying command id {cid} after a wait of {delay}".format(cid=taskId, delay=delay))
time.sleep(delay)
continue
else:
break
roleResult = self.commandStatuses.generate_report_template(command)
roleResult.update({
'stdout': commandresult['stdout'],
'stderr': commandresult['stderr'],
'exitCode': commandresult['exitcode'],
'status': status,
})
if roleResult['stdout'] == '':
roleResult['stdout'] = 'None'
if roleResult['stderr'] == '':
roleResult['stderr'] = 'None'
# let ambari know name of custom command
if command['hostLevelParams'].has_key('custom_command'):
roleResult['customCommand'] = command['hostLevelParams']['custom_command']
if 'structuredOut' in commandresult:
roleResult['structuredOut'] = str(json.dumps(commandresult['structuredOut']))
else:
roleResult['structuredOut'] = ''
# let recovery manager know the current state
if status == self.COMPLETED_STATUS:
if self.controller.recovery_manager.enabled() and command.has_key('roleCommand') \
and self.controller.recovery_manager.configured_for_recovery(command['role']):
if command['roleCommand'] == self.ROLE_COMMAND_START:
self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.LIVE_STATUS)
self.controller.recovery_manager.update_config_staleness(command['role'], False)
logger.info("After EXECUTION_COMMAND (START), current state of " + command['role'] + " to " +
self.controller.recovery_manager.get_current_status(command['role']) )
elif command['roleCommand'] == self.ROLE_COMMAND_STOP or command['roleCommand'] == self.ROLE_COMMAND_INSTALL:
self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.DEAD_STATUS)
logger.info("After EXECUTION_COMMAND (STOP/INSTALL), current state of " + command['role'] + " to " +
self.controller.recovery_manager.get_current_status(command['role']) )
elif command['roleCommand'] == self.ROLE_COMMAND_CUSTOM_COMMAND:
if command['hostLevelParams'].has_key('custom_command') and \
command['hostLevelParams']['custom_command'] == self.CUSTOM_COMMAND_RESTART:
self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.LIVE_STATUS)
self.controller.recovery_manager.update_config_staleness(command['role'], False)
logger.info("After EXECUTION_COMMAND (RESTART), current state of " + command['role'] + " to " +
self.controller.recovery_manager.get_current_status(command['role']) )
pass
# let ambari know that configuration tags were applied
configHandler = ActualConfigHandler(self.config, self.configTags)
#update
if command.has_key('forceRefreshConfigTags') and len(command['forceRefreshConfigTags']) > 0 :
forceRefreshConfigTags = command['forceRefreshConfigTags']
logger.info("Got refresh additional component tags command")
for configTag in forceRefreshConfigTags :
configHandler.update_component_tag(command['role'], configTag, command['configurationTags'][configTag])
roleResult['customCommand'] = self.CUSTOM_COMMAND_RESTART # force restart for component to evict stale_config on server side
command['configurationTags'] = configHandler.read_actual_component(command['role'])
if command.has_key('configurationTags'):
configHandler.write_actual(command['configurationTags'])
roleResult['configurationTags'] = command['configurationTags']
component = {'serviceName':command['serviceName'],'componentName':command['role']}
if command.has_key('roleCommand') and \
(command['roleCommand'] == self.ROLE_COMMAND_START or \
(command['roleCommand'] == self.ROLE_COMMAND_INSTALL \
and component in LiveStatus.CLIENT_COMPONENTS) or \
(command['roleCommand'] == self.ROLE_COMMAND_CUSTOM_COMMAND and \
command['hostLevelParams'].has_key('custom_command') and \
command['hostLevelParams']['custom_command'] == self.CUSTOM_COMMAND_RESTART)):
configHandler.write_actual_component(command['role'], command['configurationTags'])
if command['hostLevelParams'].has_key('clientsToUpdateConfigs') and \
command['hostLevelParams']['clientsToUpdateConfigs']:
configHandler.write_client_components(command['serviceName'], command['configurationTags'],
command['hostLevelParams']['clientsToUpdateConfigs'])
roleResult['configurationTags'] = configHandler.read_actual_component(command['role'])
self.commandStatuses.put_command_status(command, roleResult)
def sync_ldap(options):
if not is_root():
err = 'Ambari-server sync-ldap should be run with ' \
'root-level privileges'
raise FatalException(4, err)
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
properties = get_ambari_properties()
if properties == -1:
raise FatalException(1, "Failed to read properties file.")
ldap_configured = properties.get_property(IS_LDAP_CONFIGURED)
if ldap_configured != 'true':
err = "LDAP is not configured. Run 'ambari-server setup-ldap' first."
raise FatalException(1, err)
# set ldap sync options
ldap_sync_options = LdapSyncOptions(options)
if ldap_sync_options.no_ldap_sync_options_set():
err = 'Must specify a sync option (all, existing, users or groups). Please invoke ambari-server.py --help to print the options.'
raise FatalException(1, err)
admin_login = get_validated_string_input(prompt="Enter Ambari Admin login: "******"Enter Ambari Admin password: "******"Event":{"specs":[{"principal_type":"users","sync_type":"all"},{"principal_type":"groups","sync_type":"all"}]}}]
elif ldap_sync_options.ldap_sync_existing:
sys.stdout.write('Syncing existing.')
bodies = [{"Event":{"specs":[{"principal_type":"users","sync_type":"existing"},{"principal_type":"groups","sync_type":"existing"}]}}]
else:
sys.stdout.write('Syncing specified users and groups.')
bodies = [{"Event":{"specs":[]}}]
body = bodies[0]
events = body['Event']
specs = events['specs']
if ldap_sync_options.ldap_sync_users is not None:
new_specs = [{"principal_type":"users","sync_type":"specific","names":""}]
get_ldap_event_spec_names(ldap_sync_options.ldap_sync_users, specs, new_specs)
if ldap_sync_options.ldap_sync_groups is not None:
new_specs = [{"principal_type":"groups","sync_type":"specific","names":""}]
get_ldap_event_spec_names(ldap_sync_options.ldap_sync_groups, specs, new_specs)
if get_verbose():
sys.stdout.write('\nCalling API ' + url + ' : ' + str(bodies) + '\n')
request.add_data(json.dumps(bodies))
request.get_method = lambda: 'POST'
try:
response = urllib2.urlopen(request)
except Exception as e:
err = 'Sync event creation failed. Error details: %s' % e
raise FatalException(1, err)
response_status_code = response.getcode()
if response_status_code != 201:
err = 'Error during syncing. Http status code - ' + str(response_status_code)
raise FatalException(1, err)
response_body = json.loads(response.read())
url = response_body['resources'][0]['href']
request = urllib2.Request(url)
request.add_header('Authorization', 'Basic %s' % admin_auth)
request.add_header('X-Requested-By', 'ambari')
body = [{"LDAP":{"synced_groups":"*","synced_users":"*"}}]
request.add_data(json.dumps(body))
request.get_method = lambda: 'GET'
request_in_progress = True
while request_in_progress:
sys.stdout.write('.')
sys.stdout.flush()
try:
response = urllib2.urlopen(request)
except Exception as e:
request_in_progress = False
err = 'Sync event check failed. Error details: %s' % e
raise FatalException(1, err)
response_status_code = response.getcode()
if response_status_code != 200:
err = 'Error during syncing. Http status code - ' + str(response_status_code)
raise FatalException(1, err)
response_body = json.loads(response.read())
sync_info = response_body['Event']
if sync_info['status'] == 'ERROR':
raise FatalException(1, str(sync_info['status_detail']))
elif sync_info['status'] == 'COMPLETE':
print '\n\nCompleted LDAP Sync.'
print 'Summary:'
for principal_type, summary in sync_info['summary'].iteritems():
print ' {0}:'.format(principal_type)
for action, amount in summary.iteritems():
print ' {0} = {1!s}'.format(action, amount)
request_in_progress = False
else:
time.sleep(1)
sys.stdout.write('\n')
sys.stdout.flush()
def flume(action = None):
import params
if action == 'config':
# remove previously defined meta's
for n in find_expected_agent_names(params.flume_conf_dir):
File(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'),
action = "delete",
)
Directory(params.flume_run_dir,
)
Directory(params.flume_conf_dir,
create_parents = True,
owner=params.flume_user,
)
Directory(params.flume_log_dir,
owner=params.flume_user,
cd_access="a",
mode=0755,
)
flume_agents = {}
if params.flume_conf_content is not None:
flume_agents = build_flume_topology(params.flume_conf_content)
for agent in flume_agents.keys():
flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent)
flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf')
flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json')
flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties')
flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.sh')
Directory(flume_agent_conf_dir,
owner=params.flume_user,
)
PropertiesFile(flume_agent_conf_file,
properties=flume_agents[agent],
owner=params.flume_user,
mode = 0644)
File(flume_agent_log4j_file,
content=Template('log4j.properties.j2', agent_name = agent),
owner=params.flume_user,
mode = 0644)
File(flume_agent_meta_file,
content = json.dumps(ambari_meta(agent, flume_agents[agent])),
owner=params.flume_user,
mode = 0644)
File(flume_agent_env_file,
owner=params.flume_user,
content=InlineTemplate(params.flume_env_sh_template)
)
if params.has_metric_collector:
File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"),
owner=params.flume_user,
content=Template("flume-metrics2.properties.j2")
)
elif action == 'start':
# desired state for service should be STARTED
if len(params.flume_command_targets) == 0:
_set_desired_state('STARTED')
# It is important to run this command as a background process.
flume_base = as_user(format("{flume_bin} agent --name {{0}} --conf {{1}} --conf-file {{2}} {{3}} > {flume_log_dir}/{{4}}.out 2>&1"), params.flume_user, env={'JAVA_HOME': params.java_home}) + " &"
for agent in cmd_target_names():
flume_agent_conf_dir = params.flume_conf_dir + os.sep + agent
flume_agent_conf_file = flume_agent_conf_dir + os.sep + "flume.conf"
flume_agent_pid_file = params.flume_run_dir + os.sep + agent + ".pid"
if not os.path.isfile(flume_agent_conf_file):
continue
if not is_flume_process_live(flume_agent_pid_file):
# TODO someday make the ganglia ports configurable
extra_args = ''
if params.ganglia_server_host is not None:
extra_args = '-Dflume.monitoring.type=ganglia -Dflume.monitoring.hosts={0}:{1}'
extra_args = extra_args.format(params.ganglia_server_host, '8655')
if params.has_metric_collector:
extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \
'-Dflume.monitoring.node={0}:{1}'
extra_args = extra_args.format(params.metric_collector_host, params.metric_collector_port)
flume_cmd = flume_base.format(agent, flume_agent_conf_dir,
flume_agent_conf_file, extra_args, agent)
Execute(flume_cmd,
wait_for_finish=False,
environment={'JAVA_HOME': params.java_home}
)
# sometimes startup spawns a couple of threads - so only the first line may count
pid_cmd = as_sudo(('pgrep', '-o', '-u', params.flume_user, '-f', format('^{java_home}.*{agent}.*'))) + \
" | " + as_sudo(('tee', flume_agent_pid_file)) + " && test ${PIPESTATUS[0]} -eq 0"
Execute(pid_cmd,
logoutput=True,
tries=20,
try_sleep=10)
pass
elif action == 'stop':
# desired state for service should be INSTALLED
if len(params.flume_command_targets) == 0:
_set_desired_state('INSTALLED')
pid_files = glob.glob(params.flume_run_dir + os.sep + "*.pid")
if 0 == len(pid_files):
return
agent_names = cmd_target_names()
for agent in agent_names:
pid_file = format("{flume_run_dir}/{agent}.pid")
if is_flume_process_live(pid_file):
pid = shell.checked_call(("cat", pid_file), sudo=True)[1].strip()
Execute(("kill", "-15", pid), sudo=True) # kill command has to be a tuple
if not await_flume_process_termination(pid_file):
raise Fail("Can't stop flume agent: {0}".format(agent))
File(pid_file, action = 'delete')
ranger_hive_url = format("{hive_url}/default;principal={hive_principal}") if security_enabled else hive_url
if stack_supports_ranger_hive_jdbc_url_change:
ranger_hive_url = format("jdbc:hive2://{hive_zookeeper_quorum}/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace={hive_server2_zookeeper_namespace}")
hive_ranger_plugin_config = {
'username': repo_config_username,
'password': repo_config_password,
'jdbc.driverClassName': jdbc_driver_class_name,
'jdbc.url': ranger_hive_url,
'commonNameForCertificate': common_name_for_certificate
}
hive_ranger_plugin_repo = {
'isActive': 'true',
'config': json.dumps(hive_ranger_plugin_config),
'description': 'hive repo',
'name': repo_name,
'repositoryType': 'hive',
'assetType': '3'
}
custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties)
if len(custom_ranger_service_config) > 0:
hive_ranger_plugin_config.update(custom_ranger_service_config)
if stack_supports_ranger_kerberos and security_enabled:
hive_ranger_plugin_config['policy.download.auth.users'] = hive_user
hive_ranger_plugin_config['tag.download.auth.users'] = hive_user
hive_ranger_plugin_config['policy.grantrevoke.auth.users'] = hive_user
def create_ambari_admin_user(self, ambari_admin_username, ambari_admin_password, usernamepassword):
"""
:param ambari_admin_username: username of user to be created
:param ambari_admin_password: user password of user to be created
:param usernamepassword: user credentials using which repository needs to be searched.
:return: Returns user credentials if user exist otherwise rerutns credentials of created user.
"""
flag_ambari_admin_present = False
match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password)
if match is None:
raise Fail('Invalid password given for Ranger Admin user for Ambari')
try:
url = self.url_users + '?name=' + str(ambari_admin_username)
request = urllib2.Request(url)
base_64_string = base64.encodestring(usernamepassword).replace('\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization", "Basic {0}".format(base_64_string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXUsers']) >= 0:
for vxuser in response['vXUsers']:
if vxuser['name'] == ambari_admin_username:
flag_ambari_admin_present = True
break
else:
flag_ambari_admin_present = False
if flag_ambari_admin_present:
Logger.info(ambari_admin_username + ' user already exists.')
return response_code
else:
Logger.info(ambari_admin_username + ' user is not present, creating user using given configurations')
url = self.url_sec_users
admin_user = dict()
admin_user['status'] = 1
admin_user['userRoleList'] = ['ROLE_SYS_ADMIN']
admin_user['name'] = ambari_admin_username
admin_user['password'] = ambari_admin_password
admin_user['description'] = ambari_admin_username
admin_user['firstName'] = ambari_admin_username
data = json.dumps(admin_user)
base_64_string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(url, data, headers)
request.add_header("Authorization", "Basic {0}".format(base_64_string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200 and response is not None:
Logger.info('Ambari admin user creation successful.')
return response_code
else:
Logger.error('Ambari admin user creation failed.')
return None
else:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail("Error creating ambari admin user. Http status code - {0}. \n {1}".format(e.code, e.read()))
else:
raise Fail("Error creating ambari admin user. Reason - {0}.".format(e.reason))
hbase_ranger_plugin_config = {
'username': repo_config_username,
'password': repo_config_password,
'hadoop.security.authentication': hadoop_security_authentication,
'hbase.security.authentication': hbase_security_authentication,
'hbase.zookeeper.property.clientPort': hbase_zookeeper_property_clientPort,
'hbase.zookeeper.quorum': hbase_zookeeper_quorum,
'zookeeper.znode.parent': zookeeper_znode_parent,
'commonNameForCertificate': common_name_for_certificate,
'hbase.master.kerberos.principal': master_jaas_princ if security_enabled else ''
}
hbase_ranger_plugin_repo = {
'isActive': 'true',
'config': json.dumps(hbase_ranger_plugin_config),
'description': 'hbase repo',
'name': repo_name,
'repositoryType': 'hbase',
'assetType': '2'
}
ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
xa_audit_db_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None
xa_audit_hdfs_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
ssl_keystore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
ssl_truststore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}")
driver_curl_target = format("/usr/hdp/current/knox-server/ext/{jdbc_jar_name}")
knox_ranger_plugin_config = {
'username': repo_config_username,
'password': repo_config_password,
'knox.url': format("https://{knox_host_name}:{knox_host_port}/gateway/admin/api/v1/topologies"),
'commonNameForCertificate': common_name_for_certificate
}
knox_ranger_plugin_repo = {
'isActive': 'true',
'config': json.dumps(knox_ranger_plugin_config),
'description': 'knox repo',
'name': repo_name,
'repositoryType': 'knox',
'assetType': '5',
}
ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
xa_audit_db_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None
xa_audit_hdfs_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
ssl_keystore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
ssl_truststore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
def create_ambari_admin_user(self, ambari_admin_username,
ambari_admin_password, usernamepassword):
"""
:param ambari_admin_username: username of user to be created
:param ambari_admin_password: user password of user to be created
:param usernamepassword: user credentials using which repository needs to be searched.
:return: Returns user credentials if user exist otherwise rerutns credentials of created user.
"""
flag_ambari_admin_present = False
match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password)
if match is None:
raise Fail(
'Invalid password given for Ranger Admin user for Ambari')
try:
url = self.url_users + '?name=' + str(ambari_admin_username)
request = urllib2.Request(url)
base_64_string = base64.encodestring(usernamepassword).replace(
'\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization",
"Basic {0}".format(base_64_string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXUsers']) >= 0:
for vxuser in response['vXUsers']:
if vxuser['name'] == ambari_admin_username:
flag_ambari_admin_present = True
break
else:
flag_ambari_admin_present = False
if flag_ambari_admin_present:
Logger.info(ambari_admin_username +
' user already exists.')
return response_code
else:
Logger.info(
ambari_admin_username +
' user is not present, creating user using given configurations'
)
url = self.url_sec_users
admin_user = dict()
admin_user['status'] = 1
admin_user['userRoleList'] = ['ROLE_SYS_ADMIN']
admin_user['name'] = ambari_admin_username
admin_user['password'] = ambari_admin_password
admin_user['description'] = ambari_admin_username
admin_user['firstName'] = ambari_admin_username
data = json.dumps(admin_user)
base_64_string = base64.encodestring(
'{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(url, data, headers)
request.add_header("Authorization",
"Basic {0}".format(base_64_string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(
result.read()))
if response_code == 200 and response is not None:
Logger.info('Ambari admin user creation successful.')
return response_code
else:
Logger.error('Ambari admin user creation failed.')
return None
else:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail(
"Error creating ambari admin user. Http status code - {0}. \n {1}"
.format(e.code, e.read()))
else:
raise Fail(
"Error creating ambari admin user. Reason - {0}.".format(
e.reason))
def execute_command(self, command):
'''
Executes commands of type EXECUTION_COMMAND
'''
clusterName = command['clusterName']
commandId = command['commandId']
isCommandBackground = command['commandType'] == self.BACKGROUND_EXECUTION_COMMAND
isAutoExecuteCommand = command['commandType'] == self.AUTO_EXECUTION_COMMAND
message = "Executing command with id = {commandId}, taskId = {taskId} for role = {role} of " \
"cluster {cluster}.".format(
commandId = str(commandId), taskId = str(command['taskId']),
role=command['role'], cluster=clusterName)
logger.info(message)
taskId = command['taskId']
# Preparing 'IN_PROGRESS' report
in_progress_status = self.commandStatuses.generate_report_template(command)
# The path of the files that contain the output log and error log use a prefix that the agent advertises to the
# server. The prefix is defined in agent-config.ini
if not isAutoExecuteCommand:
in_progress_status.update({
'tmpout': self.tmpdir + os.sep + 'output-' + str(taskId) + '.txt',
'tmperr': self.tmpdir + os.sep + 'errors-' + str(taskId) + '.txt',
'structuredOut' : self.tmpdir + os.sep + 'structured-out-' + str(taskId) + '.json',
'status': self.IN_PROGRESS_STATUS
})
else:
in_progress_status.update({
'tmpout': self.tmpdir + os.sep + 'auto_output-' + str(taskId) + '.txt',
'tmperr': self.tmpdir + os.sep + 'auto_errors-' + str(taskId) + '.txt',
'structuredOut' : self.tmpdir + os.sep + 'auto_structured-out-' + str(taskId) + '.json',
'status': self.IN_PROGRESS_STATUS
})
self.commandStatuses.put_command_status(command, in_progress_status)
numAttempts = 0
retryDuration = 0 # even with 0 allow one attempt
retryAble = False
delay = 1
log_command_output = True
if 'commandParams' in command and 'log_output' in command['commandParams'] and "false" == command['commandParams']['log_output']:
log_command_output = False
if 'commandParams' in command:
if 'max_duration_for_retries' in command['commandParams']:
retryDuration = int(command['commandParams']['max_duration_for_retries'])
if 'command_retry_enabled' in command['commandParams']:
retryAble = command['commandParams']['command_retry_enabled'] == "true"
if isAutoExecuteCommand:
retryAble = False
logger.info("Command execution metadata - taskId = {taskId}, retry enabled = {retryAble}, max retry duration (sec) = {retryDuration}, log_output = {log_command_output}".
format(taskId=taskId, retryAble=retryAble, retryDuration=retryDuration, log_command_output=log_command_output))
while retryDuration >= 0:
numAttempts += 1
start = 0
if retryAble:
start = int(time.time())
# running command
commandresult = self.customServiceOrchestrator.runCommand(command,
in_progress_status['tmpout'],
in_progress_status['tmperr'],
override_output_files=numAttempts == 1,
retry=numAttempts > 1)
end = 1
if retryAble:
end = int(time.time())
retryDuration -= (end - start)
# dumping results
if isCommandBackground:
logger.info("Command is background command, quit retrying. Exit code: {exitCode}, retryAble: {retryAble}, retryDuration (sec): {retryDuration}, last delay (sec): {delay}"
.format(cid=taskId, exitCode=commandresult['exitcode'], retryAble=retryAble, retryDuration=retryDuration, delay=delay))
return
else:
if commandresult['exitcode'] == 0:
status = self.COMPLETED_STATUS
else:
status = self.FAILED_STATUS
if (commandresult['exitcode'] == -signal.SIGTERM) or (commandresult['exitcode'] == -signal.SIGKILL):
logger.info('Command {cid} was canceled!'.format(cid=taskId))
break
if status != self.COMPLETED_STATUS and retryAble and retryDuration > 0:
delay = self.get_retry_delay(delay)
if delay > retryDuration:
delay = retryDuration
retryDuration -= delay # allow one last attempt
commandresult['stderr'] += "\n\nCommand failed. Retrying command execution ...\n\n"
logger.info("Retrying command id {cid} after a wait of {delay}".format(cid=taskId, delay=delay))
time.sleep(delay)
continue
else:
logger.info("Quit retrying for command id {cid}. Status: {status}, retryAble: {retryAble}, retryDuration (sec): {retryDuration}, last delay (sec): {delay}"
.format(cid=taskId, status=status, retryAble=retryAble, retryDuration=retryDuration, delay=delay))
break
# final result to stdout
commandresult['stdout'] += '\n\nCommand completed successfully!\n' if status == self.COMPLETED_STATUS else '\n\nCommand failed after ' + str(numAttempts) + ' tries\n'
logger.info('Command {cid} completed successfully!'.format(cid=taskId) if status == self.COMPLETED_STATUS else 'Command {cid} failed after {attempts} tries'.format(cid=taskId, attempts=numAttempts))
roleResult = self.commandStatuses.generate_report_template(command)
roleResult.update({
'stdout': commandresult['stdout'],
'stderr': commandresult['stderr'],
'exitCode': commandresult['exitcode'],
'status': status,
})
if self.config.has_option("logging","log_command_executes") \
and int(self.config.get("logging", "log_command_executes")) == 1 \
and log_command_output:
if roleResult['stdout'] != '':
logger.info("Begin command output log for command with id = " + str(command['taskId']) + ", role = "
+ command['role'] + ", roleCommand = " + command['roleCommand'])
self.log_command_output(roleResult['stdout'], str(command['taskId']))
logger.info("End command output log for command with id = " + str(command['taskId']) + ", role = "
+ command['role'] + ", roleCommand = " + command['roleCommand'])
if roleResult['stderr'] != '':
logger.info("Begin command stderr log for command with id = " + str(command['taskId']) + ", role = "
+ command['role'] + ", roleCommand = " + command['roleCommand'])
self.log_command_output(roleResult['stderr'], str(command['taskId']))
logger.info("End command stderr log for command with id = " + str(command['taskId']) + ", role = "
+ command['role'] + ", roleCommand = " + command['roleCommand'])
if roleResult['stdout'] == '':
roleResult['stdout'] = 'None'
if roleResult['stderr'] == '':
roleResult['stderr'] = 'None'
# let ambari know name of custom command
if command['hostLevelParams'].has_key('custom_command'):
roleResult['customCommand'] = command['hostLevelParams']['custom_command']
if 'structuredOut' in commandresult:
roleResult['structuredOut'] = str(json.dumps(commandresult['structuredOut']))
else:
roleResult['structuredOut'] = ''
# let recovery manager know the current state
if status == self.COMPLETED_STATUS:
if self.controller.recovery_manager.enabled() and command.has_key('roleCommand') \
and self.controller.recovery_manager.configured_for_recovery(command['role']):
if command['roleCommand'] == self.ROLE_COMMAND_START:
self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.LIVE_STATUS)
self.controller.recovery_manager.update_config_staleness(command['role'], False)
logger.info("After EXECUTION_COMMAND (START), with taskId=" + str(command['taskId']) +
", current state of " + command['role'] + " to " +
self.controller.recovery_manager.get_current_status(command['role']) )
elif command['roleCommand'] == self.ROLE_COMMAND_STOP or command['roleCommand'] == self.ROLE_COMMAND_INSTALL:
self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.DEAD_STATUS)
logger.info("After EXECUTION_COMMAND (STOP/INSTALL), with taskId=" + str(command['taskId']) +
", current state of " + command['role'] + " to " +
self.controller.recovery_manager.get_current_status(command['role']) )
elif command['roleCommand'] == self.ROLE_COMMAND_CUSTOM_COMMAND:
if command['hostLevelParams'].has_key('custom_command') and \
command['hostLevelParams']['custom_command'] == self.CUSTOM_COMMAND_RESTART:
self.controller.recovery_manager.update_current_status(command['role'], LiveStatus.LIVE_STATUS)
self.controller.recovery_manager.update_config_staleness(command['role'], False)
logger.info("After EXECUTION_COMMAND (RESTART), current state of " + command['role'] + " to " +
self.controller.recovery_manager.get_current_status(command['role']) )
pass
# let ambari know that configuration tags were applied
configHandler = ActualConfigHandler(self.config, self.configTags)
if command.has_key('configurationTags'):
configHandler.write_actual(command['configurationTags'])
roleResult['configurationTags'] = command['configurationTags']
component = {'serviceName':command['serviceName'],'componentName':command['role']}
if 'roleCommand' in command and \
(command['roleCommand'] == self.ROLE_COMMAND_START or
(command['roleCommand'] == self.ROLE_COMMAND_INSTALL and component in LiveStatus.CLIENT_COMPONENTS) or
(command['roleCommand'] == self.ROLE_COMMAND_CUSTOM_COMMAND and
'custom_command' in command['hostLevelParams'] and
command['hostLevelParams']['custom_command'] == self.CUSTOM_COMMAND_RESTART)):
configHandler.write_actual_component(command['role'],
command['configurationTags'])
if 'clientsToUpdateConfigs' in command['hostLevelParams'] and command['hostLevelParams']['clientsToUpdateConfigs']:
configHandler.write_client_components(command['serviceName'],
command['configurationTags'],
command['hostLevelParams']['clientsToUpdateConfigs'])
roleResult['configurationTags'] = configHandler.read_actual_component(
command['role'])
elif status == self.FAILED_STATUS:
if self.controller.recovery_manager.enabled() and command.has_key('roleCommand') \
and self.controller.recovery_manager.configured_for_recovery(command['role']):
if command['roleCommand'] == self.ROLE_COMMAND_INSTALL:
self.controller.recovery_manager.update_current_status(command['role'], self.controller.recovery_manager.INSTALL_FAILED)
logger.info("After EXECUTION_COMMAND (INSTALL), with taskId=" + str(command['taskId']) +
", current state of " + command['role'] + " to " +
self.controller.recovery_manager.get_current_status(command['role']))
self.commandStatuses.put_command_status(command, roleResult)
def create_ranger_repository(self, component, repo_name, repo_properties,
ambari_ranger_admin, ambari_ranger_password,
admin_uname, admin_password, policy_user, is_security_enabled = False, is_stack_supports_ranger_kerberos = False,
component_user = None, component_user_principal = None, component_user_keytab = None):
if not is_stack_supports_ranger_kerberos or not is_security_enabled:
response_code = self.check_ranger_login_urllib2(self.base_url)
repo_data = json.dumps(repo_properties)
ambari_ranger_password = unicode(ambari_ranger_password)
admin_password = unicode(admin_password)
ambari_username_password_for_ranger = format('{ambari_ranger_admin}:{ambari_ranger_password}')
if response_code is not None and response_code == 200:
user_resp_code = self.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}"))
if user_resp_code is not None and user_resp_code == 200:
retryCount = 0
while retryCount <= 5:
repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger)
if repo is not None:
Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name']))
break
else:
response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger)
if response is not None:
Logger.info('{0} Repository created in Ranger admin'.format(component.title()))
break
else:
if retryCount < 5:
Logger.info("Retry Repository Creation is being called")
time.sleep(30) # delay for 30 seconds
retryCount += 1
else:
Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title()))
break
else:
Logger.error('Ambari admin user creation failed')
elif not self.skip_if_rangeradmin_down:
Logger.error("Connection failed to Ranger Admin !")
elif is_stack_supports_ranger_kerberos and is_security_enabled:
response = self.check_ranger_login_curl(component_user,component_user_keytab,component_user_principal,self.url_login,True)
if response and response[0] == 200:
retryCount = 0
repo_data = json.dumps(repo_properties)
while retryCount <= 5:
response = self.get_repository_by_name_curl(component_user,component_user_keytab,component_user_principal,repo_name, component, 'true')
if response is not None:
Logger.info('{0} Repository {1} exist'.format(component.title(), (response['name'])))
break
else:
response = self.create_repository_curl(component_user,component_user_keytab,component_user_principal,repo_name, repo_data,policy_user)
if response and len(response) > 0:
Logger.info('{0} Repository created in Ranger admin'.format(component.title()))
break
else:
if retryCount < 5:
time.sleep(30) # delay for 30 seconds
retryCount += 1
else:
Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title()))
break
else:
Logger.error("Connection failed to Ranger Admin !")
'username': repo_config_username,
'password': repo_config_password,
'hadoop.security.authentication': hadoop_security_authentication,
'hadoop.security.authorization': hadoop_security_authorization,
'fs.default.name': fs_default_name,
'hadoop.security.auth_to_local': hadoop_security_auth_to_local,
'hadoop.rpc.protection': hadoop_rpc_protection,
'commonNameForCertificate': common_name_for_certificate,
'dfs.datanode.kerberos.principal': dn_principal_name if security_enabled else '',
'dfs.namenode.kerberos.principal': nn_principal_name if security_enabled else '',
'dfs.secondary.namenode.kerberos.principal': sn_principal_name if security_enabled else ''
}
hdfs_ranger_plugin_repo = {
'isActive': 'true',
'config': json.dumps(hdfs_ranger_plugin_config),
'description': 'hdfs repo',
'name': repo_name,
'repositoryType': 'hdfs',
'assetType': '1'
}
ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
xa_audit_db_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None
xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
ssl_keystore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
ssl_truststore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}")
ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_symlink_name}")
ranger_driver_curl_target = format("{java_share_dir}/{ranger_jdbc_jar_name}")
hive_ranger_plugin_config = {
"username": repo_config_username,
"password": repo_config_password,
"jdbc.driverClassName": jdbc_driver_class_name,
"jdbc.url": format("{hive_url}/default;principal={hive_principal}") if security_enabled else hive_url,
"commonNameForCertificate": common_name_for_certificate,
}
hive_ranger_plugin_repo = {
"isActive": "true",
"config": json.dumps(hive_ranger_plugin_config),
"description": "hive repo",
"name": repo_name,
"repositoryType": "hive",
"assetType": "3",
}
xa_audit_db_password = unicode(config["configurations"]["admin-properties"]["audit_db_password"])
ranger_audit_solr_urls = config["configurations"]["ranger-admin-site"]["ranger.audit.solr.urls"]
xa_audit_db_is_enabled = (
config["configurations"]["ranger-hive-audit"]["xasecure.audit.destination.db"]
if xml_configurations_supported
else None
)
ssl_keystore_password = (
unicode(config["configurations"]["ranger-hive-policymgr-ssl"]["xasecure.policymgr.clientssl.keystore.password"])
def main(argv=None):
for service in SERVICES:
livestatus = LiveStatus('', service)
print json.dumps(livestatus.build())
downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}")
driver_curl_target = format("{storm_component_home_dir}/lib/{jdbc_jar_name}")
storm_ranger_plugin_config = {
'username': repo_config_username,
'password': repo_config_password,
'nimbus.url': 'http://' + storm_ui_host[0].lower() + ':' + str(storm_ui_port),
'commonNameForCertificate': common_name_for_certificate
}
storm_ranger_plugin_repo = {
'isActive': 'true',
'config': json.dumps(storm_ranger_plugin_config),
'description': 'storm repo',
'name': repo_name,
'repositoryType': 'storm',
'assetType': '6'
}
ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
xa_audit_db_is_enabled = config['configurations']['ranger-storm-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None
xa_audit_hdfs_is_enabled = config['configurations']['ranger-storm-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
ssl_keystore_password = unicode(config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
ssl_truststore_password = unicode(config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
def flume(action = None):
import params
if action == 'config':
# remove previously defined meta's
for n in find_expected_agent_names(params.flume_conf_dir):
File(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'),
action = "delete",
)
Directory(params.flume_run_dir,
)
Directory(params.flume_conf_dir,
recursive=True,
owner=params.flume_user,
)
Directory(params.flume_log_dir, owner=params.flume_user)
flume_agents = {}
if params.flume_conf_content is not None:
flume_agents = build_flume_topology(params.flume_conf_content)
for agent in flume_agents.keys():
flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent)
flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf')
flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json')
flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties')
flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.sh')
Directory(flume_agent_conf_dir,
owner=params.flume_user,
)
PropertiesFile(flume_agent_conf_file,
properties=flume_agents[agent],
owner=params.flume_user,
mode = 0644)
File(flume_agent_log4j_file,
content=Template('log4j.properties.j2', agent_name = agent),
owner=params.flume_user,
mode = 0644)
File(flume_agent_meta_file,
content = json.dumps(ambari_meta(agent, flume_agents[agent])),
owner=params.flume_user,
mode = 0644)
File(flume_agent_env_file,
owner=params.flume_user,
content=InlineTemplate(params.flume_env_sh_template)
)
if params.has_metric_collector:
File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"),
owner=params.flume_user,
content=Template("flume-metrics2.properties.j2")
)
elif action == 'start':
# desired state for service should be STARTED
if len(params.flume_command_targets) == 0:
_set_desired_state('STARTED')
# It is important to run this command as a background process.
flume_base = as_user(format("{flume_bin} agent --name {{0}} --conf {{1}} --conf-file {{2}} {{3}} > {flume_log_dir}/{{4}}.out 2>&1"), params.flume_user, env={'JAVA_HOME': params.java_home}) + " &"
for agent in cmd_target_names():
flume_agent_conf_dir = params.flume_conf_dir + os.sep + agent
flume_agent_conf_file = flume_agent_conf_dir + os.sep + "flume.conf"
flume_agent_pid_file = params.flume_run_dir + os.sep + agent + ".pid"
if not os.path.isfile(flume_agent_conf_file):
continue
if not is_flume_process_live(flume_agent_pid_file):
# TODO someday make the ganglia ports configurable
extra_args = ''
if params.ganglia_server_host is not None:
extra_args = '-Dflume.monitoring.type=ganglia -Dflume.monitoring.hosts={0}:{1}'
extra_args = extra_args.format(params.ganglia_server_host, '8655')
if params.has_metric_collector:
extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \
'-Dflume.monitoring.node={0}:{1}'
extra_args = extra_args.format(params.metric_collector_host, params.metric_collector_port)
flume_cmd = flume_base.format(agent, flume_agent_conf_dir,
flume_agent_conf_file, extra_args, agent)
Execute(flume_cmd,
wait_for_finish=False,
environment={'JAVA_HOME': params.java_home}
)
# sometimes startup spawns a couple of threads - so only the first line may count
pid_cmd = as_sudo(('pgrep', '-o', '-u', params.flume_user, '-f', format('^{java_home}.*{agent}.*'))) + \
" | " + as_sudo(('tee', flume_agent_pid_file)) + " && test ${PIPESTATUS[0]} -eq 0"
Execute(pid_cmd,
logoutput=True,
tries=20,
try_sleep=10)
pass
elif action == 'stop':
# desired state for service should be INSTALLED
if len(params.flume_command_targets) == 0:
_set_desired_state('INSTALLED')
pid_files = glob.glob(params.flume_run_dir + os.sep + "*.pid")
if 0 == len(pid_files):
return
agent_names = cmd_target_names()
for agent in agent_names:
pid_file = format("{flume_run_dir}/{agent}.pid")
if is_flume_process_live(pid_file):
pid = shell.checked_call(("cat", pid_file), sudo=True)[1].strip()
Execute(("kill", "-15", pid), sudo=True) # kill command has to be a tuple
if not await_flume_process_termination(pid_file):
raise Fail("Can't stop flume agent: {0}".format(agent))
File(pid_file, action = 'delete')
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
else:
raise Fail('Ranger service is not started on given host')
if ambari_ranger_admin != '' and ambari_ranger_password != '':
get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
else:
raise Fail('Ambari admin username and password not available')
current_datetime = datetime.now()
File(format('{kms_conf_dir}/ranger-security.xml'),
owner = params.kms_user,
group = params.kms_group,
mode = 0644,
content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
)
Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')],
owner = params.kms_user,
group = params.kms_group,
mode=0775,
recursive = True
)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')),
owner = params.kms_user,
group = params.kms_group,
mode = 0644
)
XmlConfig("ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig("ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.credential_file,
owner = params.kms_user,
group = params.kms_group,
mode = 0640
)
def create_ranger_repository(self, component, repo_name, repo_properties,
ambari_ranger_admin, ambari_ranger_password,
admin_uname, admin_password, policy_user,
is_security_enabled, component_user,
component_user_principal,
component_user_keytab):
if not is_security_enabled:
response_code = self.check_ranger_login_urllib2(self.base_url)
repo_data = json.dumps(repo_properties)
ambari_ranger_password = unicode(ambari_ranger_password)
admin_password = unicode(admin_password)
ambari_username_password_for_ranger = format(
'{ambari_ranger_admin}:{ambari_ranger_password}')
if response_code is not None and response_code == 200:
user_resp_code = self.create_ambari_admin_user(
ambari_ranger_admin, ambari_ranger_password,
format("{admin_uname}:{admin_password}"))
if user_resp_code is not None and user_resp_code == 200:
retryCount = 0
while retryCount <= 5:
repo = self.get_repository_by_name_urllib2(
repo_name, component, 'true',
ambari_username_password_for_ranger)
if repo is not None:
Logger.info('{0} Repository {1} exist'.format(
component.title(), repo['name']))
break
else:
response = self.create_repository_urllib2(
repo_data, ambari_username_password_for_ranger)
if response is not None:
Logger.info(
'{0} Repository created in Ranger admin'.
format(component.title()))
break
else:
if retryCount < 5:
Logger.info(
"Retry Repository Creation is being called"
)
time.sleep(30) # delay for 30 seconds
retryCount += 1
else:
Logger.error(
'{0} Repository creation failed in Ranger admin'
.format(component.title()))
break
else:
Logger.error('Ambari admin user creation failed')
elif not self.skip_if_rangeradmin_down:
Logger.error("Connection failed to Ranger Admin !")
else:
response = self.check_ranger_login_curl(component_user,
component_user_keytab,
component_user_principal,
self.base_url, True)
if response and response[0] == 200:
retryCount = 0
repo_data = json.dumps(repo_properties)
while retryCount <= 5:
response = self.get_repository_by_name_curl(
component_user, component_user_keytab,
component_user_principal, repo_name, component, 'true')
if response is not None:
Logger.info('{0} Repository {1} exist'.format(
component.title(), (response['name'])))
break
else:
response = self.create_repository_curl(
component_user, component_user_keytab,
component_user_principal, repo_name, repo_data,
policy_user)
if response and len(response) > 0:
Logger.info(
'{0} Repository created in Ranger admin'.
format(component.title()))
break
else:
if retryCount < 5:
time.sleep(30) # delay for 30 seconds
retryCount += 1
else:
Logger.error(
'{0} Repository creation failed in Ranger admin'
.format(component.title()))
break
else:
Logger.error("Connection failed to Ranger Admin !")
