def rebalancehdfs(self, env):
from ambari_commons.os_windows import UserHelper, run_os_command_impersonated
import params
env.set_params(params)
hdfs_username, hdfs_domain = UserHelper.parse_user_name(
params.hdfs_user, ".")
name_node_parameters = json.loads(params.name_node_params)
threshold = name_node_parameters['threshold']
_print("Starting balancer with threshold = %s\n" % threshold)
def calculateCompletePercent(first, current):
return 1.0 - current.bytesLeftToMove / first.bytesLeftToMove
def startRebalancingProcess(threshold):
rebalanceCommand = 'hdfs balancer -threshold %s' % threshold
return ['cmd', '/C', rebalanceCommand]
command = startRebalancingProcess(threshold)
basedir = os.path.join(env.config.basedir, 'scripts')
_print("Executing command %s\n" % command)
parser = hdfs_rebalance.HdfsParser()
returncode, stdout, err = run_os_command_impersonated(
' '.join(command), hdfs_username,
Script.get_password(params.hdfs_user), hdfs_domain)
for line in stdout.split('\n'):
_print('[balancer] %s %s' % (str(datetime.now()), line))
pl = parser.parseLine(line)
if pl:
res = pl.toJson()
res['completePercent'] = calculateCompletePercent(
parser.initialLine, pl)
self.put_structured_out(res)
elif parser.state == 'PROCESS_FINISED':
_print('[balancer] %s %s' %
(str(datetime.now()), 'Process is finished'))
self.put_structured_out({'completePercent': 1})
break
if returncode != None and returncode != 0:
raise Fail(
'Hdfs rebalance process exited with error. See the log output')
def rebalancehdfs(self, env):
from ambari_commons.os_windows import UserHelper, run_os_command_impersonated
import params
env.set_params(params)
hdfs_username, hdfs_domain = UserHelper.parse_user_name(params.hdfs_user, ".")
name_node_parameters = json.loads( params.name_node_params )
threshold = name_node_parameters['threshold']
_print("Starting balancer with threshold = %s\n" % threshold)
def calculateCompletePercent(first, current):
return 1.0 - current.bytesLeftToMove/first.bytesLeftToMove
def startRebalancingProcess(threshold):
rebalanceCommand = 'hdfs balancer -threshold %s' % threshold
return ['cmd', '/C', rebalanceCommand]
command = startRebalancingProcess(threshold)
basedir = os.path.join(env.config.basedir, 'scripts')
_print("Executing command %s\n" % command)
parser = hdfs_rebalance.HdfsParser()
returncode, stdout, err = run_os_command_impersonated(' '.join(command), hdfs_username, Script.get_password(params.hdfs_user), hdfs_domain)
for line in stdout.split('\n'):
_print('[balancer] %s %s' % (str(datetime.now()), line ))
pl = parser.parseLine(line)
if pl:
res = pl.toJson()
res['completePercent'] = calculateCompletePercent(parser.initialLine, pl)
self.put_structured_out(res)
elif parser.state == 'PROCESS_FINISED' :
_print('[balancer] %s %s' % (str(datetime.now()), 'Process is finished' ))
self.put_structured_out({'completePercent' : 1})
break
if returncode != None and returncode != 0:
raise Fail('Hdfs rebalance process exited with error. See the log output')
def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, user=None):
# TODO implement timeout, wait_for_finish
Logger.info("Executing %s" % (command))
if user:
domain, username = UserHelper.parse_user_name(user, ".")
proc_token = OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES)
old_states = []
privileges = [
SE_ASSIGNPRIMARYTOKEN_NAME,
SE_INCREASE_QUOTA_NAME,
]
for priv in privileges:
old_states.append(QueryPrivilegeState(proc_token, priv))
AdjustPrivilege(proc_token, priv)
QueryPrivilegeState(proc_token, priv)
user_token = LogonUser(username, domain, Script.get_password(user), win32con.LOGON32_LOGON_SERVICE,
win32con.LOGON32_PROVIDER_DEFAULT)
env_token = DuplicateTokenEx(user_token, SecurityIdentification, TOKEN_QUERY, TokenPrimary)
# getting updated environment for impersonated user and merge it with custom env
current_env = CreateEnvironmentBlock(env_token, False)
current_env = _merge_env(current_env, env)
si = STARTUPINFO()
out_handle, err_handle, out_file, err_file = _create_tmp_files(current_env)
ok, si.hStdInput = _safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE))
if not ok:
raise Exception("Unable to create StdInput for child process")
ok, si.hStdOutput = _safe_duplicate_handle(out_handle)
if not ok:
raise Exception("Unable to create StdOut for child process")
ok, si.hStdError = _safe_duplicate_handle(err_handle)
if not ok:
raise Exception("Unable to create StdErr for child process")
Logger.debug("Redirecting stdout to '{0}', stderr to '{1}'".format(out_file.name, err_file.name))
si.dwFlags = win32con.STARTF_USESTDHANDLES
si.lpDesktop = ""
try:
info = CreateProcessAsUser(user_token, None, command, None, None, 1, win32con.CREATE_NO_WINDOW, current_env, cwd, si)
hProcess, hThread, dwProcessId, dwThreadId = info
hThread.Close()
try:
WaitForSingleObject(hProcess, INFINITE)
except KeyboardInterrupt:
pass
out, err = _get_files_output(out_file, err_file)
code = GetExitCodeProcess(hProcess)
finally:
for priv in privileges:
old_state = old_states.pop(0)
AdjustPrivilege(proc_token, priv, old_state)
else:
# getting updated environment for current process and merge it with custom env
cur_token = OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY)
current_env = CreateEnvironmentBlock(cur_token, False)
current_env = _merge_env(current_env, env)
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=cwd, env=current_env, shell=False)
out, err = proc.communicate()
code = proc.returncode
if logoutput and out:
Logger.info(out)
if logoutput and err:
Logger.info(err)
return code, out, err
def _call_command(command,
logoutput=False,
cwd=None,
env=None,
wait_for_finish=True,
timeout=None,
user=None):
# TODO implement timeout, wait_for_finish
Logger.info("Executing %s" % (command))
if user:
domain, username = UserHelper.parse_user_name(user, ".")
proc_token = OpenProcessToken(GetCurrentProcess(),
TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES)
old_states = []
privileges = [
SE_ASSIGNPRIMARYTOKEN_NAME,
SE_INCREASE_QUOTA_NAME,
]
for priv in privileges:
old_states.append(QueryPrivilegeState(proc_token, priv))
AdjustPrivilege(proc_token, priv)
QueryPrivilegeState(proc_token, priv)
user_token = LogonUser(username, domain, Script.get_password(user),
win32con.LOGON32_LOGON_SERVICE,
win32con.LOGON32_PROVIDER_DEFAULT)
env_token = DuplicateTokenEx(user_token, SecurityIdentification,
TOKEN_QUERY, TokenPrimary)
# getting updated environment for impersonated user and merge it with custom env
current_env = CreateEnvironmentBlock(env_token, False)
current_env = _merge_env(current_env, env)
si = STARTUPINFO()
out_handle, err_handle, out_file, err_file = _create_tmp_files(
current_env)
ok, si.hStdInput = _safe_duplicate_handle(
GetStdHandle(STD_INPUT_HANDLE))
if not ok:
raise Exception("Unable to create StdInput for child process")
ok, si.hStdOutput = _safe_duplicate_handle(out_handle)
if not ok:
raise Exception("Unable to create StdOut for child process")
ok, si.hStdError = _safe_duplicate_handle(err_handle)
if not ok:
raise Exception("Unable to create StdErr for child process")
Logger.debug("Redirecting stdout to '{0}', stderr to '{1}'".format(
out_file.name, err_file.name))
si.dwFlags = win32con.STARTF_USESTDHANDLES
si.lpDesktop = ""
try:
info = CreateProcessAsUser(user_token, None, command, None, None,
1, win32con.CREATE_NO_WINDOW,
current_env, cwd, si)
hProcess, hThread, dwProcessId, dwThreadId = info
hThread.Close()
try:
WaitForSingleObject(hProcess, INFINITE)
except KeyboardInterrupt:
pass
out, err = _get_files_output(out_file, err_file)
code = GetExitCodeProcess(hProcess)
finally:
for priv in privileges:
old_state = old_states.pop(0)
AdjustPrivilege(proc_token, priv, old_state)
else:
# getting updated environment for current process and merge it with custom env
cur_token = OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY)
current_env = CreateEnvironmentBlock(cur_token, False)
current_env = _merge_env(current_env, env)
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=cwd,
env=current_env,
shell=False)
out, err = proc.communicate()
code = proc.returncode
if logoutput and out:
Logger.info(out)
if logoutput and err:
Logger.info(err)
return code, out, err
