def runAgent(passPhrase, expected_hostname, user_run_as, verbose, ret=None):
os.environ[AMBARI_PASSPHRASE_VAR] = passPhrase
vo = ""
if verbose:
vo = " -v"
cmd = [
'su', user_run_as, '-l', '-c',
'/usr/sbin/ambari-agent restart --expected-hostname=%1s %2s' %
(expected_hostname, vo)
]
log = ""
p = subprocess32.Popen(cmd, stdout=subprocess32.PIPE)
p.communicate()
agent_retcode = p.returncode
for i in range(3):
time.sleep(1)
ret = execOsCommand(
["tail", "-20", "/var/log/ambari-agent/ambari-agent.log"], ret=ret)
if (0 == ret['exitstatus']):
try:
log = ret['log']
except Exception:
log = "Log not found"
print log
break
return {"exitstatus": agent_retcode, "log": log}
def run(self, script, user=None):
import pwd
try:
if user is not None:
user = pwd.getpwnam(user)[2]
else:
user = os.getuid()
self._threadLocal.uid = user
except Exception:
_logger.warn("can not switch user for RUN_COMMAND.")
cmd = script
if isinstance(script, list):
cmd = " ".join(script)
cmd_list = ["/bin/bash", "--login", "--noprofile", "-c", cmd]
p = subprocess.Popen(cmd_list,
preexec_fn=self._change_uid,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
close_fds=True)
out, err = p.communicate()
code = p.wait()
_logger.debug("Exitcode for %s is %d" % (cmd, code))
return {'exitCode': code, 'output': out, 'error': err}
def swap_memory():
sin, sout = cext.swap_mem()
# XXX
# we are supposed to get total/free by doing so:
# http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/
# usr/src/cmd/swap/swap.c
# ...nevertheless I can't manage to obtain the same numbers as 'swap'
# cmdline utility, so let's parse its output (sigh!)
p = subprocess32.Popen(['swap', '-l', '-k'], stdout=subprocess32.PIPE)
stdout, stderr = p.communicate()
if PY3:
stdout = stdout.decode(sys.stdout.encoding)
if p.returncode != 0:
raise RuntimeError("'swap -l -k' failed (retcode=%s)" % p.returncode)
lines = stdout.strip().split('\n')[1:]
if not lines:
raise RuntimeError('no swap device(s) configured')
total = free = 0
for line in lines:
line = line.split()
t, f = line[-2:]
t = t.replace('K', '')
f = f.replace('K', '')
total += int(int(t) * 1024)
free += int(int(f) * 1024)
used = total - free
percent = usage_percent(used, total, _round=1)
return _common.sswap(total, used, free, percent, sin * PAGE_SIZE,
sout * PAGE_SIZE)
def _get_unix_sockets(self, pid):
"""Get UNIX sockets used by process by parsing 'pfiles' output."""
# TODO: rewrite this in C (...but the damn netstat source code
# does not include this part! Argh!!)
cmd = "pfiles %s" % pid
p = subprocess32.Popen(cmd,
shell=True,
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE)
stdout, stderr = p.communicate()
if PY3:
stdout, stderr = [
x.decode(sys.stdout.encoding) for x in (stdout, stderr)
]
if p.returncode != 0:
if 'permission denied' in stderr.lower():
raise AccessDenied(self.pid, self._name)
if 'no such process' in stderr.lower():
raise NoSuchProcess(self.pid, self._name)
raise RuntimeError("%r command error\n%s" % (cmd, stderr))
lines = stdout.split('\n')[2:]
for i, line in enumerate(lines):
line = line.lstrip()
if line.startswith('sockname: AF_UNIX'):
path = line.split(' ', 2)[2]
type = lines[i - 2].strip()
if type == 'SOCK_STREAM':
type = socket.SOCK_STREAM
elif type == 'SOCK_DGRAM':
type = socket.SOCK_DGRAM
else:
type = -1
yield (-1, socket.AF_UNIX, type, path, "", _common.CONN_NONE)
def genAgentCrtReq(self, keyname):
keysdir = os.path.abspath(self.config.get('security', 'keysdir'))
generate_script = GEN_AGENT_KEY % {
'hostname': hostname.hostname(self.config),
'keysdir': keysdir}
logger.info(generate_script)
if platform.system() == 'Windows':
p = subprocess32.Popen(generate_script, stdout=subprocess32.PIPE)
p.communicate()
else:
p = subprocess32.Popen([generate_script], shell=True,
stdout=subprocess32.PIPE)
p.communicate()
# this is required to be 600 for security concerns.
os.chmod(keyname, 0600)
def run(self):
scpcommand = [
"scp", "-r", "-o", "ConnectTimeout=60", "-o", "BatchMode=yes",
"-o", "StrictHostKeyChecking=no", "-P", self.sshPort, "-i",
self.sshkey_file, self.inputFile,
self.user + "@" + self.host + ":" + self.remote
]
if DEBUG:
self.host_log.write("Running scp command " + ' '.join(scpcommand))
self.host_log.write("==========================")
self.host_log.write("\nCommand start time " +
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
scpstat = subprocess32.Popen(scpcommand,
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE)
log = scpstat.communicate()
errorMsg = log[1]
log = log[0] + "\n" + log[1]
self.host_log.write(log)
self.host_log.write("scp " + self.inputFile)
self.host_log.write("host=" + self.host + ", exitcode=" +
str(scpstat.returncode))
self.host_log.write("Command end time " +
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
return {
"exitstatus": scpstat.returncode,
"log": log,
"errormsg": errorMsg
}
def run_os_command_in_shell(self, command):
process = subprocess32.Popen(command,
stdout=subprocess32.PIPE,
stdin=subprocess32.PIPE,
stderr=subprocess32.PIPE,
shell=True)
return process.communicate()
def os_call(command, logoutput=None, env={}):
shell = not isinstance(command, list)
print_output = logoutput == True or (logoutput == None
and Utils.verbose)
if not print_output:
stdout = subprocess32.PIPE
stderr = subprocess32.STDOUT
else:
stdout = stderr = None
logger.info("Running '{0}'".format(command))
proc = subprocess32.Popen(command,
shell=shell,
stdout=stdout,
stderr=stderr,
env=env)
if not print_output:
out = proc.communicate()[0].strip('\n')
else:
proc.wait()
out = None
code = proc.returncode
if code:
err_msg = ("Execution of '%s'\n returned %d. %s") % (command, code,
out)
raise OsCallFailure(err_msg)
return out
def main(argv=None):
scriptDir = os.path.realpath(os.path.dirname(argv[0]))
onlyargs = argv[1:]
if len(onlyargs) < 3:
sys.stderr.write(
"Usage: <comma separated hosts> "
"<tmpdir for storage> <user> <sshPort> <sshkey_file> <agent setup script>"
" <ambari-server name> <cluster os type> <ambari version> <ambari port> <user_run_as> <passwordFile>\n"
)
sys.exit(2)
pass
#Parse the input
hostList = onlyargs[0].split(",")
bootdir = onlyargs[1]
user = onlyargs[2]
sshPort = onlyargs[3]
sshkey_file = onlyargs[4]
setupAgentFile = onlyargs[5]
ambariServer = onlyargs[6]
cluster_os_type = onlyargs[7]
ambariVersion = onlyargs[8]
server_port = onlyargs[9]
user_run_as = onlyargs[10]
passwordFile = onlyargs[11]
if not OSCheck.is_windows_family():
# ssh doesn't like open files
subprocess32.Popen(["chmod", "600", sshkey_file],
stdout=subprocess32.PIPE)
if passwordFile is not None and passwordFile != 'null':
subprocess32.Popen(["chmod", "600", passwordFile],
stdout=subprocess32.PIPE)
logging.info("BootStrapping hosts " + pprint.pformat(hostList) +
" using " + scriptDir + " cluster primary OS: " + cluster_os_type +
" with user '" + user + "'with ssh Port '" + sshPort + "' sshKey File " + sshkey_file + " password File " + passwordFile +\
" using tmp dir " + bootdir + " ambari: " + ambariServer +"; server_port: " + server_port +\
"; ambari version: " + ambariVersion+"; user_run_as: " + user_run_as)
sharedState = SharedState(user, sshPort, sshkey_file, scriptDir, bootdir,
setupAgentFile, ambariServer, cluster_os_type,
ambariVersion, server_port, user_run_as,
passwordFile)
pbootstrap = PBootstrap(hostList, sharedState)
pbootstrap.run()
return 0 # Hack to comply with current usage
def run_os_command(cmd):
shell = (type(cmd) == str)
process = subprocess32.Popen(cmd,
shell=shell,
stdout=subprocess32.PIPE,
stdin=subprocess32.PIPE,
stderr=subprocess32.PIPE)
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
def get_alternatives_desc(self, alt_name):
command = ALT_DISP_CMD.format(alt_name)
out = None
try:
p1 = subprocess32.Popen(shlex.split(command),
stdout=subprocess32.PIPE)
p2 = subprocess32.Popen(["grep", "priority"],
stdin=p1.stdout,
stdout=subprocess32.PIPE)
p1.stdout.close()
out = p2.communicate()[0]
logger.debug('alternatives --display ' + alt_name + '\n, out = ' +
out)
except:
logger.warn('Cannot process alternative named: ' + alt_name + ',' + \
'error: ' + str(sys.exc_info()[0]))
return out
def launch_subprocess(command): """ Process launch helper :param command Command to execute :type command list[str]|str :return Popen object """ is_shell = not isinstance(command, (list, tuple)) return subprocess32.Popen(command, stdout=subprocess32.PIPE, stderr=subprocess32.PIPE, shell=is_shell, close_fds=True)
def run_os_command(self, cmd, runWithSudo=True):
if runWithSudo:
cmd = "/var/lib/ambari-agent/" + AMBARI_SUDO_BINARY + " " + cmd
logger.info('Executing command: ' + str(cmd))
if type(cmd) == str:
cmd = shlex.split(cmd)
process = subprocess32.Popen(cmd,
stdout=subprocess32.PIPE,
stdin=subprocess32.PIPE,
stderr=subprocess32.PIPE)
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
def launch_python_subprocess32(self, command, tmpout, tmperr):
"""
Creates subprocess32 with given parameters. This functionality was moved to separate method
to make possible unit testing
"""
command_env = dict(os.environ)
return subprocess32.Popen(command,
stdout=tmpout,
stderr=tmperr,
close_fds=True,
env=command_env,
preexec_fn=lambda: os.setpgid(0, 0))
def os_run_os_command(cmd, env=None, shell=False, cwd=None):
if type(cmd) == str:
cmd = shlex.split(cmd)
process = subprocess32.Popen(cmd,
stdout=subprocess32.PIPE,
stdin=subprocess32.PIPE,
stderr=subprocess32.PIPE,
env=env,
cwd=cwd,
shell=shell)
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
def run(self, script, user=None):
logger.warn("user argument ignored on windows")
code = 0
if isinstance(script, list):
cmd = " ".join(script)
else:
cmd = script
p = subprocess32.Popen(cmd, stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE, shell=False)
out, err = p.communicate()
code = p.wait()
logger.debug("Exitcode for %s is %d" % (cmd, code))
return {'exitCode': code, 'output': out, 'error': err}
def exec_ams_env_cmd(options):
ams_env_cmd = os.path.join(options.conf_dir, AMS_ENV_CMD)
if os.path.exists(ams_env_cmd):
cmds = ["cmd.exe", "/C", ams_env_cmd]
procAms = subprocess32.Popen(cmds, env=os.environ)
out, err = procAms.communicate()
if err is not None and err is not "":
print_warning_msg(AMS_ENV_CMD + " error output: " + err)
if out is not None and out is not "":
print_info_msg(AMS_ENV_CMD + " output: " + out)
else:
err = 'ERROR: Cannot execute "{0}"'.format(ams_env_cmd)
raise FatalException(1, err)
def run(self, script, user=None):
global _logger
_logger.warn("user argument ignored on windows")
code = 0
if isinstance(script, list):
cmd = " ".join(script)
else:
cmd = script
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
out, err = p.communicate()
code = p.wait()
_logger.debug("Exitcode for %s is %d" % (cmd, code))
return SubprocessCallResult(out=out, error=err, code=code)
def runPowershell(self, file=None, script_block=None, args=[]):
logger.warn("user argument ignored on windows")
code = 0
cmd = None
if file:
cmd = ['powershell', '-WindowStyle', 'Hidden', '-File', file] + args
elif script_block:
cmd = ['powershell', '-WindowStyle', 'Hidden', '-Command', script_block] + args
p = subprocess32.Popen(cmd, stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE, shell=False)
out, err = p.communicate()
code = p.wait()
logger.debug("Exitcode for %s is %d" % (cmd, code))
return _dict_to_object({'exitCode': code, 'output': out, 'error': err})
def os_run_os_command(cmd, env=None, shell=False, cwd=None):
print_info_msg('about to run command: ' + str(cmd))
if type(cmd) == str:
cmd = shlex.split(cmd)
process = subprocess32.Popen(cmd,
stdout=subprocess32.PIPE,
stdin=subprocess32.PIPE,
stderr=subprocess32.PIPE,
env=env,
cwd=cwd,
shell=shell)
print_info_msg("\nprocess_pid=" + str(process.pid))
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
def runPowershell(self, f=None, script_block=None, args=set()):
global _logger
_logger.warn("user argument ignored on windows")
cmd = None
if f:
cmd = ['powershell', '-WindowStyle', 'Hidden', '-File', f] + list(args)
elif script_block:
cmd = ['powershell', '-WindowStyle', 'Hidden', '-Command', script_block] + list(args)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
out, err = p.communicate()
code = p.wait()
_logger.debug("Exitcode for %s is %d" % (cmd, code))
return {'exitCode': code, 'output': out, 'error': err}
def exec_ssh_cmd(hostname, cmd):
"""
Runs the command on the remote host as gpadmin user
"""
# Only gpadmin should be allowed to run command via ssh, thus not exposing user as a parameter
cmd = "su - {0} -c \"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {1} \\\"{2} \\\" \"".format(
hawq_constants.hawq_user, hostname, cmd)
Logger.info("Command executed: {0}".format(cmd))
process = subprocess32.Popen(cmd,
stdout=subprocess32.PIPE,
stdin=subprocess32.PIPE,
stderr=subprocess32.PIPE,
shell=True)
(stdout, stderr) = process.communicate()
return process.returncode, stdout, stderr
def execOsCommand(osCommand, tries=1, try_sleep=0, ret=None, cwd=None):
ret = _ret_init(ret)
for i in range(0, tries):
if i > 0:
time.sleep(try_sleep)
osStat = subprocess32.Popen(osCommand,
stdout=subprocess32.PIPE,
cwd=cwd)
log = osStat.communicate(0)
ret = {"exitstatus": osStat.returncode, "log": log}
if ret['exitstatus'] == 0:
break
return ret
def public_hostname(config):
global cached_public_hostname
if cached_public_hostname is not None:
return cached_public_hostname
out = ''
err = ''
try:
if config.has_option('agent', 'public_hostname_script'):
scriptname = config.get('agent', 'public_hostname_script')
output = subprocess32.Popen(scriptname,
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
shell=True)
out, err = output.communicate()
if (0 == output.returncode and 0 != len(out.strip())):
cached_public_hostname = out.strip().lower()
logger.info("Read public hostname '" + cached_public_hostname +
"' using agent:public_hostname_script")
return cached_public_hostname
else:
logger.warn("Execution of '{0}' returned {1}. {2}\n{3}".format(
scriptname, output.returncode, err.strip(), out.strip()))
except:
#ignore for now.
trace_info = traceback.format_exc()
logger.info("Error using the scriptname:" + trace_info + " :out " +
out + " :err " + err)
logger.info("Defaulting to fqdn.")
try:
handle = urllib2.urlopen(
'http://169.254.169.254/latest/meta-data/public-hostname', '', 2)
str = handle.read()
handle.close()
cached_public_hostname = str.lower()
logger.info(
"Read public hostname '" + cached_public_hostname +
"' from http://169.254.169.254/latest/meta-data/public-hostname")
except:
cached_public_hostname = socket.getfqdn().lower()
logger.info("Read public hostname '" + cached_public_hostname +
"' using socket.getfqdn()")
return cached_public_hostname
def checkUsers(self, user_mask, results):
get_users_cmd = [
"powershell", '-noProfile', '-NonInteractive', '-nologo',
"-Command",
self.GET_USERS_CMD.format(user_mask)
]
try:
osStat = subprocess32.Popen(get_users_cmd,
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE)
out, err = osStat.communicate()
except:
raise Exception("Failed to get users.")
for user in out.split(os.linesep):
result = {}
result['name'] = user
result['homeDir'] = ""
result['status'] = "Available"
results.append(result)
def launch_python_subprocess32(self, command, tmpout, tmperr):
"""
Creates subprocess32 with given parameters. This functionality was moved to separate method
to make possible unit testing
"""
close_fds = None if OSCheck.get_os_family(
) == OSConst.WINSRV_FAMILY else True
command_env = dict(os.environ)
if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
command_env["PYTHONPATH"] = os.pathsep.join(sys.path)
for k, v in command_env.iteritems():
command_env[k] = str(v)
return subprocess32.Popen(command,
stdout=tmpout,
stderr=tmperr,
close_fds=close_fds,
env=command_env,
preexec_fn=self.preexec_fn)
def getResponse(path, address, ssl_enabled):
command = "curl"
httpGssnegotiate = "--negotiate"
userpswd = "-u:"
insecure = "-k"# This is smoke test, no need to check CA of server
if ssl_enabled:
url = 'https://' + address + path
else:
url = 'http://' + address + path
command_with_flags = [command,httpGssnegotiate,userpswd,insecure,url]
proc = subprocess32.Popen(command_with_flags, stdout=subprocess32.PIPE, stderr=subprocess32.PIPE)
(stdout, stderr) = proc.communicate()
response = json.loads(stdout)
if response == None:
print 'There is no response for url: ' + str(url)
raise Exception('There is no response for url: ' + str(url))
return response
def adjust_jce_permissions(self, jdk_path):
ambari_user = read_ambari_user()
cmds = []
if ambari_user:
cmds.append(self.SET_JCE_PERMISSIONS.format(ambari_user, jdk_path, configDefaults.JDK_SECURITY_DIR))
cmds.append(self.SET_JCE_FILE_MODE.format(jdk_path, configDefaults.JDK_SECURITY_DIR, "*"))
cmds.append(self.SET_JCE_JAR_MODE.format(jdk_path, configDefaults.JDK_SECURITY_DIR, "*.jar"))
cmd = " && ".join(cmds)
process = subprocess32.Popen(cmd,
stdout=subprocess32.PIPE,
stdin=subprocess32.PIPE,
stderr=subprocess32.PIPE,
shell=True
)
(stdoutdata, stderrdata) = process.communicate()
if process.returncode != 0:
print_warning_msg("Failed to change jce permissions. {0}\n{1}".format(stderrdata, stdoutdata))
def run(self):
sshcommand = [
"ssh",
"-o",
"ConnectTimeOut=60",
"-o",
"StrictHostKeyChecking=no",
"-o",
"BatchMode=yes",
"-tt", # Should prevent "tput: No value for $TERM and no -T specified" warning
"-i",
self.sshkey_file,
"-p",
self.sshPort,
self.user + "@" + self.host,
self.command
]
if DEBUG:
self.host_log.write("Running ssh command " + ' '.join(sshcommand))
self.host_log.write("==========================")
self.host_log.write("\nCommand start time " +
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
sshstat = subprocess32.Popen(sshcommand,
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE)
log = sshstat.communicate()
errorMsg = log[1]
if self.errorMessage and sshstat.returncode != 0:
errorMsg = self.errorMessage + "\n" + errorMsg
log = log[0] + "\n" + errorMsg
self.host_log.write(log)
self.host_log.write("SSH command execution finished")
self.host_log.write("host=" + self.host + ", exitcode=" +
str(sshstat.returncode))
self.host_log.write("Command end time " +
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
return {
"exitstatus": sshstat.returncode,
"log": log,
"errormsg": errorMsg
}
def hostname(config):
global cached_hostname
if cached_hostname is not None:
return cached_hostname
try:
scriptname = config.get('agent', 'hostname_script')
try:
osStat = subprocess32.Popen([scriptname],
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE)
out, err = osStat.communicate()
if (0 == osStat.returncode and 0 != len(out.strip())):
cached_hostname = out.strip()
logger.info(
"Read hostname '{0}' using agent:hostname_script '{1}'".
format(cached_hostname, scriptname))
else:
logger.warn(
"Execution of '{0}' failed with exit code {1}. err='{2}'\nout='{3}'"
.format(scriptname, osStat.returncode, err.strip(),
out.strip()))
cached_hostname = socket.getfqdn()
logger.info(
"Read hostname '{0}' using socket.getfqdn() as '{1}' failed"
.format(cached_hostname, scriptname))
except:
cached_hostname = socket.getfqdn()
logger.warn(
"Unexpected error while retrieving hostname: '{0}', defaulting to socket.getfqdn()"
.format(sys.exc_info()))
logger.info("Read hostname '{0}' using socket.getfqdn().".format(
cached_hostname))
except:
cached_hostname = socket.getfqdn()
logger.info(
"agent:hostname_script configuration not defined thus read hostname '{0}' using socket.getfqdn()."
.format(cached_hostname))
cached_hostname = cached_hostname.lower()
return cached_hostname