def get_cred(HostNm, body):
"""Get the credentials from the message in the form of a (user, pwd) tuple.
If there is no ssh object present, or keyBased is present and True, return
(User, passphrase, key_file) block."""
try:
if ((not body.has_key('ssh') or util.get_val(body['ssh'], 'keyBased', False)) and
(not is_host_local(HostNm))):
# It's key-based, implement new logic. {keyBased: true, key: "", key_user: "", key_passp: "", key_file: ""};
_logger.warning('get_cred, new logic 1.')
return (True, body['ssh']['key_user'], body['ssh']['key_passp'],
body['ssh']['key_file'])
except KeyError:
_logger.warning('get_cred, KeyError handler.')
if ((not body.has_key('ssh') or util.get_val(body['ssh'], 'keyBased', False)) and
(not is_host_local(HostNm))):
# It's key-based, implement new logic. {keyBased: true, key: "", key_user: "", key_passp: "", key_file: ""};
_logger.warning('get_cred, new logic 2.')
return (True, body['ssh']['key_user'], body['ssh']['key_passp'],
body['ssh']['key_file'])
_logger.warning('get_cred, old(ish) code.')
if (is_host_local(HostNm)):
return (False, "", "", None)
else:
return (False, body['ssh']['user'], body['ssh']['pwd'], None)
def get_cred(body):
"""Get the credentials from the message in the form of a (user, pwd) tuple.
If there is no ssh object present, or keyBased is present and True, a
(None, None) tuple is returned."""
if not body.has_key('ssh') or util.get_val(body['ssh'], 'keyBased', False):
return (None, None)
return (body['ssh']['user'], body['ssh']['pwd'])
def get_cred(body):
"""Get the credentials from the message in the form of a (user, pwd) tuple.
If there is no ssh object present, or keyBased is present and True, a
(None, None) tuple is returned."""
if not body.has_key('ssh') or util.get_val(body['ssh'], 'keyBased', False):
return (None, None)
return (body['ssh']['user'], body['ssh']['pwd'])
def _exec_cmdv(self, cmdv, procCtrl, stdinFile):
"""Execute an OS command on the local host, using subprocess module.
cmdv - complete command vector (argv) of the OS command
procCtrl - procCtrl object from message which controls how the process
is started (blocking vs non-blocking and output reporting)
"""
# Add nohup when running locally to prevent server from
# waiting on the children
if util.get_val(procCtrl, 'nohup'):
cmdv[:0] = ['nohup']
output = tempfile.TemporaryFile()
stdin = output
if (stdinFile != None):
stdin = self.open(stdinFile)
try:
if util.get_val(procCtrl, 'waitForCompletion'):
try:
subprocess.check_call(cmdv,
stdin=stdin,
stdout=output,
stderr=output)
except subprocess.CalledProcessError as cpe:
if cpe.returncode != util.get_val(procCtrl, 'noRaise', 0):
output.seek(0)
_logger.error('output=' + output.read())
raise
output.seek(0)
return output.read()
proc = subprocess.Popen(cmdv, stdin=stdin, stderr=output)
if proc.poll() == None and procCtrl.has_key('daemonWait'):
_logger.debug('Popen waits {0} sec for {1}'.format(
procCtrl['daemonWait'], ' '.join(cmdv)))
time.sleep(procCtrl['daemonWait'])
if proc.poll() != None:
output.seek(0)
raise ExecException(' '.join(cmdv), proc.returncode, output)
finally:
#output.seek(0)
#print output.read()
if (stdin != output):
stdin.close()
output.close()
def _exec_cmdln(self, cmdln, procCtrl, stdinFile):
"""Execute an OS command line (as a single string) on the remote host.
cmdln - complete command line of the OS command
procCtrl - procCtrl object from message which controls how the process
is started (blocking vs non-blocking and output reporting)
"""
contents = None
if (stdinFile != None):
with self.open(stdinFile) as stdin:
contents = stdin.read()
with contextlib.closing(
self.client.get_transport().open_session()) as chan:
chan.set_combine_stderr(True)
_logger.debug('cmdln=' + cmdln)
chan.exec_command(cmdln)
if (contents != None):
_logger.debug('Using supplied stdin from ' + stdinFile + ': ')
_logger.debug(contents[0:50] + '...')
chan.sendall(contents)
chan.shutdown_write()
if util.get_val(procCtrl, 'waitForCompletion'):
output = chan.makefile('rb')
_logger.debug('Waiting for command...')
exitstatus = chan.recv_exit_status()
if exitstatus != 0 and exitstatus != util.get_val(
procCtrl, 'noRaise'):
raise RemoteExecException(self.host, cmdln, exitstatus,
output)
return output.read()
else:
if not chan.exit_status_ready() and procCtrl.has_key(
'daemonWait'):
_logger.debug('Waiting {0} sec for {1}'.format(
procCtrl['daemonWait'], cmdln))
time.sleep(procCtrl['daemonWait'])
if chan.exit_status_ready():
output = chan.makefile('rb')
raise RemoteExecException(self.host, cmdln,
chan.recv_exit_status(), output)
def is_local_sshd_available():
with contextlib.closing(SSHClient()) as c:
c.set_missing_host_key_policy(WarningPolicy())
#c.load_system_host_keys()
try:
c.connect(local_ipv4_ssh_addr(), password=util.get_val(os.environ, 'SSH_PWD'))
except:
logging.exception('No usable sshd on this machine: ')
return False
else:
return True
def is_local_sshd_available():
with contextlib.closing(SSHClient()) as c:
c.set_missing_host_key_policy(WarningPolicy())
#c.load_system_host_keys()
try:
c.connect(local_ipv4_ssh_addr(), password=util.get_val(os.environ, 'SSH_PWD'))
except:
logging.exception('No usable sshd on this machine: ')
return False
else:
return True
def get_cred(HostNm, body):
"""Get the credentials from the message in the form of a (user, pwd) tuple.
If there is no ssh object present, or keyBased is present and True, return
(User, passphrase, key_file) block."""
try:
if ((not body.has_key('ssh')
or util.get_val(body['ssh'], 'keyBased', False))
and (HostNm != 'localhost' and HostNm != '127.0.0.1')):
# It's key-based, implement new logic. {keyBased: true, key: "", key_user: "", key_passp: "", key_file: ""};
return (True, body['ssh']['key_user'], body['ssh']['key_passp'],
body['ssh']['key_file'])
except KeyError:
if ((not body.has_key('ssh')
or util.get_val(body['ssh'], 'keyBased', False))
and (HostNm != 'localhost' and HostNm != '127.0.0.1')):
# It's key-based, implement new logic. {keyBased: true, key: "", key_user: "", key_passp: "", key_file: ""};
return (True, body['ssh']['key_user'], body['ssh']['key_passp'],
body['ssh']['key_file'])
return (False, body['ssh']['user'], body['ssh']['pwd'], None)
def _exec_cmdv(self, cmdv, procCtrl, stdinFile):
"""Execute an OS command on the local host, using subprocess module.
cmdv - complete command vector (argv) of the OS command
procCtrl - procCtrl object from message which controls how the process
is started (blocking vs non-blocking and output reporting)
"""
# Add nohup when running locally to prevent server from
# waiting on the children
if util.get_val(procCtrl, 'nohup'):
cmdv[:0] = ['nohup']
output = tempfile.TemporaryFile()
stdin = output
if (stdinFile != None):
stdin = self.open(stdinFile)
try:
if util.get_val(procCtrl, 'waitForCompletion'):
try:
subprocess.check_call(cmdv, stdin=stdin, stdout=output, stderr=output)
except subprocess.CalledProcessError as cpe:
if cpe.returncode != util.get_val(procCtrl, 'noRaise', 0):
output.seek(0)
_logger.error('output='+output.read())
raise
output.seek(0)
return output.read()
proc = subprocess.Popen(cmdv, stdin=stdin, stderr=output)
if proc.poll() == None and procCtrl.has_key('daemonWait'):
_logger.debug('Popen waits {0} sec for {1}'.format(procCtrl['daemonWait'], ' '.join(cmdv)))
time.sleep(procCtrl['daemonWait'])
if proc.poll() != None:
output.seek(0)
raise ExecException(' '.join(cmdv), proc.returncode, output)
finally:
#output.seek(0)
#print output.read()
if (stdin != output):
stdin.close()
output.close()
def _exec_cmdln(self, cmdln, procCtrl, stdinFile):
"""Execute an OS command line (as a single string) on the remote host.
cmdln - complete command line of the OS command
procCtrl - procCtrl object from message which controls how the process
is started (blocking vs non-blocking and output reporting)
"""
contents = None
if (stdinFile != None):
with self.open(stdinFile) as stdin:
contents = stdin.read()
with contextlib.closing(self.client.get_transport().open_session()) as chan:
chan.set_combine_stderr(True)
_logger.debug('cmdln='+cmdln)
chan.exec_command(cmdln)
if (contents != None):
_logger.debug('Using supplied stdin from ' + stdinFile + ': ')
_logger.debug(contents[0:50] + '...')
chan.sendall(contents)
chan.shutdown_write()
if util.get_val(procCtrl, 'waitForCompletion'):
output = chan.makefile('rb')
_logger.debug('Waiting for command...')
exitstatus = chan.recv_exit_status()
if exitstatus != 0 and exitstatus != util.get_val(procCtrl, 'noRaise'):
raise RemoteExecException(self.host, cmdln, exitstatus, output)
return output.read()
else:
if not chan.exit_status_ready() and procCtrl.has_key('daemonWait'):
_logger.debug('Waiting {0} sec for {1}'.format(procCtrl['daemonWait'], cmdln))
time.sleep(procCtrl['daemonWait'])
if chan.exit_status_ready():
output = chan.makefile('rb')
raise RemoteExecException(self.host, cmdln, chan.recv_exit_status(), output)
def get_request(self):
"""Override get_request from SocketServer.TCPServer so that we can wrap
the socket in an ssl socket when using ssl."""
sock,addr = SocketServer.TCPServer.get_request(self)
if util.get_val(self.opts, 'ssl', False):
if self.opts['ca_certs']:
cert_reqs = ssl.CERT_REQUIRED
else:
cert_reqs = ssl.CERT_NONE
ssl_sock = ssl.wrap_socket(sock, certfile=self.opts['certfile'], keyfile=self.opts['keyfile'],
cert_reqs=cert_reqs, ca_certs=self.opts['ca_certs'], server_side=True)
#self.logger.debug('ssl_sock.getpeercert()='+str(ssl_sock.getpeercert())
return ssl_sock, addr
return sock, addr
def get_request(self):
"""Override get_request from SocketServer.TCPServer so that we can wrap
the socket in an ssl socket when using ssl."""
sock,addr = SocketServer.TCPServer.get_request(self)
if util.get_val(self.opts, 'ssl', False):
if self.opts['ca_certs']:
cert_reqs = ssl.CERT_REQUIRED
else:
cert_reqs = ssl.CERT_NONE
ssl_sock = ssl.wrap_socket(sock, certfile=self.opts['certfile'], keyfile=self.opts['keyfile'],
cert_reqs=cert_reqs, ca_certs=self.opts['ca_certs'], server_side=True)
#self.logger.debug('ssl_sock.getpeercert()='+str(ssl_sock.getpeercert())
return ssl_sock, addr
return sock, addr
def handle_shutdownServerReq(req, body):
"""x"""
if body.has_key('deathkey') and body['deathkey'] == deathkey:
raise ShutdownException("Shutdown request received")
time.sleep(util.get_val(body, 'sleeptime', 0))
return make_rep(req, 'incorrect death key')
def _exec_cmdln(self, cmdln, procCtrl, stdInFile):
"""Execute an OS command line (as a single string) on the remote host.
cmdln - complete command line of the OS command
procCtrl - procCtrl object from message which controls how the process
is started (blocking vs non-blocking and output reporting)
"""
_logger.debug('\nRCH--> remote_ch, exec cmdln %s', cmdln)
check_connected(self.client)
contents = None
if stdInFile != None:
with self.open(stdInFile) as stdin:
contents = stdin.read()
with contextlib.closing(self.client.get_transport().open_session()) as chan:
chan.set_combine_stderr(True)
_logger.debug('\nRCH--> cmdln=%s', cmdln)
chan.exec_command(cmdln)
if contents != None:
_logger.debug('\nRCH--> Using supplied stdin from %s:', stdInFile)
_logger.debug('\nRCH--> %s ...', contents[0:50])
chan.sendall(contents)
chan.shutdown_write()
if util.get_val(procCtrl, 'waitForCompletion'):
_logger.debug("RCH--> Has waitForCompletion")
output = chan.makefile('rb')
if procCtrl is not None and procCtrl.has_key('daemonWait'):
procDW = procCtrl['daemonWait']
_logger.warning(bcolors.WARNING + '\nRCH--> Waiting for command %s seconds.'
+ bcolors.ENDC, procDW)
time.sleep(procDW)
for t in xrange(180): #Wait MAX 180s for remote execution to return.
time.sleep(0.5)
if chan.exit_status_ready():
_logger.warning(bcolors.WARNING + '\nRCH--> Waited %s sec (on top '
'of %s sec) for %s' + bcolors.ENDC, t/2, procDW, cmdln)
break
if chan.exit_status_ready():
exitstatus = chan.recv_exit_status()
if exitstatus != 0 and exitstatus != util.get_val(procCtrl, 'noRaise'):
raise RemoteExecException(self.host, cmdln, exitstatus, output)
out_ret = output.read()
if exitstatus != 0:
out_ret += ' errcode:' + str(exitstatus)
return out_ret
else:
#Timeout. RemoteExecException will not work as it assumes command completed.
return 'Command `{0}\', running on {1} timed out.'.format(cmdln, self.host)
else:
# Block.
_logger.debug('\nRCH--> Waiting for %s...', cmdln)
exitstatus = chan.recv_exit_status()
if exitstatus != 0 and exitstatus != util.get_val(procCtrl, 'noRaise'):
raise RemoteExecException(self.host, cmdln, exitstatus, output)
out_ret = output.read()
if exitstatus != 0:
out_ret += ' errcode:' + str(exitstatus)
return out_ret
else:
_logger.debug("RCH--> NO waitForCompletion")
dbg_msg = '\nRCH--> chan.exit_status_ready(1) %s'
_logger.debug(dbg_msg, str(chan.exit_status_ready()))
if not chan.exit_status_ready() and procCtrl is not None \
and procCtrl.has_key('daemonWait'):
procDW = procCtrl['daemonWait']
#Let's not waste user's time:
_logger.debug('\nRCH--> Waiting for command %s seconds.', procDW)
time.sleep(procCtrl['daemonWait'])
for t in xrange(180):
time.sleep(0.5)
if chan.exit_status_ready():
_logger.warning(bcolors.WARNING + '\nRCH--> Waited %s sec for %s'
+ bcolors.ENDC, t/2, cmdln)
break
_logger.debug('\nRCH--> chan.exit_status_ready(2) %s',
str(chan.exit_status_ready()))
if chan.exit_status_ready():
output = chan.makefile('rb')
#Strange to return with exception without checking...
exitstatus = chan.recv_exit_status()
# noRaise = 1 exclusively for installing Windows service.
if exitstatus != 0 and exitstatus != util.get_val(procCtrl, 'noRaise'):
raise RemoteExecException(self.host, cmdln, chan.recv_exit_status(), output)
else:
_logger.debug('\nRCH--> remote_ch, exec cmdln, returning')
out_ret = output.read()
if exitstatus != 0:
out_ret += ' errcode:' + str(exitstatus)
return out_ret
else:
# this is for commands that we collect status for differently, say mysqld start
return 'Command `{0}\', running on {1} timed out.'.format(cmdln, self.host)
def mock_ABClusterHost(hle):
return produce_ABClusterHost(hostname=hle['hostInfoRep']['host']['name'],
user=util.get_val(hle, 'username'),
pwd=util.get_val(hle, 'password'))
def mock_ABClusterHost(hle):
return produce_ABClusterHost(hostname=hle['hostInfoRep']['host']['name'], user=util.get_val(hle, 'username'), pwd=util.get_val(hle, 'password'))
def setUp(self):
self.ch = RemoteClusterHost(local_ipv4_ssh_addr(),
password=util.get_val(
os.environ, 'SSH_PWD'))
_TestABClusterHost.setUp(self)
np.random.shuffle(data)
data = data[:100]
for record in data:
t = int(record[13]) # 0 => Healthy
# 1-4 => Has heart disease
if t > 4:
print "Error"
exit()
if t > 0:
t = 1
for attr in simple_attr + ranged_attr:
if abs(float(record[attr]) + 9.0) < 0.01:
continue
val = util.get_val(record, attr)
util.attr_sum[t][attr] += 1
o_cnt[attr][val][t] += 1
c_cnt[t] += 1
total = 0
correct = 0
slack = 0
data = util.read_data(sys.argv[2])
for record in data:
p = [1. * c_cnt[0] / (c_cnt[0] + c_cnt[1]),
1. * c_cnt[1] / (c_cnt[0] + c_cnt[1])]
t = int(record[13])
if t > 4:
print "Error"
def handle_shutdownServerReq(req, body):
"""x"""
if body.has_key('deathkey') and body['deathkey'] == deathkey:
raise ShutdownException("Shutdown request received")
time.sleep(util.get_val(body, 'sleeptime', 0))
return make_rep(req, 'incorrect death key')
def setUp(self):
self.ch = RemoteClusterHost(local_ipv4_ssh_addr(), password=util.get_val(os.environ, 'SSH_PWD'))
_TestABClusterHost.setUp(self)
def _exec_cmdv(self, cmdv, procCtrl, stdin_file):
"""Execute an OS command on the local host, using subprocess module.
cmdv - complete command vector (argv) of the OS command
procCtrl - procCtrl object from message which controls how the process
is started (blocking vs non-blocking and output reporting)
"""
# Add nohup when running locally to prevent server from waiting on the children
if util.get_val(procCtrl, 'nohup'):
cmdv[:0] = ['nohup']
stdin = None
if stdin_file != None:
stdin = self.open(stdin_file)
#We need Popen to have all possible FE commands to work. Using single mechanism to run all
# commands adds to readability. I am hoping to reduce number of exec_cmd functions too :-/
# Several possible cases:
# 1) !waitForCompletion and !daemonWait: Just fire and forget.
# 2) waitForCompletion and !daemonWait: Wait for result.
# 3) waitForCompletion and daemonWait: Wait for result but poll after daemonWait seconds.
# 4) !waitForCompletion and daemonWait: Wait for result only daemonWait seconds.
# Old code was partial 2) and 3)+4).
#New code:
# Wait for daemonWait AND MAX 120 seconds for command to return but check if it finished
# every second (after waiting daemonWait).
#Rationale: We have commands that can not return immediately even on localhost. For such
# commands we set daemonWait member of procControl to appropriate amount of seconds (say 2)
# forcing this exec routine to always wait that long for response. After that, we allow
# maximum of 120s for command to complete before we kill it. If command does not have
# procControl.daemonWait this means we expect entire process to finish as soon as command
# returns. For such commands, we wait max 120s.
proc_DW_tot = 120
proc_DW = 0
if procCtrl.has_key('daemonWait'):
proc_DW = util.get_val(procCtrl, 'daemonWait')
tst = ""
if isinstance(cmdv, list):
tst = ' '.join(cmdv)
else:
tst = str(cmdv)
_logger.debug("CHS--> daemonWait is %s", proc_DW)
_logger.debug("CHS--> cmdv %s", tst)
# hasKill = procCtrl.has_key('kill'). Commands with Kill member have different processing.
kill_it = True
try:
if isinstance(cmdv, list):
proc = subprocess.Popen(cmdv,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
else:
proc = subprocess.Popen(shlex.split(cmdv),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# There are commands we have to wait for to start with, such as NET START/STOP etc.
_logger.debug("CHS--> proc %s executed", tst)
if proc_DW > 0:
_logger.debug("CHS--> proc %s executed, sleeping", tst)
time.sleep(proc_DW)
#We have waited set amount of time, now see if it returns in 2min and if not, kill it.
for t in xrange(proc_DW_tot):
time.sleep(0.5)
if proc.poll() is not None:
kill_it = False
_logger.debug("CHS--> %ss, breaking for %s", t / 2, tst)
if proc_DW > 0:
dbg_msg = "CHS--> Waited %ssec (on top of %ssec) for %s to complete."
_logger.debug(dbg_msg, t / 2, proc_DW, tst)
else:
_logger.debug(
"CHS--> Waited %s seconds for %s to complete.",
t / 2, tst)
break
if kill_it:
_logger.warning(
bcolors.WARNING + "CHS--> killing %s" + bcolors.ENDC, tst)
proc.kill()
retcode = proc.returncode
if retcode is None:
retcode = 1
rd = proc.stdout.read()
wrn_msg = "CHS--> killing, raising %s rc %s, msg %s"
_logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC, tst,
retcode, rd)
raise ExecException(tst, retcode, proc.stdout)
else:
retcode = proc.returncode
rd = proc.stdout.read()
if retcode is None: #REALLY BAD except if it's meant to be so, such as with KILL -9
retcode = retcode or -1
rd = rd or "No output from command."
wrn_msg = "CHS--> killing, raising %s rc %s, msg %s"
_logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC,
tst, retcode, rd)
raise ExecException(tst, proc.returncode, proc.stdout)
elif retcode == 0 or (retcode != 0 and \
retcode == util.get_val(procCtrl, 'noRaise')):
_logger.debug("CHS--> Conditional succ. noRise:%s.",
retcode or -1)
# When successful, cut it to command and 100 characters of response. To remove
# just empty lines and not lines with spaces:
#"".join([s for s in t.strip().splitlines(True) if s.strip("\r\n")])
rd = rd or "No output from command." # rd can be None, for example for KILL -9
_logger.debug("CHS--> Succ. %s(%s), %s...", tst, retcode, \
"".join([t for t in rd.strip().splitlines(True) if t.strip()])[0:100])
return rd
elif retcode <= 125:
wrn_msg = "CHS--> %s failed, exit-code=%d error = %s"
_logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC,
tst, retcode, rd)
return 'errcode:' + str(retcode) + '\n' + rd
elif retcode == 127:
wrn_msg = bcolors.WARNING + "CHS--> %s, program not found: %s" + bcolors.ENDC
_logger.warning(wrn_msg, tst, rd)
return 'errcode:' + str(retcode) + '\n' + rd
else:
# Things get hairy and unportable - different shells return
# different values for coredumps, signals, etc.
rd = rd or "No output from command." # rd can be None
_logger.warning(
bcolors.WARNING +
"CHS--> %s, program reported OS-dependant "
"exit-code=%d error = %s" + bcolors.ENDC, tst, retcode,
rd)
return 'errcode:' + str(retcode) + '\n' + rd
finally:
if stdin is not None:
stdin.close()
def tst_tempify(*fs):
td = util.get_val(os.environ, 'MYBLOCKCHAIN_TMP_DIR',
tempfile.gettempdir())
#td = tempfile.mkdtemp()
return os.path.join(td, *fs)
def tst_tempify(*fs): td = util.get_val(os.environ, 'MYSQL_TMP_DIR', tempfile.gettempdir()) #td = tempfile.mkdtemp() return os.path.join(td, *fs)
