def run():
parser = get_parser()
options = parser.parse_args()
log_level = logging.WARNING
if options.verbose:
log_level = logging.DEBUG
baron = Baron()
if options.start or options.stop or options.restart:
pid_file = os.path.join(options.repository, "daemon.pid")
baron.ensure_pid_path(pid_file, options.user or "")
if options.restart:
baron.stop_daemon(pid_file)
if options.stop:
baron.stop_daemon(pid_file)
return
pid = baron.start_daemon(pid_file, working_dir=options.repository)
if os.geteuid() == 0:
baron.fork()
if options.user:
baron.set_owner(options.user)
node = LocalNode(options.repository, log_level=log_level, baron=baron, address=options.address)
node.serve()
def test_node_wait(self):
local = LocalNode()
self.assertTrue(
local.wait_for_port(host='127.0.0.1', attempts=2, timeout=5))
#self.assertFalse(
with self.assertRaises(Exception):
local.wait_for_port(host='192.0.2.1', attempts=2, timeout=5)
class RemoteNodeInterface(IObserver):
"""Interface from a local workSlave to the networked workMaster"""
def __init__(self):
IObserver.__init__(self)
self.client = ClientWrapper()
self.node = LocalNode()
self.node.registerObserver(self)
self.logger = Logger()
def setupClient(self,host='localhost',port=55555):
self.client.setupClient(host,port)
self.client.registerObserver(self)
def update(self,data):
"""
Receive a message from the network
or send results back to the host
"""
copy = data.split(':',1)
if copy[0] == 'Cmd':
cmd = "self.node." + copy[1]
self.log('Signal','Running command %s' % cmd,'update')
exec cmd
elif copy[0] == 'NodeId':
self.log('Signal','Setting nodeId to %s' % repr(copy[1]),'update')
self.node.nodeId = copy[1]
elif copy[0] == 'Results':
self.log('Signal','Sending back results %s' % repr(copy[1]),'update')
self.client.sendMessage(data)
elif copy[0] == 'Bench':
self.log('Signal','Sending back benches %s' % repr(copy[1]),'update')
self.client.sendMessage(data)
elif copy[0] == 'Work':
if copy[1] == 'Done':
self.log('Signal','Sending back signal of work done','update')
self.client.sendMessage(data)
def log(self,level,data,method=''):
self.logger.log(level,data,method,'WorkSlave')
def last_nonempty_line(filepath):
"""Return last non empty and not auxillary conman line
filepath: path to file for finding last line
Function is using some magic string which are defined
in conman.
"""
tail = LocalNode()
tail.shell('tail -n 100 ' + filepath, stop=True, quiet=True, die=False)
for str in reversed(tail.stdout.rstrip().splitlines()):
if str == '' or \
str.startswith('<ConMan> Console') or \
re.search(r'\d\d\d\d-\d\d-\d\d\s+\d\d:\d\d:\d\d\s*$', str):
continue
else:
return str
return 'no_meaningful_line_found'
def exec_bmc_command(node, ipmi_command):
cmd = get_ipmi_cmd(node['bmc_ip'], ipmi_command)
local = LocalNode()
try:
logging.debug('Execute command: ' + cmd)
local.shell(cmd, trace=True)
except:
BMCException(sys.exc_info()[1])
logging.debug('Status:' + str(local.status))
logging.debug('Output:' + local.stdout.rstrip())
logging.debug('Errors:' + local.stderr)
if local.status == 0:
logging.debug("Command '%s' executed successfully" % cmd)
# TODO check retrun results from results
else:
logging.debug("Command '%s' failed, raise exception" % cmd)
raise BMCException(
"ipmitool call failed with status[{}], stdout [{}], stderr[{}]".
format(local.status, local.stdout.rstrip(), local.stderr))
def minimal_needed_configuration(node, timeout=60, extra_sls=[]):
full_sls = sls_list + extra_sls
logging.debug('Executing salt script[{}]'.format(full_sls))
for sls in full_sls:
local = LocalNode()
local.pwd()
try:
local.shell(get_salt_cmd(sls, get_provision_ip(node)))
except:
raise Exception('Salt execution failed: ' + sys.exc_info()[1])
finally:
logging.debug('Salt Status:' + str(local.status))
logging.debug('Satl Output:' + local.stdout.rstrip())
logging.debug('Salt Errors:' + local.stderr)
logging.debug('Executed salt script[{}]'.format(full_sls))
def test_node(self):
local = LocalNode()
remote = RemoteNode('127.0.0.1',
username=os.environ.get('USER'),
identity=os.environ.get('HOME') + '/.ssh/id_rsa')
for i in [remote, local]:
i.pwd()
i.ls('notexisting', die=False)
print('Status:', i.status)
print('Output:', i.stdout.rstrip())
print('Errors:', i.stderr)
for i in [local, remote]:
i.shell('hostname',
'false',
'date',
'true',
stop=True,
quiet=True,
die=False)
print('Status:', i.status)
print('Output:', i.stdout.rstrip())
print('Errors:', i.stderr)
def __init__(self):
IObserver.__init__(self)
self.client = ClientWrapper()
self.node = LocalNode()
self.node.registerObserver(self)
self.logger = Logger()
def test_node_wait(self):
local = LocalNode()
self.assertTrue(
local.wait_for_node(host='127.0.0.1', attempts=2, timeout=5))
self.assertFalse(
local.wait_for_node(host='192.0.2.1', attempts=2, timeout=5))
def wait_node_is_ready(node,
timeout=900,
conman_line_max_age=None,
max_cold_restart=3,
port_lookup=22,
port_lookup_timeout=None,
port_lookup_attempts=None):
""" Return true if node(and ssh) start in time
Raise exceptions in other cases
Parameters:
timeout: overall timeout for boot and start ssh
(ssh starts after end of kiwi provisioning)
conman_line_max_age: how long last conman line
coud be not changed (mean node stuck)
max_cold_restart: maximum nuber of cold restarts
"""
if port_lookup_timeout is None:
port_lookup_timeout = default_port_lookup_timeout
if port_lookup_attempts is None:
port_lookup_attempts = default_port_lookup_attempts
if conman_line_max_age is None:
conman_line_max_age = default_conman_line_max_age
starttime = time.time()
conman_line = 'start_line'
conman_line_time = time.time()
conmanfile = conman_log_prefix + node['node'].split('.')[0]
cold_restart_count = 0
while starttime + timeout > time.time():
# check last line in conman log
new_conman_line = last_nonempty_line(conmanfile)
if conman_line != new_conman_line:
# logging.debug("New log detected:"+new_conman_line)
conman_line = new_conman_line
conman_line_time = time.time()
if (time.time() - conman_line_time) > conman_line_max_age:
logging.info("Node boot failure detected, make cold restart")
exec_bmc_command(node, 'power off')
time.sleep(default_cold_restart_timeout)
exec_bmc_command(node, 'power on')
if cold_restart_count >= max_cold_restart:
logging.error('Achieved max cold restart couter ' +
'%s for node %s,throws exception' %
(max_cold_restart, node['node']))
raise CannotBootException(
"max cold restart couter (%s) for %s" %
(max_cold_restart, node['node']))
cold_restart_count += 1
conman_line_time = time.time()
next
# check port status
try:
logging.debug("timeout=" + str(port_lookup_timeout))
local = LocalNode()
local.wait_for_port(host=get_provision_ip(node),
port=port_lookup,
timeout=port_lookup_timeout,
attempts=port_lookup_attempts)
logging.debug("Connected to node %s " % node['node'])
return True
except StopThread as st:
raise StopThread
except Exception as es:
logging.debug("Node {} have not started in timeout {}".format(
get_provision_ip(node), port_lookup_timeout))
raise TimeoutException("{} have not started in timeout {}".format(
get_provision_ip(node), timeout))