def stopDevice(device):
deviceIP = getIPAddress(device)
devicePath = os.path.join(options.bbpath, device)
errorFile = os.path.join(devicePath, 'error.flg')
log.info('%s: %s - stopping all processes' % (device, deviceIP))
stopProcess(os.path.join(devicePath, 'remotereftest.pid'), 'remotereftest')
stopProcess(os.path.join(devicePath, 'runtestsremote.pid'),
'runtestsremote')
stopProcess(os.path.join(devicePath, 'remotereftest.pid.xpcshell.pid'),
'xpcshell')
stopProcess(os.path.join(devicePath, 'clientproxy.pid'), 'clientproxy')
stopProcess(os.path.join(devicePath, 'twistd.pid'), 'buildslave')
log.debug(' clearing flag files')
if os.path.isfile(errorFile):
log.info(' error.flg cleared')
os.remove(errorFile)
log.debug(' sending rebt to device')
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((deviceIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
except:
log.error(' device socket error')
def stopDevice(device):
deviceIP = getIPAddress(device)
devicePath = os.path.join(options.bbpath, device)
errorFile = os.path.join(devicePath, 'error.flg')
log.info('%s: %s - stopping all processes' % (device, deviceIP))
stopProcess(os.path.join(devicePath, 'remotereftest.pid'), 'remotereftest')
stopProcess(
os.path.join(devicePath, 'runtestsremote.pid'), 'runtestsremote')
stopProcess(os.path.join(
devicePath, 'remotereftest.pid.xpcshell.pid'), 'xpcshell')
stopProcess(os.path.join(devicePath, 'clientproxy.pid'), 'clientproxy')
stopProcess(os.path.join(devicePath, 'twistd.pid'), 'buildslave')
log.debug(' clearing flag files')
if os.path.isfile(errorFile):
log.info(' error.flg cleared')
os.remove(errorFile)
log.debug(' sending rebt to device')
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((deviceIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
except:
log.error(' device socket error')
sys.exit(1)
options = loadOptions(defaultOptions)
initLogs(options)
options.bbpath = os.path.abspath(options.bbpath)
if options.tegra is None:
if 'tegra-' in ourPath.lower():
options.tegra = os.path.basename(os.path.split(ourPath)[-1])
if options.tegra is None:
log.error('Tegra has not been specified, exiting')
sys.exit(1)
if options.tegraIP is None:
options.tegraIP = getIPAddress(options.tegra)
try:
n = int(options.tegra.split('-')[1])
except:
n = 0
sutDialbackPort += n
p = os.getpid()
log.info('%s: ourIP %s tegra %s tegraIP %s bbpath %s' % (p, ourIP, options.tegra, options.tegraIP, options.bbpath))
pidFile = os.path.join(ourPath, '%s.pid' % ourName)
if options.background and not 'stop' in options.args:
daemon = Daemon(pidFile)
daemon.start()
def checkTegra(master, tegra):
tegraIP = getIPAddress(tegra)
tegraPath = os.path.join(options.bbpath, tegra)
exportFile = os.path.join(tegraPath, '%s_status.log' % tegra)
errorFile = os.path.join(tegraPath, 'error.flg')
errorFlag = os.path.isfile(errorFile)
sTegra = 'OFFLINE'
sutFound = False
logTD = None
status = {'tegra': tegra,
'active': False,
'cp': 'OFFLINE',
'bs': 'OFFLINE',
'msg': '',
}
log.debug('%s: %s' % (tegra, tegraIP))
if master is None:
status['environment'] = 's'
status['master'] = 'localhost'
else:
status['environment'] = master['environment'][0]
status['master'] = 'http://%s:%s' % (
master['hostname'], master['http_port'])
fPing, lPing = pingDevice(tegra)
if fPing:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
sutFound = True
time.sleep(2)
hbSocket.send('info\n')
d = hbSocket.recv(4096)
log.debug('socket data length %d' % len(d))
log.debug(d)
status['active'] = True
hbSocket.close()
except:
status['active'] = False
dumpException('socket')
if status['active']:
sTegra = 'online'
else:
sTegra = 'INACTIVE'
if not sutFound:
status['msg'] += 'SUTAgent not present;'
else:
status['msg'] += '%s %s;' % (lPing[0], lPing[1])
# Cheat until we have a better check solution for new watch_devices.sh
status['cp'] = 'active' # pretend all is well
if checkSlaveAlive(tegraPath):
logTD = checkSlaveActive(tegraPath)
if logTD is not None:
if (logTD.days > 0) or (logTD.days == 0 and logTD.seconds > 3600):
status['bs'] = 'INACTIVE'
status['msg'] += 'BS %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['bs'] = 'active'
else:
status['bs'] = 'INACTIVE'
else:
# scan thru tegra-### dir and see if any buildbot.tac.bug#### files
# exist but ignore buildbot.tac file itself (except to note that it is
# missing)
files = os.listdir(tegraPath)
found = False
for f in files:
if f.startswith('buildbot.tac'):
found = True
if len(f) > 12:
status['msg'] += '%s;' % f
if not found:
status['msg'] += 'buildbot.tac NOT found;'
if errorFlag:
status['msg'] += 'error.flg [%s] ' % getLastLine(errorFile)
s = '%s %s %9s %8s %8s :: %s' % (status['tegra'], status['environment'],
sTegra, status['cp'], status['bs'],
status['msg'])
ts = time.strftime('%Y-%m-%d %H:%M:%S')
log.info(s)
open(exportFile, 'a+').write('%s %s\n' % (ts, s))
summary(status['tegra'], status['environment'], sTegra, status[
'cp'], status['bs'], status['msg'], ts, status['master'])
if errorFlag and options.reset:
stopProcess(os.path.join(tegraPath, 'twistd.pid'), 'buildslave')
if not options.reboot:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
log.info('rebooting tegra')
except:
dumpException('socket')
if errorFlag:
log.info('clearing error.flg')
os.remove(errorFile)
# Here we try to catch the state where sutagent and cp are inactive that
# is determined by:
# sTegra == 'INACTIVE' and
# status['cp'] == 'INACTIVE'
# status['cp'] will be set to INACTIVE only if logTD.seconds (last time
# clientproxy updated it's log file) is > 3600
if options.reboot:
if not sutFound and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_device(tegra)
else:
if sTegra == 'OFFLINE' and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_device(tegra)
if options.reset and sTegra == 'INACTIVE' and status['cp'] == 'INACTIVE':
log.info('stopping hung clientproxy')
stopDevice(tegra)
time.sleep(5)
log.info('starting clientproxy for %s' % tegra)
os.chdir(tegraPath)
runCommand(['python', 'clientproxy.py', '-b', '--device=%s' % tegra])
def checkTegra(master, tegra):
tegraIP = getIPAddress(tegra)
tegraPath = os.path.join(options.bbpath, tegra)
exportFile = os.path.join(tegraPath, '%s_status.log' % tegra)
errorFile = os.path.join(tegraPath, 'error.flg')
proxyFile = os.path.join(tegraPath, 'proxy.flg')
errorFlag = os.path.isfile(errorFile)
proxyFlag = os.path.isfile(proxyFile)
sTegra = 'OFFLINE'
sutFound = False
status = { 'tegra': tegra,
'active': False,
'cp': 'OFFLINE',
'bs': 'OFFLINE',
'msg': '',
}
log.debug('%s: %s' % (tegra, tegraIP))
if master is None:
status['environment'] = 's'
status['master'] = 'localhost'
else:
status['environment'] = master['environment'][0]
status['master'] = 'http://%s:%s' % (master['hostname'], master['http_port'])
fPing, lPing = pingTegra(tegra)
if fPing:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
sutFound = True
time.sleep(2)
hbSocket.send('info\n')
d = hbSocket.recv(4096)
log.debug('socket data length %d' % len(d))
log.debug(d)
status['active'] = True
hbSocket.close()
except:
status['active'] = False
dumpException('socket')
if status['active']:
sTegra = 'online'
else:
sTegra = 'INACTIVE'
if not sutFound:
status['msg'] += 'SUTAgent not present;'
else:
status['msg'] += '%s %s;' % (lPing[0], lPing[1])
if checkCPAlive(tegraPath):
logTD = checkCPActive(tegraPath)
if logTD is not None and logTD.days > 0 or (logTD.days == 0 and logTD.seconds > 300):
status['cp'] = 'INACTIVE'
status['msg'] += 'CP %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['cp'] = 'active'
else:
if os.path.isfile(os.path.join(tegraPath, 'clientproxy.pid')):
status['msg'] += 'clientproxy.pid found;'
if checkSlaveAlive(tegraPath):
logTD = checkSlaveActive(tegraPath)
if logTD is not None and logTD.days > 0 or (logTD.days == 0 and logTD.seconds > 3600):
status['bs'] = 'INACTIVE'
status['msg'] += 'BS %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['bs'] = 'active'
else:
# scan thru tegra-### dir and see if any buildbot.tac.bug#### files exist
# but ignore buildbot.tac file itself (except to note that it is missing)
files = os.listdir(tegraPath)
found = False
for f in files:
if f.startswith('buildbot.tac'):
found = True
if len(f) > 12:
status['msg'] += '%s;' % f
if not found:
status['msg'] += 'buildbot.tac NOT found;'
if errorFlag:
status['msg'] += 'error.flg [%s] ' % getLastLine(errorFile)
if proxyFlag:
status['msg'] += 'REBOOTING '
s = '%s %s %9s %8s %8s :: %s' % (status['tegra'], status['environment'], sTegra, status['cp'], status['bs'], status['msg'])
ts = time.strftime('%Y-%m-%d %H:%M:%S')
log.info(s)
open(exportFile, 'a+').write('%s %s\n' % (ts, s))
summary(status['tegra'], status['environment'], sTegra, status['cp'], status['bs'], status['msg'], ts, status['master'])
if errorFlag and options.reset:
stopProcess(os.path.join(tegraPath, 'twistd.pid'), 'buildslave')
if not options.reboot:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
log.info('rebooting tegra')
except:
dumpException('socket')
if errorFlag:
log.info('clearing error.flg')
os.remove(errorFile)
if proxyFlag:
log.info('clearing proxy.flg')
os.remove(proxyFile)
if options.reboot:
if not sutFound and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_tegra(tegra)
else:
if sTegra == 'OFFLINE' and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_tegra(tegra)
def checkTegra(master, tegra):
tegraIP = getIPAddress(tegra)
tegraPath = os.path.join(options.bbpath, tegra)
exportFile = os.path.join(tegraPath, '%s_status.log' % tegra)
errorFile = os.path.join(tegraPath, 'error.flg')
proxyFile = os.path.join(tegraPath, 'proxy.flg')
errorFlag = os.path.isfile(errorFile)
proxyFlag = os.path.isfile(proxyFile)
sTegra = 'OFFLINE'
sutFound = False
status = {
'tegra': tegra,
'active': False,
'cp': 'OFFLINE',
'bs': 'OFFLINE',
'msg': '',
}
log.debug('%s: %s' % (tegra, tegraIP))
if master is None:
status['environment'] = 's'
status['master'] = 'localhost'
else:
status['environment'] = master['environment'][0]
status['master'] = 'http://%s:%s' % (master['hostname'],
master['http_port'])
fPing, lPing = pingTegra(tegra)
if fPing:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
sutFound = True
time.sleep(2)
hbSocket.send('info\n')
d = hbSocket.recv(4096)
log.debug('socket data length %d' % len(d))
log.debug(d)
status['active'] = True
hbSocket.close()
except:
status['active'] = False
dumpException('socket')
if status['active']:
sTegra = 'online'
else:
sTegra = 'INACTIVE'
if not sutFound:
status['msg'] += 'SUTAgent not present;'
else:
status['msg'] += '%s %s;' % (lPing[0], lPing[1])
if checkCPAlive(tegraPath):
logTD = checkCPActive(tegraPath)
if logTD is not None and logTD.days > 0 or (logTD.days == 0
and logTD.seconds > 300):
status['cp'] = 'INACTIVE'
status['msg'] += 'CP %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['cp'] = 'active'
else:
if os.path.isfile(os.path.join(tegraPath, 'clientproxy.pid')):
status['msg'] += 'clientproxy.pid found;'
if checkSlaveAlive(tegraPath):
logTD = checkSlaveActive(tegraPath)
if logTD is not None and logTD.days > 0 or (logTD.days == 0
and logTD.seconds > 3600):
status['bs'] = 'INACTIVE'
status['msg'] += 'BS %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['bs'] = 'active'
else:
# scan thru tegra-### dir and see if any buildbot.tac.bug#### files exist
# but ignore buildbot.tac file itself (except to note that it is missing)
files = os.listdir(tegraPath)
found = False
for f in files:
if f.startswith('buildbot.tac'):
found = True
if len(f) > 12:
status['msg'] += '%s;' % f
if not found:
status['msg'] += 'buildbot.tac NOT found;'
if errorFlag:
status['msg'] += 'error.flg [%s] ' % getLastLine(errorFile)
if proxyFlag:
status['msg'] += 'REBOOTING '
s = '%s %s %9s %8s %8s :: %s' % (status['tegra'], status['environment'],
sTegra, status['cp'], status['bs'],
status['msg'])
ts = time.strftime('%Y-%m-%d %H:%M:%S')
log.info(s)
open(exportFile, 'a+').write('%s %s\n' % (ts, s))
summary(status['tegra'], status['environment'], sTegra, status['cp'],
status['bs'], status['msg'], ts, status['master'])
if errorFlag and options.reset:
stopProcess(os.path.join(tegraPath, 'twistd.pid'), 'buildslave')
if not options.reboot:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
log.info('rebooting tegra')
except:
dumpException('socket')
if errorFlag:
log.info('clearing error.flg')
os.remove(errorFile)
if proxyFlag:
log.info('clearing proxy.flg')
os.remove(proxyFile)
if options.reboot:
if not sutFound and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_tegra(tegra)
else:
if sTegra == 'OFFLINE' and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_tegra(tegra)
sys.exit(1)
options = loadOptions(defaultOptions)
initLogs(options)
options.bbpath = os.path.abspath(options.bbpath)
if options.tegra is None:
if 'tegra-' in ourPath.lower():
options.tegra = os.path.basename(os.path.split(ourPath)[-1])
if options.tegra is None:
log.error('Tegra has not been specified, exiting')
sys.exit(1)
if options.tegraIP is None:
options.tegraIP = getIPAddress(options.tegra)
try:
n = int(options.tegra.split('-')[1])
except:
n = 0
sutDialbackPort += n
p = os.getpid()
log.info('%s: ourIP %s tegra %s tegraIP %s bbpath %s' %
(p, ourIP, options.tegra, options.tegraIP, options.bbpath))
pidFile = os.path.join(ourPath, '%s.pid' % ourName)
if options.background and not 'stop' in options.args:
daemon = Daemon(pidFile)
def checkTegra(master, tegra):
tegraIP = getIPAddress(tegra)
tegraPath = os.path.join(options.bbpath, tegra)
exportFile = os.path.join(tegraPath, '%s_status.log' % tegra)
errorFile = os.path.join(tegraPath, 'error.flg')
errorFlag = os.path.isfile(errorFile)
sTegra = 'OFFLINE'
sutFound = False
logTD = None
status = {
'tegra': tegra,
'active': False,
'cp': 'OFFLINE',
'bs': 'OFFLINE',
'msg': '',
}
log.debug('%s: %s' % (tegra, tegraIP))
if master is None:
status['environment'] = 's'
status['master'] = 'localhost'
else:
status['environment'] = master['environment'][0]
status['master'] = 'http://%s:%s' % (master['hostname'],
master['http_port'])
fPing, lPing = pingDevice(tegra)
if fPing:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
sutFound = True
time.sleep(2)
hbSocket.send('info\n')
d = hbSocket.recv(4096)
log.debug('socket data length %d' % len(d))
log.debug(d)
status['active'] = True
hbSocket.close()
except:
status['active'] = False
dumpException('socket')
if status['active']:
sTegra = 'online'
else:
sTegra = 'INACTIVE'
if not sutFound:
status['msg'] += 'SUTAgent not present;'
else:
status['msg'] += '%s %s;' % (lPing[0], lPing[1])
# Cheat until we have a better check solution for new watch_devices.sh
status['cp'] = 'active' # pretend all is well
if checkSlaveAlive(tegraPath):
logTD = checkSlaveActive(tegraPath)
if logTD is not None:
if (logTD.days > 0) or (logTD.days == 0 and logTD.seconds > 3600):
status['bs'] = 'INACTIVE'
status['msg'] += 'BS %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['bs'] = 'active'
else:
status['bs'] = 'INACTIVE'
else:
# scan thru tegra-### dir and see if any buildbot.tac.bug#### files
# exist but ignore buildbot.tac file itself (except to note that it is
# missing)
files = os.listdir(tegraPath)
found = False
for f in files:
if f.startswith('buildbot.tac'):
found = True
if len(f) > 12:
status['msg'] += '%s;' % f
if not found:
status['msg'] += 'buildbot.tac NOT found;'
if errorFlag:
status['msg'] += 'error.flg [%s] ' % getLastLine(errorFile)
s = '%s %s %9s %8s %8s :: %s' % (status['tegra'], status['environment'],
sTegra, status['cp'], status['bs'],
status['msg'])
ts = time.strftime('%Y-%m-%d %H:%M:%S')
log.info(s)
open(exportFile, 'a+').write('%s %s\n' % (ts, s))
summary(status['tegra'], status['environment'], sTegra, status['cp'],
status['bs'], status['msg'], ts, status['master'])
if errorFlag and options.reset:
stopProcess(os.path.join(tegraPath, 'twistd.pid'), 'buildslave')
if not options.reboot:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
log.info('rebooting tegra')
except:
dumpException('socket')
if errorFlag:
log.info('clearing error.flg')
os.remove(errorFile)
# Here we try to catch the state where sutagent and cp are inactive that
# is determined by:
# sTegra == 'INACTIVE' and
# status['cp'] == 'INACTIVE'
# status['cp'] will be set to INACTIVE only if logTD.seconds (last time
# clientproxy updated it's log file) is > 3600
if options.reboot:
if not sutFound and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_device(tegra)
else:
if sTegra == 'OFFLINE' and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_device(tegra)
if options.reset and sTegra == 'INACTIVE' and status['cp'] == 'INACTIVE':
log.info('stopping hung clientproxy')
stopDevice(tegra)
time.sleep(5)
log.info('starting clientproxy for %s' % tegra)
os.chdir(tegraPath)
runCommand(['python', 'clientproxy.py', '-b', '--device=%s' % tegra])
