def stopDevice(device):
deviceIP = getIPAddress(device)
devicePath = os.path.join(options.bbpath, device)
errorFile = os.path.join(devicePath, 'error.flg')
log.info('%s: %s - stopping all processes' % (device, deviceIP))
stopProcess(os.path.join(devicePath, 'remotereftest.pid'), 'remotereftest')
stopProcess(
os.path.join(devicePath, 'runtestsremote.pid'), 'runtestsremote')
stopProcess(os.path.join(
devicePath, 'remotereftest.pid.xpcshell.pid'), 'xpcshell')
stopProcess(os.path.join(devicePath, 'clientproxy.pid'), 'clientproxy')
stopProcess(os.path.join(devicePath, 'twistd.pid'), 'buildslave')
log.debug(' clearing flag files')
if os.path.isfile(errorFile):
log.info(' error.flg cleared')
os.remove(errorFile)
log.debug(' sending rebt to device')
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((deviceIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
except:
log.error(' device socket error')
def stopDevice(device):
deviceIP = getIPAddress(device)
devicePath = os.path.join(options.bbpath, device)
errorFile = os.path.join(devicePath, 'error.flg')
log.info('%s: %s - stopping all processes' % (device, deviceIP))
stopProcess(os.path.join(devicePath, 'remotereftest.pid'), 'remotereftest')
stopProcess(os.path.join(devicePath, 'runtestsremote.pid'),
'runtestsremote')
stopProcess(os.path.join(devicePath, 'remotereftest.pid.xpcshell.pid'),
'xpcshell')
stopProcess(os.path.join(devicePath, 'clientproxy.pid'), 'clientproxy')
stopProcess(os.path.join(devicePath, 'twistd.pid'), 'buildslave')
log.debug(' clearing flag files')
if os.path.isfile(errorFile):
log.info(' error.flg cleared')
os.remove(errorFile)
log.debug(' sending rebt to device')
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((deviceIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
except:
log.error(' device socket error')
def checkTegra(master, tegra):
tegraIP = getIPAddress(tegra)
tegraPath = os.path.join(options.bbpath, tegra)
exportFile = os.path.join(tegraPath, '%s_status.log' % tegra)
errorFile = os.path.join(tegraPath, 'error.flg')
errorFlag = os.path.isfile(errorFile)
sTegra = 'OFFLINE'
sutFound = False
logTD = None
status = {'tegra': tegra,
'active': False,
'cp': 'OFFLINE',
'bs': 'OFFLINE',
'msg': '',
}
log.debug('%s: %s' % (tegra, tegraIP))
if master is None:
status['environment'] = 's'
status['master'] = 'localhost'
else:
status['environment'] = master['environment'][0]
status['master'] = 'http://%s:%s' % (
master['hostname'], master['http_port'])
fPing, lPing = pingDevice(tegra)
if fPing:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
sutFound = True
time.sleep(2)
hbSocket.send('info\n')
d = hbSocket.recv(4096)
log.debug('socket data length %d' % len(d))
log.debug(d)
status['active'] = True
hbSocket.close()
except:
status['active'] = False
dumpException('socket')
if status['active']:
sTegra = 'online'
else:
sTegra = 'INACTIVE'
if not sutFound:
status['msg'] += 'SUTAgent not present;'
else:
status['msg'] += '%s %s;' % (lPing[0], lPing[1])
# Cheat until we have a better check solution for new watch_devices.sh
status['cp'] = 'active' # pretend all is well
if checkSlaveAlive(tegraPath):
logTD = checkSlaveActive(tegraPath)
if logTD is not None:
if (logTD.days > 0) or (logTD.days == 0 and logTD.seconds > 3600):
status['bs'] = 'INACTIVE'
status['msg'] += 'BS %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['bs'] = 'active'
else:
status['bs'] = 'INACTIVE'
else:
# scan thru tegra-### dir and see if any buildbot.tac.bug#### files
# exist but ignore buildbot.tac file itself (except to note that it is
# missing)
files = os.listdir(tegraPath)
found = False
for f in files:
if f.startswith('buildbot.tac'):
found = True
if len(f) > 12:
status['msg'] += '%s;' % f
if not found:
status['msg'] += 'buildbot.tac NOT found;'
if errorFlag:
status['msg'] += 'error.flg [%s] ' % getLastLine(errorFile)
s = '%s %s %9s %8s %8s :: %s' % (status['tegra'], status['environment'],
sTegra, status['cp'], status['bs'],
status['msg'])
ts = time.strftime('%Y-%m-%d %H:%M:%S')
log.info(s)
open(exportFile, 'a+').write('%s %s\n' % (ts, s))
summary(status['tegra'], status['environment'], sTegra, status[
'cp'], status['bs'], status['msg'], ts, status['master'])
if errorFlag and options.reset:
stopProcess(os.path.join(tegraPath, 'twistd.pid'), 'buildslave')
if not options.reboot:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
log.info('rebooting tegra')
except:
dumpException('socket')
if errorFlag:
log.info('clearing error.flg')
os.remove(errorFile)
# Here we try to catch the state where sutagent and cp are inactive that
# is determined by:
# sTegra == 'INACTIVE' and
# status['cp'] == 'INACTIVE'
# status['cp'] will be set to INACTIVE only if logTD.seconds (last time
# clientproxy updated it's log file) is > 3600
if options.reboot:
if not sutFound and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_device(tegra)
else:
if sTegra == 'OFFLINE' and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_device(tegra)
if options.reset and sTegra == 'INACTIVE' and status['cp'] == 'INACTIVE':
log.info('stopping hung clientproxy')
stopDevice(tegra)
time.sleep(5)
log.info('starting clientproxy for %s' % tegra)
os.chdir(tegraPath)
runCommand(['python', 'clientproxy.py', '-b', '--device=%s' % tegra])
def checkTegra(master, tegra):
tegraIP = getIPAddress(tegra)
tegraPath = os.path.join(options.bbpath, tegra)
exportFile = os.path.join(tegraPath, '%s_status.log' % tegra)
errorFile = os.path.join(tegraPath, 'error.flg')
proxyFile = os.path.join(tegraPath, 'proxy.flg')
errorFlag = os.path.isfile(errorFile)
proxyFlag = os.path.isfile(proxyFile)
sTegra = 'OFFLINE'
sutFound = False
status = { 'tegra': tegra,
'active': False,
'cp': 'OFFLINE',
'bs': 'OFFLINE',
'msg': '',
}
log.debug('%s: %s' % (tegra, tegraIP))
if master is None:
status['environment'] = 's'
status['master'] = 'localhost'
else:
status['environment'] = master['environment'][0]
status['master'] = 'http://%s:%s' % (master['hostname'], master['http_port'])
fPing, lPing = pingTegra(tegra)
if fPing:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
sutFound = True
time.sleep(2)
hbSocket.send('info\n')
d = hbSocket.recv(4096)
log.debug('socket data length %d' % len(d))
log.debug(d)
status['active'] = True
hbSocket.close()
except:
status['active'] = False
dumpException('socket')
if status['active']:
sTegra = 'online'
else:
sTegra = 'INACTIVE'
if not sutFound:
status['msg'] += 'SUTAgent not present;'
else:
status['msg'] += '%s %s;' % (lPing[0], lPing[1])
if checkCPAlive(tegraPath):
logTD = checkCPActive(tegraPath)
if logTD is not None and logTD.days > 0 or (logTD.days == 0 and logTD.seconds > 300):
status['cp'] = 'INACTIVE'
status['msg'] += 'CP %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['cp'] = 'active'
else:
if os.path.isfile(os.path.join(tegraPath, 'clientproxy.pid')):
status['msg'] += 'clientproxy.pid found;'
if checkSlaveAlive(tegraPath):
logTD = checkSlaveActive(tegraPath)
if logTD is not None and logTD.days > 0 or (logTD.days == 0 and logTD.seconds > 3600):
status['bs'] = 'INACTIVE'
status['msg'] += 'BS %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['bs'] = 'active'
else:
# scan thru tegra-### dir and see if any buildbot.tac.bug#### files exist
# but ignore buildbot.tac file itself (except to note that it is missing)
files = os.listdir(tegraPath)
found = False
for f in files:
if f.startswith('buildbot.tac'):
found = True
if len(f) > 12:
status['msg'] += '%s;' % f
if not found:
status['msg'] += 'buildbot.tac NOT found;'
if errorFlag:
status['msg'] += 'error.flg [%s] ' % getLastLine(errorFile)
if proxyFlag:
status['msg'] += 'REBOOTING '
s = '%s %s %9s %8s %8s :: %s' % (status['tegra'], status['environment'], sTegra, status['cp'], status['bs'], status['msg'])
ts = time.strftime('%Y-%m-%d %H:%M:%S')
log.info(s)
open(exportFile, 'a+').write('%s %s\n' % (ts, s))
summary(status['tegra'], status['environment'], sTegra, status['cp'], status['bs'], status['msg'], ts, status['master'])
if errorFlag and options.reset:
stopProcess(os.path.join(tegraPath, 'twistd.pid'), 'buildslave')
if not options.reboot:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
log.info('rebooting tegra')
except:
dumpException('socket')
if errorFlag:
log.info('clearing error.flg')
os.remove(errorFile)
if proxyFlag:
log.info('clearing proxy.flg')
os.remove(proxyFile)
if options.reboot:
if not sutFound and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_tegra(tegra)
else:
if sTegra == 'OFFLINE' and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_tegra(tegra)
if dm.fileExists('/system/etc/hosts'):
print "removing /system/etc/hosts file"
try:
dm.sendCMD(['exec mount -o remount,rw -t yaffs2 /dev/block/mtdblock3 /system'])
dm.sendCMD(['exec rm /system/etc/hosts'])
except devicemanager.DMError, e:
print "Exception hit while trying to remove /system/etc/hosts: %s" % str(e)
setFlag(errorFile, "failed to remove /system/etc/hosts")
sys.exit(1)
if dm.fileExists('/system/etc/hosts'):
setFlag(errorFile, "failed to remove /system/etc/hosts")
sys.exit(1)
else:
print "successfully removed hosts file, we can test!!!"
for f in ('runtestsremote', 'remotereftest', 'remotereftest.pid.xpcshell'):
pidFile = os.path.join(pidDir, '%s.pid' % f)
print "checking for previous test processes ... %s" % pidFile
if os.path.exists(pidFile):
print "pidfile from prior test run found, trying to kill"
stopProcess(pidFile, f)
if os.path.exists(pidFile):
setFlag(errorFile, "Remote Device Error: process from previous test run present [%s]" % f)
sys.exit(2)
for p in processNames:
if dm.dirExists('/data/data/%s' % p):
print dm.uninstallAppAndReboot(p)
waitForDevice(dm)
def checkTegra(master, tegra):
tegraIP = getIPAddress(tegra)
tegraPath = os.path.join(options.bbpath, tegra)
exportFile = os.path.join(tegraPath, '%s_status.log' % tegra)
errorFile = os.path.join(tegraPath, 'error.flg')
proxyFile = os.path.join(tegraPath, 'proxy.flg')
errorFlag = os.path.isfile(errorFile)
proxyFlag = os.path.isfile(proxyFile)
sTegra = 'OFFLINE'
sutFound = False
status = {
'tegra': tegra,
'active': False,
'cp': 'OFFLINE',
'bs': 'OFFLINE',
'msg': '',
}
log.debug('%s: %s' % (tegra, tegraIP))
if master is None:
status['environment'] = 's'
status['master'] = 'localhost'
else:
status['environment'] = master['environment'][0]
status['master'] = 'http://%s:%s' % (master['hostname'],
master['http_port'])
fPing, lPing = pingTegra(tegra)
if fPing:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
sutFound = True
time.sleep(2)
hbSocket.send('info\n')
d = hbSocket.recv(4096)
log.debug('socket data length %d' % len(d))
log.debug(d)
status['active'] = True
hbSocket.close()
except:
status['active'] = False
dumpException('socket')
if status['active']:
sTegra = 'online'
else:
sTegra = 'INACTIVE'
if not sutFound:
status['msg'] += 'SUTAgent not present;'
else:
status['msg'] += '%s %s;' % (lPing[0], lPing[1])
if checkCPAlive(tegraPath):
logTD = checkCPActive(tegraPath)
if logTD is not None and logTD.days > 0 or (logTD.days == 0
and logTD.seconds > 300):
status['cp'] = 'INACTIVE'
status['msg'] += 'CP %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['cp'] = 'active'
else:
if os.path.isfile(os.path.join(tegraPath, 'clientproxy.pid')):
status['msg'] += 'clientproxy.pid found;'
if checkSlaveAlive(tegraPath):
logTD = checkSlaveActive(tegraPath)
if logTD is not None and logTD.days > 0 or (logTD.days == 0
and logTD.seconds > 3600):
status['bs'] = 'INACTIVE'
status['msg'] += 'BS %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['bs'] = 'active'
else:
# scan thru tegra-### dir and see if any buildbot.tac.bug#### files exist
# but ignore buildbot.tac file itself (except to note that it is missing)
files = os.listdir(tegraPath)
found = False
for f in files:
if f.startswith('buildbot.tac'):
found = True
if len(f) > 12:
status['msg'] += '%s;' % f
if not found:
status['msg'] += 'buildbot.tac NOT found;'
if errorFlag:
status['msg'] += 'error.flg [%s] ' % getLastLine(errorFile)
if proxyFlag:
status['msg'] += 'REBOOTING '
s = '%s %s %9s %8s %8s :: %s' % (status['tegra'], status['environment'],
sTegra, status['cp'], status['bs'],
status['msg'])
ts = time.strftime('%Y-%m-%d %H:%M:%S')
log.info(s)
open(exportFile, 'a+').write('%s %s\n' % (ts, s))
summary(status['tegra'], status['environment'], sTegra, status['cp'],
status['bs'], status['msg'], ts, status['master'])
if errorFlag and options.reset:
stopProcess(os.path.join(tegraPath, 'twistd.pid'), 'buildslave')
if not options.reboot:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
log.info('rebooting tegra')
except:
dumpException('socket')
if errorFlag:
log.info('clearing error.flg')
os.remove(errorFile)
if proxyFlag:
log.info('clearing proxy.flg')
os.remove(proxyFile)
if options.reboot:
if not sutFound and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_tegra(tegra)
else:
if sTegra == 'OFFLINE' and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_tegra(tegra)
if dm.fileExists('/system/etc/hosts'):
print "removing /system/etc/hosts file"
try:
dm.sendCMD(['exec mount -o remount,rw -t yaffs2 /dev/block/mtdblock3 /system'])
dm.sendCMD(['exec rm /system/etc/hosts'])
except devicemanager.DMError, e:
print "Exception hit while trying to remove /system/etc/hosts: %s" % str(e)
setFlag(errorFile, "failed to remove /system/etc/hosts")
sys.exit(1)
if dm.fileExists('/system/etc/hosts'):
setFlag(errorFile, "failed to remove /system/etc/hosts")
sys.exit(1)
else:
print "successfully removed hosts file, we can test!!!"
for f in ('runtestsremote', 'remotereftest', 'remotereftest.pid.xpcshell'):
pidFile = os.path.join(pidDir, '%s.pid' % f)
print "checking for previous test processes ... %s" % pidFile
if os.path.exists(pidFile):
print "pidfile from prior test run found, trying to kill"
stopProcess(pidFile, f)
if os.path.exists(pidFile):
setFlag(errorFile, "Remote Device Error: process from previous test run present [%s]" % f)
sys.exit(2)
for p in processNames:
if dm.dirExists('/data/data/%s' % p):
print dm.uninstallAppAndReboot(p)
waitForDevice(dm)
def checkTegra(master, tegra):
tegraIP = getIPAddress(tegra)
tegraPath = os.path.join(options.bbpath, tegra)
exportFile = os.path.join(tegraPath, '%s_status.log' % tegra)
errorFile = os.path.join(tegraPath, 'error.flg')
errorFlag = os.path.isfile(errorFile)
sTegra = 'OFFLINE'
sutFound = False
logTD = None
status = {
'tegra': tegra,
'active': False,
'cp': 'OFFLINE',
'bs': 'OFFLINE',
'msg': '',
}
log.debug('%s: %s' % (tegra, tegraIP))
if master is None:
status['environment'] = 's'
status['master'] = 'localhost'
else:
status['environment'] = master['environment'][0]
status['master'] = 'http://%s:%s' % (master['hostname'],
master['http_port'])
fPing, lPing = pingDevice(tegra)
if fPing:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
sutFound = True
time.sleep(2)
hbSocket.send('info\n')
d = hbSocket.recv(4096)
log.debug('socket data length %d' % len(d))
log.debug(d)
status['active'] = True
hbSocket.close()
except:
status['active'] = False
dumpException('socket')
if status['active']:
sTegra = 'online'
else:
sTegra = 'INACTIVE'
if not sutFound:
status['msg'] += 'SUTAgent not present;'
else:
status['msg'] += '%s %s;' % (lPing[0], lPing[1])
# Cheat until we have a better check solution for new watch_devices.sh
status['cp'] = 'active' # pretend all is well
if checkSlaveAlive(tegraPath):
logTD = checkSlaveActive(tegraPath)
if logTD is not None:
if (logTD.days > 0) or (logTD.days == 0 and logTD.seconds > 3600):
status['bs'] = 'INACTIVE'
status['msg'] += 'BS %dd %ds;' % (logTD.days, logTD.seconds)
else:
status['bs'] = 'active'
else:
status['bs'] = 'INACTIVE'
else:
# scan thru tegra-### dir and see if any buildbot.tac.bug#### files
# exist but ignore buildbot.tac file itself (except to note that it is
# missing)
files = os.listdir(tegraPath)
found = False
for f in files:
if f.startswith('buildbot.tac'):
found = True
if len(f) > 12:
status['msg'] += '%s;' % f
if not found:
status['msg'] += 'buildbot.tac NOT found;'
if errorFlag:
status['msg'] += 'error.flg [%s] ' % getLastLine(errorFile)
s = '%s %s %9s %8s %8s :: %s' % (status['tegra'], status['environment'],
sTegra, status['cp'], status['bs'],
status['msg'])
ts = time.strftime('%Y-%m-%d %H:%M:%S')
log.info(s)
open(exportFile, 'a+').write('%s %s\n' % (ts, s))
summary(status['tegra'], status['environment'], sTegra, status['cp'],
status['bs'], status['msg'], ts, status['master'])
if errorFlag and options.reset:
stopProcess(os.path.join(tegraPath, 'twistd.pid'), 'buildslave')
if not options.reboot:
try:
hbSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
hbSocket.settimeout(float(120))
hbSocket.connect((tegraIP, 20700))
hbSocket.send('rebt\n')
hbSocket.close()
log.info('rebooting tegra')
except:
dumpException('socket')
if errorFlag:
log.info('clearing error.flg')
os.remove(errorFile)
# Here we try to catch the state where sutagent and cp are inactive that
# is determined by:
# sTegra == 'INACTIVE' and
# status['cp'] == 'INACTIVE'
# status['cp'] will be set to INACTIVE only if logTD.seconds (last time
# clientproxy updated it's log file) is > 3600
if options.reboot:
if not sutFound and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_device(tegra)
else:
if sTegra == 'OFFLINE' and status['bs'] != 'active':
log.info('power cycling tegra')
reboot_device(tegra)
if options.reset and sTegra == 'INACTIVE' and status['cp'] == 'INACTIVE':
log.info('stopping hung clientproxy')
stopDevice(tegra)
time.sleep(5)
log.info('starting clientproxy for %s' % tegra)
os.chdir(tegraPath)
runCommand(['python', 'clientproxy.py', '-b', '--device=%s' % tegra])
