def run(self):
"""
Module initialization and loop
"""
while True:
try:
# Download test-bot.config to see which versions of Firefox to run the FBTests against
utils.download(
"%sreleases/firebug/test-bot.config" % self.serverpath,
"test-bot.config")
config = ConfigParser()
config.read("test-bot.config")
for section in config.sections():
try:
if not self.testlist:
self.testlist = config.get(section, "TEST_LIST")
if not self.binary:
builds = config.get(section,
"GECKO_VERSION").split(",")
except Exception:
self.log.error("Could not parse config file")
self.log.error(traceback.format_exc())
continue
self.log.info("Starting builds and FBTests for %s" %
section)
# Run the build(s)
if not self.binary:
ret = self.prepare_builds(section, builds)
if ret != 0:
self.log.error(ret)
else:
self.start_tests(section)
self.clean_temp_folder()
except Exception:
self.log.error("Could not run the FBTests")
self.log.error(traceback.format_exc())
raise
if not self.waitTime:
mozlog.shutdown()
break
# Wait for specified number of hours
self.log.info(
"Sleeping for %s hour%s" %
(self.waitTime, "s" if int(self.waitTime) > 1 else ""))
sleep(int(self.waitTime) * 3600)
if os.path.isfile("test-bot.config"):
os.remove("test-bot.config")
def run(self):
"""
Module initialization and loop
"""
while True:
try:
# Download test-bot.config to see which versions of Firefox to run the FBTests against
utils.download(self.serverpath + "releases/firebug/test-bot.config", "test-bot.config")
config = ConfigParser()
config.read("test-bot.config")
for section in config.sections():
version = section[-3:]
if not self.version or version == self.version:
try:
if not self.testlist:
self.testlist = config.get("Firebug" + version, "TEST_LIST")
if not self.binary:
builds = config.get("Firebug" + version, "GECKO_VERSION").split(",")
except Exception as e:
print "[Error] Malformed config file: " + str(e)
continue
print "[Info] Starting builds and FBTests for Firebug" + version
# Run the build(s)
if not self.binary:
ret = self.prepare_builds(version, builds)
if ret != 0:
print ret
else:
self.start_tests(version)
self.clean_temp_folder()
except Exception, e:
print "[Error] Could not run the FBTests"
raise
if not self.waitTime:
break;
# Wait for specified number of hours
print "[Info] Sleeping for " + str(self.waitTime) + " hour" + ("s" if int(self.waitTime) > 1 else "")
sleep(int(self.waitTime) * 3600)
if os.path.isfile("test-bot.config"):
os.remove("test-bot.config")
def run(self):
"""
Module initialization and loop
"""
while True:
try:
# Download test-bot.config to see which versions of Firefox to run the FBTests against
utils.download("%sreleases/firebug/test-bot.config" % self.serverpath, "test-bot.config")
config = ConfigParser()
config.read("test-bot.config")
for section in config.sections():
try:
if not self.testlist:
self.testlist = config.get(section, "TEST_LIST")
if not self.binary:
builds = config.get(section, "GECKO_VERSION").split(",")
except Exception:
self.log.error("Could not parse config file")
self.log.error(traceback.format_exc())
continue
self.log.info("Starting builds and FBTests for %s" % section)
# Run the build(s)
if not self.binary:
ret = self.prepare_builds(section, builds)
if ret != 0:
self.log.error(ret)
else:
self.start_tests(section)
self.clean_temp_folder()
except Exception:
self.log.error("Could not run the FBTests")
self.log.error(traceback.format_exc())
raise
if not self.waitTime:
mozlog.shutdown()
break;
# Wait for specified number of hours
self.log.info("Sleeping for %s hour%s" % (self.waitTime, "s" if int(self.waitTime) > 1 else ""))
sleep(int(self.waitTime) * 3600)
if os.path.isfile("test-bot.config"):
os.remove("test-bot.config")
def prepare_builds(self, version, builds):
"""
Downloads the builds and starts the tests
"""
# For each version of Firefox, see if there is a new changeset and run the tests
for build in builds:
# put this here because the firebug team sometimes forgets and puts 'mozilla-central' instead of central
if build.lower() == "mozilla-central":
build = "central";
print "[Info] Running Firebug" + version + " tests against Mozilla " + build
# Scrape for the latest tinderbox build and extract it to the basedir
try:
# Location to save the tinderbox build
buildPath = os.path.join(self.tempdir, "mozilla-" + build);
# Get the url to the latest tinderbox build
proc = subprocess.Popen("get-latest-tinderbox --product=mozilla-" + build, shell=True, stdout=subprocess.PIPE)
tinderbox_url = proc.communicate()[0]
# Download and extract the tinderbox build
if self.platform == "darwin":
utils.download(tinderbox_url, os.path.join(buildPath, "firefox.dmg"))
proc = subprocess.Popen("hdiutil mount " + os.path.join(buildPath, "firefox.dmg"), shell=True, stdout=subprocess.PIPE)
for data in proc.communicate()[0].split():
if data.find("/Volumes/") != -1:
appDir = data
for appFile in os.listdir(appDir):
if appFile[-4:] == ".app":
appName = appFile
break
subprocess.call("cp -r " + os.path.join(appDir, appName) + " " + buildPath, shell=True)
subprocess.call("hdiutil detach " + appDir, shell=True)
buildPath = os.path.join(buildPath, appName)
else:
if self.platform == "windows":
utils.download(tinderbox_url, buildPath + ".zip")
bundle = zipfile.ZipFile(buildPath + ".zip")
else:
utils.download(tinderbox_url, buildPath + ".tar.bz2")
bundle = tarfile.open(buildPath + ".tar.bz2")
bundle.extractall(buildPath)
bundle.close()
buildPath = os.path.join(buildPath, "firefox")
except Exception as e:
print "[Error] Could not prepare the latest tinderbox build: " + str(e)
continue
# If the newest tinderbox changeset is different from the previously run changeset
if self.build_needed(version, build, buildPath):
if self.platform == "darwin":
self.binary = buildPath
else:
self.binary = os.path.join(buildPath, "firefox" + (".exe" if self.platform == "windows" else ""))
try:
self.start_tests(version)
except Exception, e:
print "[Error] Running Firebug" + version + " against Mozilla " + build + " failed"
self.binary = None
else:
print "[Info] Tests already run with this changeset"
# Remove build directories and temp files
self.clean_temp_folder(build)
def prepare_builds(self, section, builds):
"""
Downloads the builds and starts the tests
"""
# For each version of Firefox, see if there is a new changeset and run the tests
for build in builds:
# put this here because the firebug team sometimes forgets and puts 'mozilla-central' instead of central
if build.lower() == "mozilla-central":
build = "central"
self.log.info("Running " + section + " tests against Mozilla " +
build)
# Scrape for the latest tinderbox build and extract it to the basedir
try:
# Location to save the tinderbox build
buildPath = os.path.join(self.tempdir, "mozilla-" + build)
# Get the url to the latest tinderbox build
proc = subprocess.Popen(
"get-latest-tinderbox --product=mozilla-" + build,
shell=True,
stdout=subprocess.PIPE)
tinderbox_url = proc.communicate()[0]
# Download and extract the tinderbox build
if self.platform == "darwin":
utils.download(tinderbox_url,
os.path.join(buildPath, "firefox.dmg"))
proc = subprocess.Popen(
"hdiutil mount " +
os.path.join(buildPath, "firefox.dmg"),
shell=True,
stdout=subprocess.PIPE)
for data in proc.communicate()[0].split():
if data.find("/Volumes/") != -1:
appDir = data
for appFile in os.listdir(appDir):
if appFile[-4:] == ".app":
appName = appFile
break
subprocess.call("cp -r " + os.path.join(appDir, appName) +
" " + buildPath,
shell=True)
subprocess.call("hdiutil detach " + appDir, shell=True)
buildPath = os.path.join(buildPath, appName)
else:
if self.platform == "windows":
utils.download(tinderbox_url, buildPath + ".zip")
bundle = zipfile.ZipFile(buildPath + ".zip")
else:
utils.download(tinderbox_url, buildPath + ".tar.bz2")
bundle = tarfile.open(buildPath + ".tar.bz2")
bundle.extractall(buildPath)
bundle.close()
buildPath = os.path.join(buildPath, "firefox")
except Exception:
self.log.error("Could not prepare the latest tinderbox build")
self.log.error(traceback.format_exc())
continue
# If the newest tinderbox changeset is different from the previously run changeset
if self.build_needed(section, build, buildPath):
if self.platform == "darwin":
self.binary = buildPath
else:
self.binary = os.path.join(
buildPath, "firefox" +
(".exe" if self.platform == "windows" else ""))
try:
self.start_tests(section)
except Exception:
self.log.error("Running " + section + " against Mozilla " +
build + " failed")
self.log.error(traceback.format_exc())
self.binary = None
else:
self.log.info("Tests already run with this changeset")
# Remove build directories and temp files
self.clean_temp_folder(build)
self.testlist = None
return 0
def update(opt):
# Get server's ip address
dummy = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
dummy.connect(('google.com', 0))
ip = dummy.getsockname()[0]
# Grab the test_bot.config file
configDir = "releases/firebug/test-bot.config"
utils.download("http://getfirebug.com/" + configDir, os.path.join(opt.repo, configDir))
# Parse the config file
test_bot = ConfigParser()
test_bot.read(os.path.join(opt.repo, configDir))
# For each section in config file, download specified files and move to webserver
for section in test_bot.sections():
# Grab config information
SVN_REVISION = test_bot.get(section, "SVN_REVISION")
FIREBUG_XPI = test_bot.get(section, "FIREBUG_XPI")
FBTEST_XPI = test_bot.get(section, "FBTEST_XPI")
# Update or create the svn test repository
if not os.path.isdir(os.path.join(opt.repo, ".svn")):
os.system("svn co http://fbug.googlecode.com/svn/tests/ " + os.path.join(opt.repo, SVN_REVISION, "tests") + " -r " + SVN_REVISION)
else:
os.system(os.path.join(opt.repo, "svn") + " update -r " + SVN_REVISION)
# Download the extensions
print FIREBUG_XPI
relPath = getRelativeURL(FIREBUG_XPI)
savePath = os.path.join(opt.repo, relPath)
utils.download(FIREBUG_XPI, savePath)
print FBTEST_XPI
relPath = getRelativeURL(FBTEST_XPI)
savePath = os.path.join(opt.repo, relPath)
utils.download(FBTEST_XPI, savePath)
# Localize extensions for the server
relPath = getRelativeURL(FIREBUG_XPI)
FIREBUG_XPI = "http://" + ip + "/" + relPath
test_bot.set(section, "FIREBUG_XPI", FIREBUG_XPI)
relPath = getRelativeURL(FBTEST_XPI)
FBTEST_XPI = "http://" + ip + "/" + relPath
test_bot.set(section, "FBTEST_XPI", FBTEST_XPI)
# Localize testlist for the server
testlist = test_bot.get(section, "TEST_LIST")
relPath = getRelativeURL(testlist)
testlist = "http://" + ip + "/" + SVN_REVISION + "/" + relPath
test_bot.set(section, "TEST_LIST", testlist)
with open(os.path.join(opt.repo, configDir), 'wb') as configfile:
test_bot.write(configfile)
# Copy the files to the webserver document root (shutil.copytree won't work, settle for this)
if platform.system().lower() == "windows":
os.system("xcopy " + os.path.join(opt.repo, "*") + " " + opt.serverpath + "/E")
print "xcopy " + os.path.join(opt.repo, "*") + " " + opt.serverpath + "/E"
else:
os.system("cp -r " + os.path.join(opt.repo, "*") + " " + opt.serverpath)
print "cp -r " + os.path.join(opt.repo, "*") + " " + opt.serverpath
def update(self):
self.log.info("Updating server extensions and tests")
# Get server's ip address
dummy = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
dummy.connect(('mozilla.org', 0))
ip = dummy.getsockname()[0]
# Grab the test_bot.config file
utils.download("http://getfirebug.com/%s" % self.CONFIG_LOCATION, os.path.join(self.repo, "test-bot.config"))
# Parse the config file
test_bot = ConfigParser()
test_bot.read(os.path.join(self.repo, "test-bot.config"))
copyConfig = False
tags = []
# For each section in config file, download specified files and move to webserver
for section in test_bot.sections():
# Get information from config file
if not (test_bot.has_option(section, "GIT_TAG") and test_bot.has_option(section, "GIT_BRANCH")):
self.log.error("GIT_TAG and GIT_BRANCH must be specified for '%s'" % section)
continue
copyConfig = True
GIT_BRANCH = test_bot.get(section, "GIT_BRANCH")
GIT_TAG = test_bot.get(section, "GIT_TAG")
#Ensure we have a git repo to work with
fbugsrc = os.path.join(self.repo, "firebug")
if not os.path.isdir(fbugsrc):
self._run_cmd(["git","clone", self.FIREBUG_REPO, fbugsrc])
# Create the branch in case it doesn't exist. If it does git will
# spit out an error message which we can ignore
self._run_cmd(["git","fetch","origin"], cwd=fbugsrc)
self._run_cmd(["git","branch",GIT_BRANCH,"origin/%s" % GIT_BRANCH], cwd=fbugsrc)
# Because we may have added new tags we need to pull before we find
# our specific revision
self._run_cmd(["git","checkout",GIT_BRANCH], cwd=fbugsrc)
self._run_cmd(["git","reset","--hard","origin/%s" % GIT_BRANCH], cwd=fbugsrc)
# Check out the tag for the git repo - this assumes we always work
# off tags, branches or specific commit hashes.
self._run_cmd(["git", "checkout", GIT_TAG], cwd=fbugsrc)
# If using HEAD as a tag we need the actual changeset
if GIT_TAG.upper() == "HEAD":
GIT_TAG = self._run_cmd(["git", "rev-parse", GIT_TAG],
cwd=fbugsrc, stdout=subprocess.PIPE).strip()
tags.append(GIT_TAG)
# Localize testlist for the server
if test_bot.has_option(section, "TEST_LIST"):
testlist = test_bot.get(section, "TEST_LIST")
else:
testlist = "http://%s/%s/%s" % (ip, GIT_TAG, self.TESTLIST_LOCATION)
test_bot.set(section, "TEST_LIST", testlist)
if test_bot.has_option(section, "FIREBUG_XPI"):
FIREBUG_XPI = test_bot.get(section, "FIREBUG_XPI")
# Download the extensions
firebug_path = self.getRelativeURL(FIREBUG_XPI)
save_path = os.path.join(self.serverpath, firebug_path)
self.log.debug("Downloading Firebug XPI '%s' to '%s'" % (FIREBUG_XPI, save_path))
utils.download(FIREBUG_XPI, save_path)
else:
# build the extension from the source
# requires java and ant on the webserver
self.log.debug("Building Firebug extension from source")
self._run_cmd(['ant'], cwd=os.path.join(fbugsrc, 'extension'))
ext = os.path.join('extension', 'release')
firebug_path = os.path.join(GIT_TAG, ext,
[f for f in os.listdir(os.path.join(fbugsrc, ext))
if f.startswith('firebug') if f.find('amo') == -1 if f.endswith('.xpi')][0])
if test_bot.has_option(section, 'FBTEST_XPI'):
FBTEST_XPI = test_bot.get(section, 'FBTEST_XPI')
fbtest_path = self.getRelativeURL(FBTEST_XPI)
save_path = os.path.join(self.serverpath, fbtest_path)
self.log.debug("Downloading FBTest XPI '%s' to '%s'" % (FBTEST_XPI, save_path))
utils.download(FBTEST_XPI, save_path)
else:
# build the extension from the source
# requires java and ant on the webserver
self.log.debug("Building FBTest extension from source")
self._run_cmd(['ant'], cwd=os.path.join(fbugsrc, 'tests', 'FBTest'))
ext = os.path.join('tests', 'FBTest', 'release')
fbtest_path = os.path.join(GIT_TAG, ext,
[f for f in os.listdir(os.path.join(fbugsrc, ext))
if f.startswith('fbTest') if f.endswith('.xpi')][0])
# Localize extensions for the server
FIREBUG_XPI = "http://%s/%s" % (ip, firebug_path)
test_bot.set(section, "FIREBUG_XPI", FIREBUG_XPI)
FBTEST_XPI = "http://%s/%s" % (ip, fbtest_path)
test_bot.set(section, "FBTEST_XPI", FBTEST_XPI)
# Copy this to the serverpath
self.recursivecopy(fbugsrc, os.path.join(self.serverpath, GIT_TAG))
if copyConfig:
# Write the complete config file
saveloc = os.path.join(self.serverpath, os.path.dirname(self.CONFIG_LOCATION))
if not os.path.exists(saveloc):
os.makedirs(saveloc)
with open(os.path.join(self.serverpath, self.CONFIG_LOCATION), 'wb') as configfile:
test_bot.write(configfile)
# Remove old revisions to save space
tags.extend(["releases", "tests"])
for name in os.listdir(self.serverpath):
path = os.path.join(self.serverpath, name)
if name not in tags and os.path.isdir(path):
# only remove if it is more than a day old
# this is so we don't delete files that are currently being used in the middle of a test run
mtime = os.path.getmtime(path)
if time.time() - mtime > 24 * 60 * 60: # number of seconds in a day
self.log.debug("Deleting unused changeset: %s" % path)
shutil.rmtree(path)
