def testGetChunkWithPathWithSpaces(self):
self.uvc.product = "Firefox"
self.uvc.platform = "Linux_x86-gcc3"
self.uvc.channel = "betatest"
self.uvc.aus_server = "https://aus2.mozilla.org"
self.uvc.ftp_server = "stage.mozilla.org/firefox"
self.uvc.to = "/firefox/Firefox 4.0 Beta 2.exe"
self.uvc.addRelease("4.0b1",
build_id="222",
locales=["en-US", "ja", "zh-TW"],
patch_types=["complete"],
from_path="/firefox/Firefox 4.0 Beta 1.exe")
uvc2 = UpdateVerifyConfig()
uvc2.product = "Firefox"
uvc2.platform = "Linux_x86-gcc3"
uvc2.channel = "betatest"
uvc2.aus_server = "https://aus2.mozilla.org"
uvc2.ftp_server = "stage.mozilla.org/firefox"
uvc2.to = "/firefox/Firefox 4.0 Beta 2.exe"
uvc2.addRelease("4.0b1",
build_id="222",
locales=["en-US", "ja"],
patch_types=["complete"],
from_path="/firefox/Firefox 4.0 Beta 1.exe")
chunkedConfig = self.uvc.getChunk(chunks=2, thisChunk=1)
self.assertEquals(chunkedConfig, uvc2)
def testRead(self):
ftp_server_from = "stage.mozilla.org/firefox"
ftp_server_to = "stage.mozilla.org/firefox"
uvc2 = UpdateVerifyConfig()
uvc2.product = "Firefox"
uvc2.channel = "betatest"
uvc2.aus_server = "https://aus4.mozilla.org"
uvc2.to = "/firefox/4.0rc2.tar.bz2"
uvc2.addRelease("4.0", build_id="888", platform="Linux_x86-gcc3",
locales=["af", "de", "en-US", "ja", "zh-TW"],
patch_types=["partial", "complete"],
from_path="/firefox/4.0rc1.tar.bz2",
ftp_server_from=ftp_server_from,
ftp_server_to=ftp_server_to,
mar_channel_IDs="firefox-mozilla-beta")
uvc2.addRelease("4.0b12", build_id="777", platform="Linux_x86-gcc3",
locales=["af", "en-US"],
from_path="/firefox/4.0b12.tar.bz2",
ftp_server_from=ftp_server_from,
ftp_server_to=ftp_server_to)
uvc2.addRelease("4.0b12", build_id="777", platform="Linux_x86-gcc3",
locales=["de", "ja", "zh-TW"],
ftp_server_from=ftp_server_from,
ftp_server_to=ftp_server_to)
uvc2.addRelease("3.7a1", build_id="666", locales=["en-US"],
ftp_server_from=ftp_server_from,
ftp_server_to=ftp_server_to, platform="Linux_x86-gcc3")
self.uvc.read(self.config)
self.assertEquals(self.uvc, uvc2)
def testGetQuickReleaseTests(self):
ftp_server_from = "stage.mozilla.org/firefox"
ftp_server_to = "stage.mozilla.org/firefox"
self.uvc.read(self.config)
uvc2 = UpdateVerifyConfig()
uvc2.product = "Firefox"
uvc2.channel = "betatest"
uvc2.aus_server = "https://aus4.mozilla.org"
uvc2.to = "/firefox/4.0rc2.tar.bz2"
uvc2.to_build_id = "999"
uvc2.to_display_version = "99.0 Zeta 9"
uvc2.to_app_version = "99.0"
uvc2.addRelease("4.0b12",
build_id="777",
platform="Linux_x86-gcc3",
locales=["de", "ja", "zh-TW"],
patch_types=["complete"],
from_path=None,
ftp_server_from=ftp_server_from,
ftp_server_to=ftp_server_to)
uvc2.addRelease("3.7a1",
build_id="666",
platform="Linux_x86-gcc3",
locales=["en-US"],
patch_types=["complete"],
from_path=None,
ftp_server_from=ftp_server_from,
ftp_server_to=ftp_server_to)
self.assertEquals(self.uvc.getQuickReleaseTests(), uvc2.releases)
def testGetChunk(self):
ftp_server_from = "stage.mozilla.org/firefox"
ftp_server_to = "stage.mozilla.org/firefox"
self.uvc.read(self.config)
uvc2 = UpdateVerifyConfig()
uvc2.product = "Firefox"
uvc2.channel = "betatest"
uvc2.aus_server = "https://aus4.mozilla.org"
uvc2.to = "/firefox/4.0rc2.tar.bz2"
uvc2.to_build_id = "999"
uvc2.to_display_version = "99.0 Zeta 9"
uvc2.to_app_version = "99.0"
uvc2.addRelease("4.0",
build_id="888",
platform="Linux_x86-gcc3",
locales=["af", "de", "en-US"],
patch_types=["partial", "complete"],
from_path="/firefox/4.0rc1.tar.bz2",
ftp_server_from=ftp_server_from,
ftp_server_to=ftp_server_to,
mar_channel_IDs="firefox-mozilla-beta")
uvc2.addRelease("4.0b12",
build_id="777",
platform="Linux_x86-gcc3",
locales=["de", "ja"],
patch_types=["complete"],
ftp_server_from=ftp_server_from,
ftp_server_to=ftp_server_to,
from_path=None)
chunkedConfig = self.uvc.getChunk(chunks=3, thisChunk=1)
self.assertEquals(chunkedConfig, uvc2)
def testEq(self):
self.uvc.product = "foo"
self.uvc.channel = "betatest"
self.uvc.platform = "bar"
self.uvc.aus_server = "aus"
self.uvc.ftp_server = "ftp"
self.uvc.to = "/firefox/4.0rc2.tar.bz2"
uvc2 = UpdateVerifyConfig()
uvc2.product = "foo"
uvc2.channel = "betatest"
uvc2.platform = "bar"
uvc2.aus_server = "aus"
uvc2.ftp_server = "ftp"
uvc2.to = "/firefox/4.0rc2.tar.bz2"
self.assertEquals(self.uvc, uvc2)
def testEq(self):
self.uvc.product = "foo"
self.uvc.channel = "betatest"
self.uvc.aus_server = "aus"
self.uvc.ftp_server_from = "ftp"
self.uvc.ftp_server_to = "ftp"
self.uvc.to = "/firefox/4.0rc2.tar.bz2"
self.uvc.mar_channel_IDs = 'baz'
uvc2 = UpdateVerifyConfig()
uvc2.product = "foo"
uvc2.channel = "betatest"
uvc2.aus_server = "aus"
uvc2.ftp_server_form = "ftp"
uvc2.ftp_server_to = "ftp"
uvc2.to = "/firefox/4.0rc2.tar.bz2"
uvc2.mar_channel_IDs = 'baz'
self.assertEquals(self.uvc, uvc2)
def testGetFullReleaseTests(self):
self.uvc.read(self.config)
uvc2 = UpdateVerifyConfig()
uvc2.product = "Firefox"
uvc2.platform = "Linux_x86-gcc3"
uvc2.channel = "betatest"
uvc2.aus_server = "https://aus2.mozilla.org"
uvc2.ftp_server = "stage.mozilla.org/firefox"
uvc2.to = "/firefox/4.0rc2.tar.bz2"
uvc2.addRelease("4.0",
build_id="888",
locales=["af", "de", "en-US", "ja", "zh-TW"],
patch_types=["partial", "complete"],
from_path="/firefox/4.0rc1.tar.bz2")
uvc2.addRelease("4.0b12",
build_id="777",
locales=["af", "en-US"],
patch_types=["complete"],
from_path="/firefox/4.0b12.tar.bz2")
self.assertEquals(self.uvc.getFullReleaseTests(), uvc2.releases)
def testEq(self):
self.uvc.product = "foo"
self.uvc.channel = "betatest"
self.uvc.aus_server = "aus"
self.uvc.ftp_server_from = "ftp"
self.uvc.ftp_server_to = "ftp"
self.uvc.to = "/firefox/4.0rc2.tar.bz2"
self.uvc.mar_channel_IDs = 'baz'
self.uvc.to_build_id = "999"
self.uvc.to_display_version = "99.0 Zeta 9"
self.uvc.to_app_version = "99.0"
uvc2 = UpdateVerifyConfig()
uvc2.product = "foo"
uvc2.channel = "betatest"
uvc2.aus_server = "aus"
uvc2.ftp_server_form = "ftp"
uvc2.ftp_server_to = "ftp"
uvc2.to = "/firefox/4.0rc2.tar.bz2"
uvc2.mar_channel_IDs = 'baz'
uvc2.to_build_id = "999"
uvc2.to_display_version = "99.0 Zeta 9"
uvc2.to_app_version = "99.0"
self.assertEquals(self.uvc, uvc2)
def testNe(self):
self.uvc.product = "foo"
uvc2 = UpdateVerifyConfig()
# assertNotEqual doesn't test the __ne__ function, so we do this
self.assertTrue(self.uvc != uvc2)
def setUp(self):
self.uvc = UpdateVerifyConfig()
fd, self.tmpfilename = mkstemp()
self.tmpfile = os.fdopen(fd, "w")
def read_release_update_config(self):
'''
Builds a testing matrix based on an update verification configuration
file under the tools repository (release/updates/*.cfg).
Each release info line of the update verification files look similar to the following.
NOTE: This shows each pair of information as a new line but in reality
there is one white space separting them. We only show the values we care for.
release="38.0"
platform="Linux_x86_64-gcc3"
build_id="20150429135941"
locales="ach af ... zh-TW"
channel="beta-localtest"
from="/firefox/releases/38.0b9/linux-x86_64/%locale%/firefox-38.0b9.tar.bz2"
ftp_server_from="http://archive.mozilla.org/pub"
We will store this information in self.releases as a list of releases.
NOTE: We will talk of full and quick releases. Full release info normally contains a subset
of all locales (except for the most recent releases). A quick release has all locales,
however, it misses the fields 'from' and 'ftp_server_from'.
Both pairs of information complement each other but differ in such manner.
'''
dirs = self.query_abs_dirs()
assert os.path.exists(dirs['abs_tools_dir']), \
'Without the tools/ checkout we can\'t use releng\'s config parser.'
if self.config.get('release_update_config'):
# The config file is part of the tools repository. Make sure that if specified
# we force a revision of that repository to be set.
if self.tools_tag is None:
self.fatal('Make sure to specify the --tools-tag')
self.release_update_config = self.config['release_update_config']
# Import the config parser
sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib',
'python'))
from release.updates.verify import UpdateVerifyConfig
uvc = UpdateVerifyConfig()
config_file = os.path.join(dirs['abs_tools_dir'], 'release', 'updates',
self.config['release_update_config'])
uvc.read(config_file)
if not hasattr(self, 'update_channel'):
self.update_channel = uvc.channel
# Filter out any releases that are less than Gecko 38
uvc.releases = [
r for r in uvc.releases if int(r['release'].split('.')[0]) >= 38
]
temp_releases = []
for rel_info in uvc.releases:
# This is the full release info
if 'from' in rel_info and rel_info['from'] is not None:
# Let's find the associated quick release which contains the remaining locales
# for all releases except for the most recent release which contain all locales
quick_release = uvc.getRelease(build_id=rel_info['build_id'],
from_path=None)
if quick_release != {}:
rel_info['locales'] = sorted(rel_info['locales'] +
quick_release['locales'])
temp_releases.append(rel_info)
uvc.releases = temp_releases
chunked_config = uvc.getChunk(chunks=int(self.config['total_chunks']),
thisChunk=int(self.config['this_chunk']))
self.releases = chunked_config.releases
to_version = pc['current-update']['to']
to_ = makeReleaseRepackUrls(product_name,
app_name,
to_version,
options.platform,
locale='%locale%',
signed=True,
exclude_secondary=True).values()[0]
candidates_dir = makeCandidatesDir(product_name,
to_version,
build_number,
ftp_root='/')
to_path = "%s%s" % (candidates_dir, to_)
uvc = UpdateVerifyConfig(product=app_name,
channel=options.channel,
aus_server=aus_server_url,
to=to_path)
# getUpdatePaths yields all of the update paths, but we need to know
# everything about a fromVersion before we can add it to the update
# verify config, so we need to collect everything it yields before
# acting on it.
updatePaths = {}
for fromVersion, platform, locale, _, _ in pc.getUpdatePaths():
# Skip paths from platforms we don't care about.
if platform != ftp_platform:
continue
if fromVersion not in updatePaths:
updatePaths[fromVersion] = []
updatePaths[fromVersion].append(locale)
app_name,
to_version,
args.platform,
locale='%locale%',
signed=True,
exclude_secondary=True).values()[0]
candidates_dir = makeCandidatesDir(stage_product_name,
to_version,
build_number,
ftp_root='/')
to_path = "%s%s" % (candidates_dir, to_)
uvc = UpdateVerifyConfig(product=app_name,
channel=args.update_verify_channel,
aus_server=aus_server_url,
to=to_path,
to_build_id=to_release["platforms"][ftp_platform],
to_app_version=to_release["version"],
to_display_version=to_release["prettyVersion"])
# getUpdatePaths yields all of the update paths, but we need to know
# everything about a fromVersion before we can add it to the update
# verify config, so we need to collect everything it yields before
# acting on it.
updatePaths = {}
for fromVersion, platform, locale, _, _ in pc.getUpdatePaths():
# Skip paths from platforms we don't care about.
if platform != ftp_platform:
continue
if fromVersion not in updatePaths:
updatePaths[fromVersion] = []
thisChunk=None,
)
parser.add_option("--config-dict", dest="configDict")
parser.add_option("-t", "--release-tag", dest="releaseTag")
parser.add_option("-r", "--release-config", dest="releaseConfig")
parser.add_option("-b", "--buildbot-configs", dest="buildbotConfigs")
parser.add_option("-p", "--platform", dest="platform")
parser.add_option("--chunks", dest="chunks", type="int")
parser.add_option("--this-chunk", dest="thisChunk", type="int")
options, args = parser.parse_args()
mercurial(options.buildbotConfigs, "buildbot-configs")
update("buildbot-configs", revision=options.releaseTag)
releaseConfig = validate(options, args)
verifyConfigFile = releaseConfig[options.configDict][options.platform]
fd, configFile = mkstemp()
fh = os.fdopen(fd, "w")
try:
verifyConfig = UpdateVerifyConfig()
verifyConfig.read(path.join(UPDATE_VERIFY_DIR, verifyConfigFile))
myVerifyConfig = verifyConfig.getChunk(options.chunks,
options.thisChunk)
myVerifyConfig.write(fh)
fh.close()
run_cmd(["cat", configFile])
run_cmd(UPDATE_VERIFY_COMMAND + [configFile], cwd=UPDATE_VERIFY_DIR)
finally:
if path.exists(configFile):
os.unlink(configFile)
to_version = pc['current-update']['to']
to_ = makeReleaseRepackUrls(product_name,
app_name,
to_version,
args.platform,
locale='%locale%',
signed=True,
exclude_secondary=True).values()[0]
candidates_dir = makeCandidatesDir(product_name,
to_version,
build_number,
ftp_root='/')
to_path = "%s%s" % (candidates_dir, to_)
uvc = UpdateVerifyConfig(product=app_name,
channel=args.update_verify_channel,
aus_server=aus_server_url,
to=to_path)
# getUpdatePaths yields all of the update paths, but we need to know
# everything about a fromVersion before we can add it to the update
# verify config, so we need to collect everything it yields before
# acting on it.
updatePaths = {}
for fromVersion, platform, locale, _, _ in pc.getUpdatePaths():
# Skip paths from platforms we don't care about.
if platform != ftp_platform:
continue
if fromVersion not in updatePaths:
updatePaths[fromVersion] = []
updatePaths[fromVersion].append(locale)
def determine_testing_configuration(self):
'''
This method builds a testing matrix either based on an update verification
configuration file under the tools repo (release/updates/*.cfg)
OR it skips it when we use --installer-url --installer-path
Each release info line of the update verification files look similar to the following.
NOTE: This shows each pair of information as a new line but in reality
there is one white space separting them. We only show the values we care for.
release="38.0"
platform="Linux_x86_64-gcc3"
build_id="20150429135941"
locales="ach af ... zh-TW"
channel="beta-localtest"
from="/firefox/releases/38.0b9/linux-x86_64/%locale%/firefox-38.0b9.tar.bz2"
ftp_server_from="http://stage.mozilla.org/pub/mozilla.org"
We will store this information in self.releases as a list of releases.
NOTE: We will talk of full and quick releases. Full release info normally contains a subset
of all locales (except for the most recent releases). A quick release has all locales,
however, it misses the fields 'from' and 'ftp_server_from'.
Both pairs of information complement each other but differ in such manner.
'''
if self.installer_url or self.installer_path:
return
dirs = self.query_abs_dirs()
assert os.path.exists(dirs['abs_tools_dir']), \
"Without the tools/ checkout we can't use releng's config parser."
# Import the config parser
sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib',
'python'))
from release.updates.verify import UpdateVerifyConfig
uvc = UpdateVerifyConfig()
uvc.read(self.updates_config_file)
self.channel = uvc.channel
# Filter out any releases that are less than Gecko 38
uvc.releases = [r for r in uvc.releases \
if int(r["release"].split('.')[0]) >= 38]
temp_releases = []
for rel_info in uvc.releases:
# This is the full release info
if 'from' in rel_info and rel_info['from'] is not None:
# Let's find the associated quick release which contains the remaining locales
# for all releases except for the most recent release which contain all locales
quick_release = uvc.getRelease(build_id=rel_info['build_id'],
from_path=None)
if quick_release != {}:
rel_info['locales'] = sorted(rel_info['locales'] +
quick_release['locales'])
temp_releases.append(rel_info)
uvc.releases = temp_releases
chunked_config = uvc.getChunk(chunks=int(self.config['total_chunks']),
thisChunk=int(self.config['this_chunk']))
self.releases = chunked_config.releases
