def query_manifest(self, device_name):
if device_name in self.device_manifests:
return self.device_manifests[device_name]
dirs = self.query_abs_dirs()
c = self.config
manifest_file = c["devices"][device_name].get("manifest_file", "%s.xml" % device_name)
manifest_file = os.path.join(dirs["manifests_dir"], manifest_file)
self.info("Loading %s" % manifest_file)
manifest = repo_manifest.load_manifest(manifest_file)
self.device_manifests[device_name] = manifest
return manifest
def query_manifest(self, device_name):
if device_name in self.device_manifests:
return self.device_manifests[device_name]
dirs = self.query_abs_dirs()
c = self.config
manifest_file = c['devices'][device_name].get('manifest_file',
'%s.xml' % device_name)
manifest_file = os.path.join(dirs['manifests_dir'], manifest_file)
self.info("Loading %s" % manifest_file)
manifest = repo_manifest.load_manifest(manifest_file)
self.device_manifests[device_name] = manifest
return manifest
def query_manifest(self, device_name):
if device_name in self.device_manifests:
return self.device_manifests[device_name]
dirs = self.query_abs_dirs()
c = self.config
manifest_file = c['devices'][device_name].get('manifest_file',
'%s.xml' % device_name)
manifest_file = os.path.join(dirs['manifests_dir'], manifest_file)
self.info("Loading %s" % manifest_file)
manifest = repo_manifest.load_manifest(manifest_file)
self.device_manifests[device_name] = manifest
return manifest
def query_manifest(self):
if self.manifest:
return self.manifest
dirs = self.query_abs_dirs()
c = self.config
manifest_file = c.get('manifest_file')
if not manifest_file:
manifest_file = os.path.join(dirs['manifests_dir'],
c['manifest_name'] + ".xml")
self.info("Loading %s" % manifest_file)
self.manifest = repo_manifest.load_manifest(manifest_file)
return self.manifest
def query_manifest(self):
if self.manifest:
return self.manifest
dirs = self.query_abs_dirs()
c = self.config
manifest_file = c.get('manifest_file')
if not manifest_file:
manifest_file = os.path.join(dirs['manifests_dir'],
c['manifest_name'] + ".xml")
self.info("Loading %s" % manifest_file)
self.manifest = repo_manifest.load_manifest(manifest_file)
return self.manifest
def checkout_sources(self):
dirs = self.query_abs_dirs()
gecko_config = self.load_gecko_config()
b2g_manifest_intree = gecko_config.get('b2g_manifest_intree')
if gecko_config.get('config_version') >= 2:
repos = [
{'vcs': 'gittool', 'repo': 'https://git.mozilla.org/b2g/B2G.git', 'dest': dirs['work_dir']},
]
if b2g_manifest_intree:
# Checkout top-level B2G repo now
self.vcs_checkout_repos(repos)
b2g_manifest_branch = 'master'
# That may have blown away our build-tools checkout. It would
# be better if B2G were checked out into a subdirectory, but
# for now, just redo it.
self.checkout_tools()
# Now checkout gecko inside the build directory
self.checkout_gecko()
conf_dir = os.path.join(dirs['gecko_src'], os.path.dirname(self.query_gecko_config_path()))
manifest_filename = os.path.join(conf_dir, 'sources.xml')
self.info("Using manifest at %s" % manifest_filename)
have_gecko = True
else:
# Checkout B2G and b2g-manifests. We'll do gecko later
b2g_manifest_branch = gecko_config.get('b2g_manifest_branch', 'master')
repos.append(
{'vcs': 'gittool',
'repo': 'https://git.mozilla.org/b2g/b2g-manifest.git',
'dest': os.path.join(dirs['work_dir'], 'b2g-manifest'),
'branch': b2g_manifest_branch},
)
manifest_filename = gecko_config.get('b2g_manifest', self.config['target'] + '.xml')
manifest_filename = os.path.join(dirs['work_dir'], 'b2g-manifest', manifest_filename)
self.vcs_checkout_repos(repos)
have_gecko = False
manifest = load_manifest(manifest_filename)
if not b2g_manifest_intree:
# Now munge the manifest by mapping remotes to local remotes
mapping_func = functools.partial(map_remote, mappings=self.config['repo_remote_mappings'])
rewrite_remotes(manifest, mapping_func)
# Remove gecko, since we'll be checking that out ourselves
gecko_node = remove_project(manifest, path='gecko')
if not gecko_node:
self.fatal("couldn't remove gecko from manifest")
# Write out our manifest locally
manifest_dir = os.path.join(dirs['work_dir'], 'tmp_manifest')
self.rmtree(manifest_dir)
self.mkdir_p(manifest_dir)
manifest_filename = os.path.join(manifest_dir, self.config['target'] + '.xml')
self.info("Writing manifest to %s" % manifest_filename)
manifest_file = open(manifest_filename, 'w')
manifest.writexml(manifest_file)
manifest_file.close()
# Set up repo
repo_link = os.path.join(dirs['work_dir'], '.repo')
if 'repo_mirror_dir' in self.config:
# Make our local .repo directory a symlink to the shared repo
# directory
repo_mirror_dir = self.config['repo_mirror_dir']
self.mkdir_p(repo_mirror_dir)
repo_link = os.path.join(dirs['work_dir'], '.repo')
if not os.path.exists(repo_link) or not os.path.islink(repo_link):
self.rmtree(repo_link)
self.info("Creating link from %s to %s" % (repo_link, repo_mirror_dir))
os.symlink(repo_mirror_dir, repo_link)
# Checkout the repo tool
if 'repo_repo' in self.config:
repo_dir = os.path.join(dirs['work_dir'], '.repo', 'repo')
self.checkout_repotool(repo_dir)
cmd = ['./repo', '--version']
if not self.run_command(cmd, cwd=dirs['work_dir']) == 0:
# Set return code to RETRY
self.fatal("repo is broken", exit_code=4)
# Check it out!
max_tries = 5
sleep_time = 60
max_sleep_time = 300
for _ in range(max_tries):
# If .repo points somewhere, then try and reset our state
# before running config.sh
if os.path.isdir(repo_link):
# Delete any projects with broken HEAD references
self.info("Deleting broken projects...")
cmd = ['./repo', 'forall', '-c', 'git show-ref -q --head HEAD || rm -rfv $PWD']
self.run_command(cmd, cwd=dirs['work_dir'])
config_result = self.run_command([
'./config.sh', '-q', self.config['target'], manifest_filename,
], cwd=dirs['work_dir'], output_timeout=55 * 60)
# TODO: Check return code from these? retry?
# Run git reset --hard to make sure we're in a clean state
self.info("Resetting all git projects")
cmd = ['./repo', 'forall', '-c', 'git reset --hard']
self.run_command(cmd, cwd=dirs['work_dir'])
self.info("Cleaning all git projects")
cmd = ['./repo', 'forall', '-c', 'git clean -f -x -d']
self.run_command(cmd, cwd=dirs['work_dir'])
if config_result == 0:
break
else:
# We may have died due to left-over lock files. Make sure
# we clean those up before trying again.
self.info("Deleting stale lock files")
cmd = ['find', '.repo/', '-name', '*.lock', '-print', '-delete']
self.run_command(cmd, cwd=dirs['work_dir'])
# Try again in a bit. Broken clones should be deleted and
# re-tried above
self.info("config.sh failed; sleeping %i and retrying" % sleep_time)
time.sleep(sleep_time)
# Exponential backoff with random jitter
sleep_time = min(sleep_time * 1.5, max_sleep_time) + random.randint(1, 60)
else:
self.fatal("failed to run config.sh")
# Workaround bug 985837
if self.config['target'] == 'emulator-kk':
self.info("Forcing -j4 for emulator-kk")
dotconfig_file = os.path.join(dirs['abs_work_dir'], '.config')
with open(dotconfig_file, "a+") as f:
f.write("\nMAKE_FLAGS=-j1\n")
# output our sources.xml, make a copy for update_sources_xml()
self.run_command(
["./gonk-misc/add-revision.py", "-o", "sources.xml", "--force",
".repo/manifest.xml"], cwd=dirs["work_dir"],
halt_on_failure=True, fatal_exit_code=3)
self.run_command(["cat", "sources.xml"], cwd=dirs['work_dir'], halt_on_failure=True, fatal_exit_code=3)
self.run_command(["cp", "-p", "sources.xml", "sources.xml.original"], cwd=dirs['work_dir'], halt_on_failure=True, fatal_exit_code=3)
manifest = load_manifest(os.path.join(dirs['work_dir'], 'sources.xml'))
gaia_node = get_project(manifest, path="gaia")
gaia_rev = gaia_node.getAttribute("revision")
gaia_remote = get_remote(manifest, gaia_node.getAttribute('remote'))
gaia_repo = "%s/%s" % (gaia_remote.getAttribute('fetch'), gaia_node.getAttribute('name'))
gaia_url = self.query_gitweb_url(gaia_repo, gaia_rev)
self.set_buildbot_property("gaia_revision", gaia_rev, write_to_file=True)
self.info("TinderboxPrint: gaia_revlink: %s" % gaia_url)
# Now we can checkout gecko and other stuff
if not have_gecko:
self.checkout_gecko()
return
# Old behaviour
self.checkout_gecko()
self.checkout_gaia()
def munge_manifests(self):
""" Switch the branched repos to the new branch; lock down third
party revisions.
"""
branch_repos = self.query_branch_repos()
dirs = self.query_abs_dirs()
new_branch = self.config['branch_name']
unused_manifests = []
if not self.check_existing_branch(new_branch, cwd=dirs['abs_manifest_dir'])[0]:
self.fatal("b2g-manifest isn't branched properly! Run --clean-repos --branch-repos")
for manifest in self.query_manifests():
self.info("Munging %s..." % manifest)
doc = repo_manifest.load_manifest(manifest)
try:
repo_manifest.get_default(doc).getAttribute("revision")
except IndexError:
self.info("No default revision; skipping.")
unused_manifests.append(manifest)
continue
for p in doc.getElementsByTagName('project'):
name = self._query_repo_name(p.getAttribute('name'))
fetch = repo_manifest.get_project_remote_url(doc, p)
self.debug("Remote %s Name %s" % (fetch, name))
current_revision = repo_manifest.get_project_revision(doc, p)
if repo_manifest.is_commitid(current_revision):
self.info("%s: %s is already locked to %s; skipping." % (manifest, name, current_revision))
# I could setAttribute() here, but I very much doubt the
# default_revision is a commitid.
continue
# We've branched this repo; do we set the revision to
# new_branch or not? ('fetch' needs to match, since we have
# same-named repos with different urls =P )
if name in branch_repos and branch_repos[name]['fetch'] == fetch:
orig_branch = branch_repos[name]['branch_revisions'].keys()[0]
if manifest in branch_repos[name]['branch_revisions'][orig_branch]:
if current_revision != orig_branch:
self.fatal("I don't know how we got here, but %s in %s's revision %s is not the branching point %s." %
(name, manifest, current_revision, orig_branch))
self.info("Setting %s (%s) to %s (was %s)" % (name, manifest, new_branch, current_revision))
p.setAttribute('revision', new_branch)
continue
# Should we keep the old branch or lock revision? Doing
# the former for now.
self.info("%s %s is off a different branch point (%s, not %s). Keeping the old branch..." %
(manifest, name, current_revision, orig_branch))
continue
if name in self.config['extra_branch_manifest_repos']:
p.setAttribute('revision', new_branch)
continue
# Lock revision?
if not self.config["lock_manifest_revisions"]:
self.info("%s: Not locking revision for %s due to config." % (manifest, name))
continue
lock_revision = self._query_remote_branch_revision(fetch, current_revision, manifest)
if lock_revision is not None:
p.setAttribute('revision', lock_revision)
with self.opened(manifest, open_mode='w') as (fh, err):
if err:
self.fatal("Can't open %s for writing!" % manifest)
else:
doc.writexml(fh)
fh.close()
if self.config["delete_unused_manifests"]:
self._delete_unused_manifests(unused_manifests)
self.info("TODO: diff, commit, --push!")
def query_branch_repos(self):
""" Parse all manifests and build a dictionary of repos with
expected revisions and/or branches.
The format will be {
name: {
'revision': branch,
'fetch': git_url,
'branch_revisions': {
revision: [manifest list] # This should only have one key/value pair
},
'all_revisions': {
revision: [manifest list] # This will have all key/value pairs
},
},
}
This depends on the pull action having run at least once.
"""
if self.branch_repo_dict is not None:
return self.branch_repo_dict
self.info("Building branch_repo_dict...")
branch_repo_dict = {
'b2g-manifest': {
'fetch': self.config['manifest_repo_url'],
'revision': self.config['manifest_repo_revision'],
},
}
for manifest in self.query_manifests():
self.info("Processing %s" % manifest)
doc = repo_manifest.load_manifest(manifest)
try:
default_revision = repo_manifest.get_default(doc).getAttribute("revision")
except IndexError:
self.info("Skipping %s (no defaults)" % manifest)
continue
for p in doc.getElementsByTagName('project'):
name = self._query_repo_name(p.getAttribute('name'))
fetch = repo_manifest.get_project_remote_url(doc, p)
self.debug("Remote %s Name %s" % (fetch, name))
# We branch github.com/mozilla repos only
if not self._query_do_branch(fetch, name):
continue
# Now start building the branch info
branch_repo_dict.setdefault(name, {}).setdefault("fetch", fetch)
revision = p.getAttribute("revision")
if not revision:
# fall back to default revision
if default_revision:
self.info("%s: %s, using default revision (%s)" % (manifest, name, default_revision))
self._add_branch_revision(branch_repo_dict[name], name, default_revision, manifest)
else:
self.warning("Can't determine revision for %s in %s" % (name, manifest))
elif revision and branch_repo_dict.get(name, {}).get("revision", revision) != revision:
self._add_branch_revision(branch_repo_dict[name], name, revision, manifest)
else:
self._add_branch_revision(branch_repo_dict[name], name, revision, manifest)
self.info("Outputting branch_repo_dict:")
self.info(pprint.pformat(branch_repo_dict))
message = ""
for name, r in branch_repo_dict.iteritems():
if r.get("revision") is None:
self.warning("Sanity: No revision set for %s %s; we'll fall back to master" % (name, r["fetch"]))
if len(r.get("branch_revisions", {})) > 1:
message += "Sanity: Not clear where to branch for %s %s %s\n" % (name, r["fetch"], pprint.pformat(r["branch_revisions"]))
if message:
self.fatal(message + "Use --branch-order or self.config['no_branch_repos'] to fix!")
self.branch_repo_dict = branch_repo_dict
return branch_repo_dict
def checkout_sources(self):
dirs = self.query_abs_dirs()
gecko_config = self.load_gecko_config()
b2g_manifest_intree = gecko_config.get('b2g_manifest_intree')
b2g_repo = gecko_config.get('b2g_repo',
'https://github.com/mozilla-b2g/B2G.git')
b2g_branch = gecko_config.get('b2g_branch', 'master')
if gecko_config.get('config_version') >= 2:
repos = [
{
'vcs': 'gittool',
'repo': b2g_repo,
'branch': b2g_branch,
'dest': dirs['work_dir']
},
]
if b2g_manifest_intree:
# Checkout top-level B2G repo now
self.vcs_checkout_repos(repos)
b2g_manifest_branch = 'master'
# That may have blown away our build-tools checkout. It would
# be better if B2G were checked out into a subdirectory, but
# for now, just redo it.
self.checkout_tools()
# Now checkout gecko inside the build directory
self.checkout_gecko()
conf_dir = os.path.join(
dirs['gecko_src'],
os.path.dirname(self.query_gecko_config_path()))
manifest_filename = os.path.join(conf_dir, 'sources.xml')
self.info("Using manifest at %s" % manifest_filename)
have_gecko = True
else:
# Checkout B2G and b2g-manifests. We'll do gecko later
b2g_manifest_branch = gecko_config.get('b2g_manifest_branch',
'master')
repos.append(
{
'vcs': 'gittool',
'repo': 'https://github.com/mozilla-b2g/b2g-manifest',
'dest': os.path.join(dirs['work_dir'], 'b2g-manifest'),
'branch': b2g_manifest_branch
}, )
manifest_filename = gecko_config.get(
'b2g_manifest', self.config['target'] + '.xml')
manifest_filename = os.path.join(dirs['work_dir'],
'b2g-manifest',
manifest_filename)
self.vcs_checkout_repos(repos)
have_gecko = False
manifest = load_manifest(manifest_filename)
if not b2g_manifest_intree:
# Now munge the manifest by mapping remotes to local remotes
mapping_func = functools.partial(
map_remote, mappings=self.config['repo_remote_mappings'])
rewrite_remotes(manifest, mapping_func)
# Remove gecko, since we'll be checking that out ourselves
gecko_node = remove_project(manifest, path='gecko')
if not gecko_node:
self.fatal("couldn't remove gecko from manifest")
# Write out our manifest locally
manifest_dir = os.path.join(dirs['work_dir'], 'tmp_manifest')
self.rmtree(manifest_dir)
self.mkdir_p(manifest_dir)
manifest_filename = os.path.join(manifest_dir,
self.config['target'] + '.xml')
self.info("Writing manifest to %s" % manifest_filename)
manifest_file = open(manifest_filename, 'w')
manifest.writexml(manifest_file)
manifest_file.close()
# Set up repo
repo_link = os.path.join(dirs['work_dir'], '.repo')
if 'repo_mirror_dir' in self.config:
# Make our local .repo directory a symlink to the shared repo
# directory
repo_mirror_dir = self.config['repo_mirror_dir']
self.mkdir_p(repo_mirror_dir)
repo_link = os.path.join(dirs['work_dir'], '.repo')
if not os.path.exists(repo_link) or not os.path.islink(
repo_link):
self.rmtree(repo_link)
self.info("Creating link from %s to %s" %
(repo_link, repo_mirror_dir))
os.symlink(repo_mirror_dir, repo_link)
# Checkout the repo tool
if 'repo_repo' in self.config:
repo_dir = os.path.join(dirs['work_dir'], '.repo', 'repo')
self.checkout_repotool(repo_dir)
cmd = ['./repo', '--version']
if not self.run_command(cmd, cwd=dirs['work_dir']) == 0:
# Set return code to RETRY
self.fatal("repo is broken", exit_code=4)
# Check it out!
max_tries = 5
sleep_time = 60
max_sleep_time = 300
for _ in range(max_tries):
# If .repo points somewhere, then try and reset our state
# before running config.sh
if os.path.isdir(repo_link):
# Delete any projects with broken HEAD references
self.info("Deleting broken projects...")
cmd = [
'./repo', 'forall', '-c',
'git show-ref -q --head HEAD || rm -rfv $PWD'
]
self.run_command(cmd, cwd=dirs['work_dir'])
# bug https://bugzil.la/1177190 - workaround - change
# timeout from 55 to 10 min, based on "normal" being
# about 7.5 minutes
config_result = self.run_command([
'./config.sh',
'-q',
self.config['target'],
manifest_filename,
],
cwd=dirs['work_dir'],
output_timeout=10 * 60)
# TODO: Check return code from these? retry?
# Run git reset --hard to make sure we're in a clean state
self.info("Resetting all git projects")
cmd = ['./repo', 'forall', '-c', 'git reset --hard']
self.run_command(cmd, cwd=dirs['work_dir'])
self.info("Cleaning all git projects")
cmd = ['./repo', 'forall', '-c', 'git clean -f -x -d']
self.run_command(cmd, cwd=dirs['work_dir'])
if config_result == 0:
break
else:
# We may have died due to left-over lock files. Make sure
# we clean those up before trying again.
self.info("Deleting stale lock files")
cmd = [
'find', '.repo/', '-name', '*.lock', '-print',
'-delete'
]
self.run_command(cmd, cwd=dirs['work_dir'])
# Try again in a bit. Broken clones should be deleted and
# re-tried above
self.info("config.sh failed; sleeping %i and retrying" %
sleep_time)
time.sleep(sleep_time)
# Exponential backoff with random jitter
sleep_time = min(sleep_time * 1.5,
max_sleep_time) + random.randint(1, 60)
else:
self.fatal("failed to run config.sh")
# output our sources.xml, make a copy for update_sources_xml()
self.run_command([
"./gonk-misc/add-revision.py", "-o", "sources.xml", "--force",
".repo/manifest.xml"
],
cwd=dirs["work_dir"],
halt_on_failure=True,
fatal_exit_code=3)
self.run_command(["cat", "sources.xml"],
cwd=dirs['work_dir'],
halt_on_failure=True,
fatal_exit_code=3)
self.run_command(
["cp", "-p", "sources.xml", "sources.xml.original"],
cwd=dirs['work_dir'],
halt_on_failure=True,
fatal_exit_code=3)
manifest = load_manifest(
os.path.join(dirs['work_dir'], 'sources.xml'))
gaia_node = get_project(manifest, path="gaia")
gaia_rev = gaia_node.getAttribute("revision")
gaia_remote = get_remote(manifest,
gaia_node.getAttribute('remote'))
gaia_repo = "%s/%s" % (gaia_remote.getAttribute('fetch'),
gaia_node.getAttribute('name'))
gaia_url = self.query_gitweb_url(gaia_repo, gaia_rev)
self.set_buildbot_property("gaia_revision",
gaia_rev,
write_to_file=True)
self.info("TinderboxPrint: gaia_revlink: %s" % gaia_url)
# Now we can checkout gecko and other stuff
if not have_gecko:
self.checkout_gecko()
return
# Old behaviour
self.checkout_gecko()
self.checkout_gaia()
def munge_manifests(self):
""" Switch the branched repos to the new branch; lock down third
party revisions.
"""
branch_repos = self.query_branch_repos()
dirs = self.query_abs_dirs()
new_branch = self.config['branch_name']
unused_manifests = []
if not self.check_existing_branch(new_branch, cwd=dirs['abs_manifest_dir'])[0]:
self.fatal("b2g-manifest isn't branched properly! Run --clean-repos --branch-repos")
for manifest in self.query_manifests():
self.info("Munging %s..." % manifest)
doc = repo_manifest.load_manifest(manifest)
try:
repo_manifest.get_default(doc).getAttribute("revision")
except IndexError:
self.info("No default revision; skipping.")
unused_manifests.append(manifest)
continue
for p in doc.getElementsByTagName('project'):
name = self._query_repo_name(p.getAttribute('name'))
fetch = repo_manifest.get_project_remote_url(doc, p)
self.debug("Remote %s Name %s" % (fetch, name))
current_revision = repo_manifest.get_project_revision(doc, p)
if repo_manifest.is_commitid(current_revision):
self.info("%s: %s is already locked to %s; skipping." % (manifest, name, current_revision))
# I could setAttribute() here, but I very much doubt the
# default_revision is a commitid.
continue
# We've branched this repo; do we set the revision to
# new_branch or not? ('fetch' needs to match, since we have
# same-named repos with different urls =P )
if name in branch_repos and branch_repos[name]['fetch'] == fetch:
orig_branch = branch_repos[name]['branch_revisions'].keys()[0]
if manifest in branch_repos[name]['branch_revisions'][orig_branch]:
if current_revision != orig_branch:
self.fatal("I don't know how we got here, but %s in %s's revision %s is not the branching point %s." %
(name, manifest, current_revision, orig_branch))
self.info("Setting %s (%s) to %s (was %s)" % (name, manifest, new_branch, current_revision))
p.setAttribute('revision', new_branch)
continue
# Should we keep the old branch or lock revision? Doing
# the former for now.
self.info("%s %s is off a different branch point (%s, not %s). Keeping the old branch..." %
(manifest, name, current_revision, orig_branch))
continue
if name in self.config['extra_branch_manifest_repos']:
p.setAttribute('revision', new_branch)
continue
# Lock revision?
if not self.config["lock_manifest_revisions"]:
self.info("%s: Not locking revision for %s due to config." % (manifest, name))
continue
lock_revision = self._query_remote_branch_revision(fetch, current_revision, manifest)
if lock_revision is not None:
p.setAttribute('revision', lock_revision)
with self.opened(manifest, open_mode='w') as (fh, err):
if err:
self.fatal("Can't open %s for writing!" % manifest)
else:
doc.writexml(fh)
fh.close()
if self.config["delete_unused_manifests"]:
self._delete_unused_manifests(unused_manifests)
self.info("TODO: diff, commit, --push!")
def query_branch_repos(self):
""" Parse all manifests and build a dictionary of repos with
expected revisions and/or branches.
The format will be {
name: {
'revision': branch,
'fetch': git_url,
'branch_revisions': {
revision: [manifest list] # This should only have one key/value pair
},
'all_revisions': {
revision: [manifest list] # This will have all key/value pairs
},
},
}
This depends on the pull action having run at least once.
"""
if self.branch_repo_dict is not None:
return self.branch_repo_dict
self.info("Building branch_repo_dict...")
branch_repo_dict = {
'b2g-manifest': {
'fetch': self.config['manifest_repo_url'],
'revision': self.config['manifest_repo_revision'],
},
}
for manifest in self.query_manifests():
self.info("Processing %s" % manifest)
doc = repo_manifest.load_manifest(manifest)
try:
default_revision = repo_manifest.get_default(doc).getAttribute("revision")
except IndexError:
self.info("Skipping %s (no defaults)" % manifest)
continue
for p in doc.getElementsByTagName('project'):
name = self._query_repo_name(p.getAttribute('name'))
fetch = repo_manifest.get_project_remote_url(doc, p)
self.debug("Remote %s Name %s" % (fetch, name))
# We branch github.com/mozilla repos only
if not self._query_do_branch(fetch, name):
continue
# Now start building the branch info
branch_repo_dict.setdefault(name, {}).setdefault("fetch", fetch)
revision = p.getAttribute("revision")
if not revision:
# fall back to default revision
if default_revision:
self.info("%s: %s, using default revision (%s)" % (manifest, name, default_revision))
self._add_branch_revision(branch_repo_dict[name], name, default_revision, manifest)
else:
self.warning("Can't determine revision for %s in %s" % (name, manifest))
elif revision and branch_repo_dict.get(name, {}).get("revision", revision) != revision:
self._add_branch_revision(branch_repo_dict[name], name, revision, manifest)
else:
self._add_branch_revision(branch_repo_dict[name], name, revision, manifest)
self.info("Outputting branch_repo_dict:")
self.info(pprint.pformat(branch_repo_dict))
message = ""
for name, r in branch_repo_dict.iteritems():
if r.get("revision") is None:
self.warning("Sanity: No revision set for %s %s; we'll fall back to master" % (name, r["fetch"]))
if len(r.get("branch_revisions", {})) > 1:
message += "Sanity: Not clear where to branch for %s %s %s\n" % (name, r["fetch"], pprint.pformat(r["branch_revisions"]))
if message:
self.fatal(message + "Use --branch-order or self.config['no_branch_repos'] to fix!")
self.branch_repo_dict = branch_repo_dict
return branch_repo_dict