def build_files(self):
with open(os.path.join(self.repo_dir, '.mozconfig'), 'w') as f:
f.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/obj-firefox\n')
f.write('ac_add_options --enable-debug\n')
f.write('ac_add_options --enable-artifact-builds\n')
retry(lambda: run_check(['gecko-env', './mach', 'build'], cwd=self.repo_dir))
retry(lambda: run_check(['gecko-env', './mach', 'build-backend', '-b', 'ChromeMap'], cwd=self.repo_dir))
logger.info('Build successful')
self.build_finished = True
with self.build_finished_cv:
self.build_finished_cv.notify_all()
def test_retry():
assert utils.retry(lambda: True)
assert utils.retry(lambda: False)
assert not utils.retry(do_raise, wait_between_retries=0)
i = {}
def try_twice():
if 'tried' in i:
return
else:
i['tried'] = True
raise Exception('Please try again.')
assert utils.retry(try_twice, wait_between_retries=0)
def go(self):
with ThreadPoolExecutorResult(max_workers=2) as executor:
# Thread 1 - Download coverage artifacts.
executor.submit(lambda: self.download_coverage_artifacts())
# Thread 2 - Clone and build mozilla-central
clone_future = executor.submit(lambda: self.clone_mozilla_central(self.revision))
# Make sure cloning mozilla-central didn't fail before building.
clone_future.add_done_callback(lambda f: f.result())
# Now we can build.
clone_future.add_done_callback(lambda f: self.build_files())
if self.from_pulse:
if self.gecko_dev_user is not None and self.gecko_dev_pwd is not None:
self.update_github_repo()
commit_sha = self.get_github_commit(self.revision)
logger.info('GitHub revision', revision=commit_sha)
if self.gecko_dev_user is not None and self.gecko_dev_pwd is not None:
self.post_github_status(commit_sha)
output = self.generate_info(commit_sha)
logger.info('Report generated successfully')
with ThreadPoolExecutorResult(max_workers=2) as executor:
executor.submit(lambda: uploader.coveralls(output))
executor.submit(lambda: uploader.codecov(output, commit_sha, self.codecov_token))
self.prepopulate_cache(commit_sha)
else:
mkdir('code-coverage-reports')
self.generate_per_suite_reports()
self.generate_zero_coverage_report()
self.generate_chunk_mapping()
os.chdir('code-coverage-reports')
run_check(['git', 'config', '--global', 'http.postBuffer', '12M'])
run_check(['git', 'config', '--global', 'user.email', '*****@*****.**'])
run_check(['git', 'config', '--global', 'user.name', 'Report Uploader'])
repo_url = 'https://%s:%[email protected]/marco-c/code-coverage-reports' % (self.gecko_dev_user, self.gecko_dev_pwd)
run_check(['git', 'init'])
run_check(['git', 'add', '*'])
run_check(['git', 'commit', '-m', 'Coverage reports upload'])
retry(lambda: run_check(['git', 'push', repo_url, 'master', '--force']))
def update_github_repo(self):
run_check(['git', 'config', '--global', 'http.postBuffer', '12M'])
repo_url = 'https://%s:%[email protected]/marco-c/gecko-dev' % (self.gecko_dev_user, self.gecko_dev_pwd)
repo_path = os.path.join(self.cache_root, 'gecko-dev')
if not os.path.isdir(repo_path):
retry(lambda: run_check(['git', 'clone', repo_url], cwd=self.cache_root))
retry(lambda: run_check(['git', 'pull', 'https://github.com/mozilla/gecko-dev', 'master'], cwd=repo_path))
retry(lambda: run_check(['git', 'push', repo_url, 'master'], cwd=repo_path))
def clone_mozilla_central(self, revision):
shared_dir = self.repo_dir + '-shared'
cmd = hglib.util.cmdbuilder('robustcheckout',
'https://hg.mozilla.org/mozilla-central',
self.repo_dir,
purge=True,
sharebase=shared_dir,
branch=b'tip')
cmd.insert(0, hglib.HGPATH)
def do_clone():
proc = hglib.util.popen(cmd)
out, err = proc.communicate()
if proc.returncode:
raise hglib.error.CommandError(cmd, proc.returncode, out, err)
hg = hglib.open(self.repo_dir)
hg.update(rev=revision, clean=True)
retry(lambda: do_clone())
logger.info('mozilla-central cloned')
def download_artifact(task_id, suite, artifact):
artifact_path = 'ccov-artifacts/' + task_id + '_' + suite + '_' + os.path.basename(
artifact['name'])
if os.path.exists(artifact_path):
return artifact_path
def perform_download():
r = requests.get(queue_base + 'task/' + task_id + '/artifacts/' +
artifact['name'],
stream=True)
with open(artifact_path, 'wb') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)
if not retry(perform_download):
raise Exception('Failed downloading artifact in %s' % artifact_path)
return artifact_path