def go(self): with ThreadPoolExecutorResult(max_workers=2) as executor: # Thread 1 - Download coverage artifacts. executor.submit(self.artifactsHandler.download_all) # Thread 2 - Clone mozilla-central. executor.submit(self.clone_mozilla_central, self.revision) if self.from_pulse: self.githubUtils.update_geckodev_repo() commit_sha = self.githubUtils.get_commit(self.revision) logger.info('GitHub revision', revision=commit_sha) self.githubUtils.post_github_status(commit_sha) r = requests.get( 'https://hg.mozilla.org/mozilla-central/json-rev/%s' % self.revision) r.raise_for_status() push_id = r.json()['pushid'] output = grcov.report(self.artifactsHandler.get(), source_dir=self.repo_dir, service_number=push_id, commit_sha=commit_sha, token=secrets[secrets.COVERALLS_TOKEN]) logger.info('Report generated successfully') with ThreadPoolExecutorResult(max_workers=2) as executor: executor.submit(uploader.coveralls, output) executor.submit(uploader.codecov, output, commit_sha) logger.info('Waiting for build to be ingested by Codecov...') # Wait until the build has been ingested by Codecov. if uploader.codecov_wait(commit_sha): logger.info('Build ingested by codecov.io') self.notifier.notify() else: logger.info('codecov.io took too much time to ingest data.') else: mkdir('code-coverage-reports') # XXX: Disabled as it is unused for now. # self.generate_suite_reports() report_generators.zero_coverage(self.artifactsHandler.get()) self.generate_chunk_mapping() os.chdir('code-coverage-reports') self.githubUtils.update_codecoveragereports_repo()
def retrieve_source_and_artifacts(self): with ThreadPoolExecutorResult(max_workers=2) as executor: # Thread 1 - Download coverage artifacts. executor.submit(self.artifactsHandler.download_all) # Thread 2 - Clone repository. executor.submit(self.clone_repository, self.repository, self.revision)
def download_all(self): os.makedirs(self.parent_dir, exist_ok=True) # The test tasks for the Linux and Windows builds are in the same group, # but the following code is generic and supports build tasks split in # separate groups. groups = set([ taskcluster.get_task_details(build_task_id)['taskGroupId'] for build_task_id in self.task_ids.values() ]) test_tasks = [ task for group in groups for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task) ] for test_task in test_tasks: status = test_task['status']['state'] while status not in FINISHED_STATUSES: assert status in ALL_STATUSES, "State '{}' not recognized".format( status) logger.info('Waiting for task {} to finish...'.format( test_task['status']['taskId'])) time.sleep(60) status = taskcluster.get_task_status( test_task['status']['taskId']) # Choose best tasks to download (e.g. 'completed' is better than 'failed') download_tasks = {} for test_task in test_tasks: status = test_task['status']['state'] assert status in FINISHED_STATUSES, "State '{}' not recognized".format( status) chunk_name = taskcluster.get_chunk( test_task['task']['metadata']['name']) platform_name = taskcluster.get_platform( test_task['task']['metadata']['name']) # Ignore awsy and talos as they aren't actually suites of tests. if any(to_ignore in chunk_name for to_ignore in self.suites_to_ignore): continue if (chunk_name, platform_name) not in download_tasks: # If the chunk hasn't been downloaded before, this is obviously the best task # to download it from. download_tasks[(chunk_name, platform_name)] = test_task else: # Otherwise, compare the status of this task with the previously selected task. prev_task = download_tasks[(chunk_name, platform_name)] if STATUS_VALUE[status] > STATUS_VALUE[prev_task['status'] ['state']]: download_tasks[(chunk_name, platform_name)] = test_task with ThreadPoolExecutorResult() as executor: for test_task in download_tasks.values(): executor.submit(self.download, test_task) logger.info('Code coverage artifacts downloaded')
def go(self): # Download datasets that were built by bugbug_data. os.makedirs('data', exist_ok=True) with ThreadPoolExecutorResult(max_workers=2) as executor: executor.submit(lambda: urlretrieve('https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_data.latest/artifacts/public/bugs.json.xz', 'data/bugs.json.xz')) # noqa executor.submit(lambda: urlretrieve('https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_data.latest/artifacts/public/commits.json.xz', 'data/commits.json.xz')) # noqa # Train classifier for bug-vs-nonbug. self.train_bug() # Train classifier for regression-vs-nonregression. self.train_regression() # Train classifier for tracking bugs. self.train_tracking() # Index the task in the TaskCluster index. self.index_service.insertTask( 'project.releng.services.project.{}.bugbug_train.latest'.format(secrets[secrets.APP_CHANNEL]), { 'taskId': os.environ['TASK_ID'], 'rank': 0, 'data': {}, 'expires': (datetime.utcnow() + timedelta(31)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), } )
def go(self): with ThreadPoolExecutorResult(max_workers=2) as executor: # Thread 1 - Download coverage artifacts. executor.submit(self.artifactsHandler.download_all) # Thread 2 - Clone mozilla-central. executor.submit(self.clone_mozilla_central, self.revision) if self.from_pulse: self.githubUtils.update_geckodev_repo() commit_sha = self.githubUtils.get_commit(self.revision) logger.info('GitHub revision', revision=commit_sha) self.githubUtils.post_github_status(commit_sha) output = self.generate_info(commit_sha) logger.info('Report generated successfully') with ThreadPoolExecutorResult(max_workers=2) as executor: executor.submit(uploader.coveralls, output) executor.submit(uploader.codecov, output, commit_sha) logger.info('Waiting for build to be ingested by Codecov...') # Wait until the build has been ingested by Codecov. if uploader.codecov_wait(commit_sha): logger.info('Build ingested by codecov.io') self.notifier.notify() else: logger.info('codecov.io took too much time to ingest data.') else: mkdir('code-coverage-reports') self.generate_per_suite_reports() self.generate_zero_coverage_report() self.generate_chunk_mapping() os.chdir('code-coverage-reports') self.githubUtils.update_codecoveragereports_repo()
def go_from_trigger_mozilla_central(self): commit_sha = self.githubUtils.mercurial_to_git(self.revision) try: uploader.get_codecov(commit_sha) logger.warn('Build was already injested') return except requests.exceptions.HTTPError: pass self.retrieve_source_and_artifacts() self.githubUtils.update_geckodev_repo() logger.info('GitHub revision', revision=commit_sha) self.githubUtils.post_github_status(commit_sha) r = requests.get('https://hg.mozilla.org/mozilla-central/json-rev/%s' % self.revision) r.raise_for_status() push_id = r.json()['pushid'] output = grcov.report(self.artifactsHandler.get(), source_dir=self.repo_dir, service_number=push_id, commit_sha=commit_sha, token=secrets[secrets.COVERALLS_TOKEN]) logger.info('Report generated successfully') report = json.loads(output) expected_extensions = ['.js', '.cpp'] for extension in expected_extensions: assert any( f['name'].endswith(extension) for f in report['source_files'] ), 'No {} file in the generated report'.format(extension) logger.info('Upload changeset coverage data to Phabricator') phabricatorUploader = PhabricatorUploader(self.repo_dir, self.revision) phabricatorUploader.upload(report) with ThreadPoolExecutorResult(max_workers=2) as executor: executor.submit(uploader.coveralls, output) executor.submit(uploader.codecov, output, commit_sha) logger.info('Waiting for build to be ingested by Codecov...') # Wait until the build has been ingested by Codecov. if uploader.codecov_wait(commit_sha): logger.info('Build ingested by codecov.io') notifier = Notifier(self.repo_dir, self.revision, self.client_id, self.access_token) notifier.notify() else: logger.error('codecov.io took too much time to ingest data.')
def go(self): # Download models that were trained by bugbug_train. with ThreadPoolExecutorResult(max_workers=3) as executor: f1 = executor.submit(lambda: urlretrieve( 'https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_train.latest/artifacts/public/bug.model.xz', 'bug.model.xz')) # noqa f1.add_done_callback(lambda f: self.decompress_file('bug.model')) f2 = executor.submit(lambda: urlretrieve( 'https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_train.latest/artifacts/public/regression.model.xz', 'regression.model.xz')) # noqa f2.add_done_callback( lambda f: self.decompress_file('regression.model')) f3 = executor.submit(lambda: urlretrieve( 'https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_train.latest/artifacts/public/tracking.model.xz', 'tracking.model.xz')) # noqa f3.add_done_callback( lambda f: self.decompress_file('tracking.model')) # Download bugs from the last week that we want to analyze. bugzilla.set_token(secrets[secrets.BUGZILLA_TOKEN]) today = datetime.utcnow() one_week_ago = today - timedelta(7) bugzilla.download_bugs_between(one_week_ago, today) # Eval classifier for bug-vs-nonbug. self.eval_bug() # Eval classifier for regression-vs-nonregression. self.eval_regression() # Eval classifier for tracking bugs. self.eval_tracking() # Index the task in the TaskCluster index. self.index_service.insertTask( 'project.releng.services.project.{}.bugbug_eval.latest'.format( secrets[secrets.APP_CHANNEL]), { 'taskId': os.environ['TASK_ID'], 'rank': 0, 'data': {}, 'expires': (datetime.utcnow() + timedelta(31)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), })
def go(self): with ThreadPoolExecutorResult(max_workers=2) as executor: # Thread 1 - Download Bugzilla data. executor.submit(self.retrieve_bugs) # Thread 2 - Clone mozilla-central and retrieve commit data. executor.submit(self.retrieve_commits) # Index the task in the TaskCluster index. self.index_service.insertTask( 'project.releng.services.project.{}.bugbug_data.latest'.format( secrets[secrets.APP_CHANNEL]), { 'taskId': os.environ['TASK_ID'], 'rank': 0, 'data': {}, 'expires': (datetime.utcnow() + timedelta(31)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), })
def go(self): # Download datasets that were built by bugbug_data. os.makedirs('data', exist_ok=True) with ThreadPoolExecutorResult(max_workers=2) as executor: f1 = executor.submit(lambda: urlretrieve( 'https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_data.latest/artifacts/public/bugs.json.xz', 'data/bugs.json.xz')) # noqa f1.add_done_callback( lambda f: self.decompress_file('data/bugs.json')) f2 = executor.submit(lambda: urlretrieve( 'https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_data.latest/artifacts/public/commits.json.xz', 'data/commits.json.xz')) # noqa f2.add_done_callback( lambda f: self.decompress_file('data/commits.json')) # Train classifier for defect-vs-enhancement-vs-task. self.train_defect_enhancement_task() # Train classifier for the component of a bug. self.train_component() # Train classifier for regression-vs-nonregression. self.train_regression() # Train classifier for tracking bugs. self.train_tracking() # Index the task in the TaskCluster index. self.index_service.insertTask( f'project.releng.services.project.{secrets[secrets.APP_CHANNEL]}.bugbug_train.latest', { 'taskId': os.environ['TASK_ID'], 'rank': 0, 'data': {}, 'expires': (datetime.utcnow() + timedelta(31)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), })
def go(self): with ThreadPoolExecutorResult(max_workers=2) as executor: # Thread 1 - Download coverage artifacts. executor.submit(self.artifactsHandler.download_all) # Thread 2 - Clone mozilla-central. executor.submit(self.clone_mozilla_central, self.revision) if self.from_pulse: self.githubUtils.update_geckodev_repo() commit_sha = self.githubUtils.get_commit(self.revision) logger.info('GitHub revision', revision=commit_sha) self.githubUtils.post_github_status(commit_sha) r = requests.get('https://hg.mozilla.org/mozilla-central/json-rev/%s' % self.revision) r.raise_for_status() push_id = r.json()['pushid'] output = grcov.report( self.artifactsHandler.get(), source_dir=self.repo_dir, service_number=push_id, commit_sha=commit_sha, token=secrets[secrets.COVERALLS_TOKEN] ) logger.info('Report generated successfully') with ThreadPoolExecutorResult(max_workers=2) as executor: executor.submit(uploader.coveralls, output) executor.submit(uploader.codecov, output, commit_sha) logger.info('Waiting for build to be ingested by Codecov...') # Wait until the build has been ingested by Codecov. if uploader.codecov_wait(commit_sha): logger.info('Build ingested by codecov.io') self.notifier.notify() else: logger.error('codecov.io took too much time to ingest data.') else: os.makedirs('code-coverage-reports', exist_ok=True) self.generate_suite_reports() report_generators.zero_coverage(self.artifactsHandler.get()) self.generate_chunk_mapping() # Index the task in the TaskCluster index. self.index_service.insertTask( 'project.releng.services.project.{}.shipit_code_coverage.{}'.format(secrets[secrets.APP_CHANNEL], self.revision), { 'taskId': os.environ['TASK_ID'], 'rank': 0, 'data': {}, 'expires': (datetime.utcnow() + timedelta(180)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), } ) os.chdir('code-coverage-reports') self.githubUtils.update_codecoveragereports_repo()
def go_from_trigger_mozilla_central(self): commit_sha = self.githubUtils.mercurial_to_git(self.revision) try: uploader.get_codecov(commit_sha) logger.warn('Build was already injested') return except requests.exceptions.HTTPError: pass self.retrieve_source_and_artifacts() self.githubUtils.update_geckodev_repo() logger.info('GitHub revision', revision=commit_sha) self.githubUtils.post_github_status(commit_sha) r = requests.get('https://hg.mozilla.org/mozilla-central/json-rev/%s' % self.revision) r.raise_for_status() push_id = r.json()['pushid'] # Check that all JavaScript files present in the coverage artifacts actually exist. # If they don't, there might be a bug in the LCOV rewriter. for artifact in self.artifactsHandler.get(): if 'jsvm' not in artifact: continue with zipfile.ZipFile(artifact, 'r') as zf: for file_name in zf.namelist(): with zf.open(file_name, 'r') as fl: source_files = [ line[3:].decode('utf-8').rstrip() for line in fl if line.startswith(b'SF:') ] missing_files = [ f for f in source_files if not os.path.exists(os.path.join(self.repo_dir, f)) ] if len(missing_files) != 0: logger.warn( f'{missing_files} are present in coverage reports, but missing from the repository' ) output = grcov.report(self.artifactsHandler.get(), source_dir=self.repo_dir, service_number=push_id, commit_sha=commit_sha, token=secrets[secrets.COVERALLS_TOKEN]) logger.info('Report generated successfully') report = json.loads(output) expected_extensions = ['.js', '.cpp'] for extension in expected_extensions: assert any( f['name'].endswith(extension) for f in report['source_files'] ), 'No {} file in the generated report'.format(extension) logger.info('Upload changeset coverage data to Phabricator') phabricatorUploader = PhabricatorUploader(self.repo_dir, self.revision) phabricatorUploader.upload(report) with ThreadPoolExecutorResult(max_workers=2) as executor: executor.submit(uploader.coveralls, output) executor.submit(uploader.codecov, output, commit_sha) logger.info('Waiting for build to be ingested by Codecov...') # Wait until the build has been ingested by Codecov. if uploader.codecov_wait(commit_sha): logger.info('Build ingested by codecov.io') notifier = Notifier(self.repo_dir, self.revision, self.client_id, self.access_token) notifier.notify() else: logger.error('codecov.io took too much time to ingest data.')
def go(self): if self.from_pulse: commit_sha = self.githubUtils.mercurial_to_git(self.revision) try: uploader.get_codecov(commit_sha) logger.warn('Build was already injested') return except requests.exceptions.HTTPError: pass with ThreadPoolExecutorResult(max_workers=2) as executor: # Thread 1 - Download coverage artifacts. executor.submit(self.artifactsHandler.download_all) # Thread 2 - Clone mozilla-central. executor.submit(self.clone_mozilla_central, self.revision) if self.from_pulse: self.githubUtils.update_geckodev_repo() logger.info('GitHub revision', revision=commit_sha) self.githubUtils.post_github_status(commit_sha) r = requests.get( 'https://hg.mozilla.org/mozilla-central/json-rev/%s' % self.revision) r.raise_for_status() push_id = r.json()['pushid'] output = grcov.report(self.artifactsHandler.get(), source_dir=self.repo_dir, service_number=push_id, commit_sha=commit_sha, token=secrets[secrets.COVERALLS_TOKEN]) logger.info('Report generated successfully') logger.info('Upload changeset coverage data to Phabricator') phabricatorUploader = PhabricatorUploader(self.repo_dir, self.revision) phabricatorUploader.upload(json.loads(output)) with ThreadPoolExecutorResult(max_workers=2) as executor: executor.submit(uploader.coveralls, output) executor.submit(uploader.codecov, output, commit_sha) logger.info('Waiting for build to be ingested by Codecov...') # Wait until the build has been ingested by Codecov. if uploader.codecov_wait(commit_sha): logger.info('Build ingested by codecov.io') self.notifier.notify() else: logger.error('codecov.io took too much time to ingest data.') else: logger.info('Generating suite reports') os.makedirs(self.ccov_reports_dir, exist_ok=True) suite_reports.generate(self.suites, self.artifactsHandler, self.ccov_reports_dir, self.repo_dir) logger.info('Generating zero coverage reports') zc = ZeroCov(self.repo_dir) zc.generate(self.artifactsHandler.get(), self.revision, self.github_revision) logger.info('Generating chunk mapping') chunk_mapping.generate(self.repo_dir, self.revision, self.artifactsHandler) # Index the task in the TaskCluster index at the given revision and as "latest". # Given that all tasks have the same rank, the latest task that finishes will # overwrite the "latest" entry. namespaces = [ 'project.releng.services.project.{}.code_coverage_bot.{}'. format(secrets[secrets.APP_CHANNEL], self.revision), 'project.releng.services.project.{}.code_coverage_bot.latest'. format(secrets[secrets.APP_CHANNEL]), ] for namespace in namespaces: self.index_service.insertTask( namespace, { 'taskId': os.environ['TASK_ID'], 'rank': 0, 'data': {}, 'expires': (datetime.utcnow() + timedelta(180)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), }) os.chdir(self.ccov_reports_dir) self.githubUtils.update_codecoveragereports_repo()