def register_db(db_format, db_compression):
db_name = f"prova.{db_format}"
if db_compression is not None:
db_name += f".{db_compression}"
db_path = tmp_path / db_name
db.register(db_path, "https://alink")
return db_path
def test_bad_format_compression(tmp_path, db_name):
db_path = tmp_path / db_name
db.register(db_path, "https://alink")
with pytest.raises(AssertionError):
db.write(db_path, range(7))
with pytest.raises(AssertionError):
db.append(db_path, range(7))
import hglib
from tqdm import tqdm
from bugbug import db, utils
logger = logging.getLogger(__name__)
hg_servers = list()
hg_servers_lock = threading.Lock()
thread_local = threading.local()
COMMITS_DB = "data/commits.json"
db.register(
COMMITS_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.data_commits.latest/artifacts/public/commits.json.zst",
4,
["commit_experiences.pickle.zst"],
)
path_to_component = {}
EXPERIENCE_TIMESPAN = 90
EXPERIENCE_TIMESPAN_TEXT = f"{EXPERIENCE_TIMESPAN}_days"
SOURCE_CODE_TYPES_TO_EXT = {
"Assembly": [".asm", ".S"],
"Javascript": [".js", ".jsm", ".sjs"],
"C/C++":
[".c", ".cpp", ".cc", ".cxx", ".m", ".mm", ".h", ".hh", ".hpp", ".hxx"],
"Java": [".java"],
"Python": [".py"],
JOBS_TO_CONSIDER = ("test-", "build-")
JOBS_TO_IGNORE = (
"build-docker-image-",
"-test-verify-",
"-test-coverage-",
"-android-hw-",
"-awsy-",
"-raptor-",
"-talos-",
"backlog",
)
ADR_CACHE_DB = "data/adr_cache.tar"
db.register(
ADR_CACHE_DB,
"https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/adr_cache.tar.zst",
3,
)
# The mozci version (to bump whenever we change the mozci regression algorithm),
# so we can keep track of which version of mozci was used to analyze a given push
# and we can decide when we want to regenerate parts of the dataset.
MOZCI_VERSION = 2
TRAINING_MONTHS = {
"label": 7,
"group": 7,
}
def filter_runnables(runnables, all_runnables, granularity):
return tuple(
import logging
from typing import Callable, Iterator, List, NewType, Tuple
import requests
from ratelimit import limits, sleep_and_retry
from bugbug import db
from bugbug.utils import get_secret
logger = logging.getLogger(__name__)
GITHUB_ISSUES_DB = "data/github_issues.json"
db.register(
GITHUB_ISSUES_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.data_github_issues.latest/artifacts/public/github_issues.json.zst",
1,
)
IssueDict = NewType("IssueDict", dict)
PER_PAGE = 100
# Rate limit period in seconds
RATE_LIMIT_PERIOD = 900
def get_issues() -> Iterator[IssueDict]:
yield from db.read(GITHUB_ISSUES_DB)
def delete_issues(match: Callable[[IssueDict], bool]) -> None:
)
basicConfig(level=INFO)
logger = getLogger(__name__)
thread_local = threading.local()
MAX_MODIFICATION_NUMBER = 50
RELATIVE_START_DATE = relativedelta(years=2, months=6)
# Only needed because mercurial<->git mapping could be behind.
RELATIVE_END_DATE = relativedelta(days=7)
IGNORED_COMMITS_DB = "data/ignored_commits.json"
db.register(
IGNORED_COMMITS_DB,
"https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/ignored_commits.json.zst",
5,
)
class RegressorFinder(object):
def __init__(
self,
git_repo_url=None,
git_repo_dir=None,
tokenized_git_repo_url=None,
tokenized_git_repo_dir=None,
):
self.git_repo_url = git_repo_url
self.git_repo_dir = git_repo_dir
self.tokenized_git_repo_url = tokenized_git_repo_url
import csv
from datetime import datetime
from typing import Dict, Iterable
import tenacity
from dateutil.relativedelta import relativedelta
from libmozdata.bugzilla import Bugzilla
from tqdm import tqdm
from bugbug import db, utils
BUGS_DB = "data/bugs.json"
db.register(
BUGS_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.data_bugs.latest/artifacts/public/bugs.json.zst",
6,
)
PRODUCTS = (
"Add-on SDK",
"Android Background Services",
"Core",
"Core Graveyard",
"DevTools",
"DevTools Graveyard",
"External Software Affecting Firefox",
"Firefox",
"Firefox Graveyard",
"Firefox Build System",
"Firefox for Android",
import matplotlib.pyplot as plt
import mozci.push
from dateutil.relativedelta import relativedelta
from pandas import DataFrame
from tqdm import tqdm
from bugbug import db, test_scheduling, utils
from bugbug.test_scheduling import ConfigGroup, Group
logger = logging.getLogger(__name__)
SHADOW_SCHEDULER_STATS_DB = "data/shadow_scheduler_stats.json"
db.register(
SHADOW_SCHEDULER_STATS_DB,
"https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/shadow_scheduler_stats.json.zst",
3,
)
def analyze_shadow_schedulers(
group_regressions: Set[Group],
config_group_regressions: Set[ConfigGroup],
push: mozci.push.Push,
) -> dict:
schedulers = []
for name, config_groups in push.generate_all_shadow_scheduler_config_groups(
):
if isinstance(config_groups, mozci.errors.TaskNotFound):
continue
from bugbug.utils import download_check_etag, retry, zstd_compress, zstd_decompress
basicConfig(level=INFO)
logger = getLogger(__name__)
thread_local = threading.local()
MAX_MODIFICATION_NUMBER = 50
RELATIVE_START_DATE = relativedelta(years=2, months=6)
# Only needed because mercurial<->git mapping could be behind.
RELATIVE_END_DATE = relativedelta(days=7)
IGNORED_COMMITS_DB = "data/ignored_commits.json"
db.register(
IGNORED_COMMITS_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug_annotate.regressor_finder.latest/artifacts/public/ignored_commits.json.zst",
1,
)
BUG_FIXING_COMMITS_DB = "data/bug_fixing_commits.json"
db.register(
BUG_FIXING_COMMITS_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug_annotate.regressor_finder.latest/artifacts/public/bug_fixing_commits.json.zst",
1,
)
BASE_URL = "https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.train_{model_name}.latest/artifacts/public/{model_name}model.zst"
def download_model(model_name):
if not os.path.exists(f"{model_name}model"):
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from bugbug import db
TEST_SCHEDULING_DB = "data/test_scheduling_history.pickle"
db.register(
TEST_SCHEDULING_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.data_test_scheduling_history.latest/artifacts/public/test_scheduling_history.pickle.zst",
4,
["past_failures.lmdb.tar.zst"],
)
def get_test_scheduling_history():
return db.read(TEST_SCHEDULING_DB)
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import requests
from libmozdata.bugzilla import Bugzilla
from tqdm import tqdm
from bugbug import db
BUGS_DB = "data/bugs.json"
db.register(BUGS_DB,
"https://www.dropbox.com/s/vekjzqu4v0pu1v8/bugs.json.xz?dl=1",
"v1")
ATTACHMENT_INCLUDE_FIELDS = [
"id",
"is_obsolete",
"flags",
"is_patch",
"creator",
"content_type",
"creation_time",
]
COMMENT_INCLUDE_FIELDS = ["id", "count", "text", "author", "creation_time"]
from bugbug.models.regression import RegressionModel
from bugbug.utils import download_check_etag, retry
basicConfig(level=INFO)
logger = getLogger(__name__)
MAX_MODIFICATION_NUMBER = 50
# TODO: Set to 2 years and 6 months. If it takes too long, make the task work incrementally like microannotate-generate.
RELATIVE_START_DATE = relativedelta(days=49)
# Only needed because mercurial<->git mapping could be behind.
RELATIVE_END_DATE = relativedelta(days=3)
BUG_FIXING_COMMITS_DB = "data/bug_fixing_commits.json"
db.register(
BUG_FIXING_COMMITS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug_annotate.regressor_finder.latest/artifacts/public/bug_fixing_commits.json.zst",
2,
)
BUG_INTRODUCING_COMMITS_DB = "data/bug_introducing_commits.json"
db.register(
BUG_INTRODUCING_COMMITS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug_annotate.regressor_finder.latest/artifacts/public/bug_introducing_commits.json.zst",
1,
)
TOKENIZED_BUG_INTRODUCING_COMMITS_DB = "data/tokenized_bug_introducing_commits.json"
db.register(
TOKENIZED_BUG_INTRODUCING_COMMITS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug_annotate.regressor_finder.latest/artifacts/public/tokenized_bug_introducing_commits.json.zst",
1,
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import itertools
import os
import pickle
import shelve
from bugbug import db, repository
from bugbug.utils import ExpQueue, LMDBDict
TEST_LABEL_SCHEDULING_DB = "data/test_label_scheduling_history.pickle"
PAST_FAILURES_LABEL_DB = "past_failures_label.lmdb.tar.zst"
db.register(
TEST_LABEL_SCHEDULING_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.data_test_label_scheduling_history.latest/artifacts/public/test_label_scheduling_history.pickle.zst",
7,
[PAST_FAILURES_LABEL_DB],
)
TEST_GROUP_SCHEDULING_DB = "data/test_group_scheduling_history.pickle"
PAST_FAILURES_GROUP_DB = "past_failures_group.lmdb.tar.zst"
TOUCHED_TOGETHER_DB = "touched_together.lmdb.tar.zst"
db.register(
TEST_GROUP_SCHEDULING_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.data_test_group_scheduling_history.latest/artifacts/public/test_group_scheduling_history.pickle.zst",
11,
[PAST_FAILURES_GROUP_DB, TOUCHED_TOGETHER_DB],
)
HISTORICAL_TIMESPAN = 56
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import requests
from libmozdata.bugzilla import Bugzilla
from tqdm import tqdm
from bugbug import db
BUGS_DB = 'data/bugs.json'
db.register(BUGS_DB,
'https://www.dropbox.com/s/vekjzqu4v0pu1v8/bugs.json.xz?dl=1',
'v1')
ATTACHMENT_INCLUDE_FIELDS = [
'id',
'is_obsolete',
'flags',
'is_patch',
'creator',
'content_type',
'creation_time',
]
COMMENT_INCLUDE_FIELDS = [
'id',
'count',
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from bugbug import db
TEST_SCHEDULING_DB = "data/test_scheduling_history.pickle"
db.register(
TEST_SCHEDULING_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug.data_test_scheduling_history.latest/artifacts/public/test_scheduling_history.pickle.zst",
2,
["past_failures.shelve.db.zst"],
)
def get_test_scheduling_history():
return db.read(TEST_SCHEDULING_DB)
import logging
from typing import Collection, Iterator, NewType, Optional
from libmozdata.phabricator import PhabricatorAPI
from tqdm import tqdm
from bugbug import db
logger = logging.getLogger(__name__)
RevisionDict = NewType("RevisionDict", dict)
REVISIONS_DB = "data/revisions.json"
db.register(
REVISIONS_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.data_revisions.latest/artifacts/public/revisions.json.zst",
1,
)
PHABRICATOR_API = None
TESTING_PROJECTS = {
"PHID-PROJ-h7y4cs7m2o67iczw62pp": "testing-approved",
"PHID-PROJ-e4fcjngxcws3egiecv3r": "testing-exception-elsewhere",
"PHID-PROJ-iciyosoekrczpf2a4emw": "testing-exception-other",
"PHID-PROJ-zjipshabawolpkllehvg": "testing-exception-ui",
"PHID-PROJ-cspmf33ku3kjaqtuvs7g": "testing-exception-unchanged",
}
def get_revisions() -> Iterator[RevisionDict]:
import dateutil.parser
import xgboost
from dateutil.relativedelta import relativedelta
from imblearn.under_sampling import RandomUnderSampler
from sklearn.compose import ColumnTransformer
from sklearn.feature_extraction import DictVectorizer
from sklearn.pipeline import Pipeline
from bugbug import commit_features, db, feature_cleanup, repository
from bugbug.model import CommitModel
BUG_FIXING_COMMITS_DB = "data/bug_fixing_commits.json"
db.register(
BUG_FIXING_COMMITS_DB,
"https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/bug_fixing_commits.json.zst",
1,
)
BUG_INTRODUCING_COMMITS_DB = "data/bug_introducing_commits.json"
db.register(
BUG_INTRODUCING_COMMITS_DB,
"https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/bug_introducing_commits.json.zst",
2,
)
TOKENIZED_BUG_INTRODUCING_COMMITS_DB = "data/tokenized_bug_introducing_commits.json"
db.register(
TOKENIZED_BUG_INTRODUCING_COMMITS_DB,
"https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/tokenized_bug_introducing_commits.json.zst",
3,
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import requests
from libmozdata import bugzilla
from bugbug import db
BUGS_DB = 'data/bugs.json'
db.register(BUGS_DB,
'https://www.dropbox.com/s/xm6wzac9jl81irz/bugs.json.xz?dl=1')
ATTACHMENT_INCLUDE_FIELDS = [
'id',
'is_obsolete',
'flags',
'is_patch',
'creator',
'content_type',
'creation_time',
]
COMMENT_INCLUDE_FIELDS = [
'id',
'text',
'author',
logger = logging.getLogger(__name__)
Revision = NewType("Revision", str)
Task = NewType("Task", str)
Group = NewType("Group", str)
ConfigGroup = NewType("ConfigGroup", Tuple[str, Group])
Runnable = Union[Task, Group, ConfigGroup]
PushResult = Tuple[Tuple[Revision], Revision, Tuple[Runnable, ...],
Tuple[Runnable, ...], Tuple[Runnable, ...], ]
TEST_LABEL_SCHEDULING_DB = "data/test_label_scheduling_history.pickle"
PAST_FAILURES_LABEL_DB = "past_failures_label.lmdb.tar.zst"
FAILING_TOGETHER_LABEL_DB = "failing_together_label.lmdb.tar.zst"
db.register(
TEST_LABEL_SCHEDULING_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.data_test_label_scheduling_history.latest/artifacts/public/test_label_scheduling_history.pickle.zst",
13,
[PAST_FAILURES_LABEL_DB, FAILING_TOGETHER_LABEL_DB],
)
PUSH_DATA_LABEL_DB = "data/push_data_label.json"
db.register(
PUSH_DATA_LABEL_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.data_test_label_scheduling_history_push_data.latest/artifacts/public/push_data_label.json.zst",
1,
)
TEST_GROUP_SCHEDULING_DB = "data/test_group_scheduling_history.pickle"
PAST_FAILURES_GROUP_DB = "past_failures_group.lmdb.tar.zst"
TOUCHED_TOGETHER_DB = "touched_together.lmdb.tar.zst"
db.register(
TEST_GROUP_SCHEDULING_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.data_test_group_scheduling_history.latest/artifacts/public/test_group_scheduling_history.pickle.zst",
import os
from collections import defaultdict
from collections import namedtuple
from datetime import datetime
import hglib
import requests
from dateutil.relativedelta import relativedelta
from parsepatch.patch import Patch
from tqdm import tqdm
from bugbug import db
COMMITS_DB = 'data/commits.json'
db.register(COMMITS_DB,
'https://www.dropbox.com/s/mz3afgncx0siijc/commits.json.xz?dl=1',
'v1')
path_to_component = {}
Commit = namedtuple('Commit', [
'node', 'author', 'desc', 'date', 'bug', 'backedoutby', 'author_email',
'files', 'file_copies'
])
author_experience = {}
author_experience_90_days = {}
components_touched_prev = defaultdict(int)
components_touched_prev_90_days = defaultdict(int)
import logging
from datetime import datetime
from typing import Iterator, List
import mozci.push
from dateutil.relativedelta import relativedelta
from tqdm import tqdm
from bugbug import db, utils
logger = logging.getLogger(__name__)
SHADOW_SCHEDULER_STATS_DB = "data/shadow_scheduler_stats.json"
db.register(
SHADOW_SCHEDULER_STATS_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.data_bugs.latest/artifacts/public/shadow_scheduler_stats.json.zst",
1,
)
def analyze_shadow_schedulers(pushes: List[mozci.push.Push]) -> Iterator[dict]:
for push in tqdm(pushes):
schedulers = []
group_regressions = push.get_likely_regressions("group")
config_group_regressions = push.get_likely_regressions("config_group")
for name, config_groups in push.generate_all_shadow_scheduler_config_groups(
):
if isinstance(config_groups, mozci.errors.TaskNotFound):
continue
def test_register_db(tmp_path):
db_path = tmp_path / "prova.json"
db.register(db_path, "https://alink", 1)
assert os.path.exists(db_path.with_suffix(db_path.suffix + ".version"))
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import pickle
import shelve
from bugbug import db
from bugbug.utils import ExpQueue, LMDBDict
TEST_SCHEDULING_DB = "data/test_scheduling_history.pickle"
PAST_FAILURES_DB = "past_failures.lmdb.tar.zst"
db.register(
TEST_SCHEDULING_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.data_test_scheduling_history.latest/artifacts/public/test_scheduling_history.pickle.zst",
6,
[PAST_FAILURES_DB],
)
HISTORICAL_TIMESPAN = 56
def get_test_scheduling_history():
return db.read(TEST_SCHEDULING_DB)
def get_past_failures():
return shelve.Shelf(
LMDBDict("data/past_failures.lmdb"),
protocol=pickle.HIGHEST_PROTOCOL,
writeback=True,
download_check_etag,
open_tar_zst,
zstd_compress,
zstd_decompress,
)
basicConfig(level=INFO)
logger = getLogger(__name__)
JOBS_TO_CONSIDER = ("test-", "build-")
JOBS_TO_IGNORE = ("build-docker-image-", )
ADR_CACHE_DB = "data/adr_cache.tar"
db.register(
ADR_CACHE_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.data_test_scheduling_history_push_data.latest/artifacts/public/adr_cache.tar.zst",
2,
support_files=[],
)
PUSH_DATA_URL = "https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.data_test_scheduling_history_push_data.latest/artifacts/public/push_data.json.zst"
TRAINING_MONTHS = 6
def filter_tasks(tasks, all_tasks):
return tuple(task for task in tasks if task in all_tasks and any(
task.startswith(j) for j in JOBS_TO_CONSIDER) and not any(
task.startswith(j) for j in JOBS_TO_IGNORE))
class Retriever(object):
def __init__(self):
import csv
import json
import os
from datetime import datetime
import requests
from dateutil.relativedelta import relativedelta
from libmozdata.bugzilla import Bugzilla
from tqdm import tqdm
from bugbug import db
BUGS_DB = "data/bugs.json"
db.register(
BUGS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug.data_bugs.latest/artifacts/public/bugs.json.xz",
1,
)
PRODUCTS = (
"Add-on SDK",
"Android Background Services",
"Core",
"Core Graveyard",
"DevTools",
"DevTools Graveyard",
"External Software Affecting Firefox",
"Firefox",
"Firefox Graveyard",
"Firefox Build System",
"Firefox for Android",
logger = logging.getLogger(__name__)
CommitDict = NewType("CommitDict", dict)
code_analysis_server: Optional[
rust_code_analysis_server.RustCodeAnalysisServer] = None
hg_servers = list()
hg_servers_lock = threading.Lock()
thread_local = threading.local()
COMMITS_DB = "data/commits.json"
COMMIT_EXPERIENCES_DB = "commit_experiences.lmdb.tar.zst"
db.register(
COMMITS_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.data_commits.latest/artifacts/public/commits.json.zst",
19,
[COMMIT_EXPERIENCES_DB],
)
commit_to_coverage = None
path_to_component = None
HG = None
REPO_DIR = None
EXPERIENCE_TIMESPAN = 90
EXPERIENCE_TIMESPAN_TEXT = f"{EXPERIENCE_TIMESPAN}_days"
SOURCE_CODE_TYPES_TO_EXT = {
"Assembly": [".asm", ".S"],
"Javascript": [".js", ".jsm", ".sjs"],
from bugbug.utils import download_check_etag, retry, zstd_compress, zstd_decompress
basicConfig(level=INFO)
logger = getLogger(__name__)
thread_local = threading.local()
MAX_MODIFICATION_NUMBER = 50
RELATIVE_START_DATE = relativedelta(years=2, months=6)
# Only needed because mercurial<->git mapping could be behind.
RELATIVE_END_DATE = relativedelta(days=7)
IGNORED_COMMITS_DB = "data/ignored_commits.json"
db.register(
IGNORED_COMMITS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug_annotate.regressor_finder.latest/artifacts/public/ignored_commits.json.zst",
1,
)
BUG_FIXING_COMMITS_DB = "data/bug_fixing_commits.json"
db.register(
BUG_FIXING_COMMITS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug_annotate.regressor_finder.latest/artifacts/public/bug_fixing_commits.json.zst",
1,
)
BASE_URL = "https://index.taskcluster.net/v1/task/project.relman.bugbug.train_{model_name}.latest/artifacts/public/{model_name}model.zst"
def download_model(model_name):
if not os.path.exists(f"{model_name}model"):
import os
import pickle
import re
import sys
from collections import deque
from datetime import datetime
import hglib
from tqdm import tqdm
from bugbug import db, utils
COMMITS_DB = "data/commits.json"
db.register(
COMMITS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug.data_commits.latest/artifacts/public/commits.json.xz",
1,
)
path_to_component = {}
EXPERIENCE_TIMESPAN = 90
EXPERIENCE_TIMESPAN_TEXT = f"{EXPERIENCE_TIMESPAN}_days"
class Commit:
def __init__(
self,
node,
author,
desc,
import hglib
from tqdm import tqdm
from bugbug import db, utils
logger = logging.getLogger(__name__)
hg_servers = list()
hg_servers_lock = threading.Lock()
thread_local = threading.local()
COMMITS_DB = "data/commits.json"
db.register(
COMMITS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug.data_commits.latest/artifacts/public/commits.json.zst",
1,
["commit_experiences.pickle.zst"],
)
path_to_component = {}
EXPERIENCE_TIMESPAN = 90
EXPERIENCE_TIMESPAN_TEXT = f"{EXPERIENCE_TIMESPAN}_days"
class Commit:
def __init__(
self,
node,
author,
desc,
import csv
import json
import os
from datetime import datetime
import requests
from dateutil.relativedelta import relativedelta
from libmozdata.bugzilla import Bugzilla
from tqdm import tqdm
from bugbug import db
BUGS_DB = "data/bugs.json"
db.register(
BUGS_DB,
"https://index.taskcluster.net/v1/task/project.relman.bugbug.data_bugs.latest/artifacts/public/bugs.json.xz",
)
ATTACHMENT_INCLUDE_FIELDS = [
"id",
"is_obsolete",
"flags",
"is_patch",
"creator",
"content_type",
"creation_time",
]
COMMENT_INCLUDE_FIELDS = ["id", "count", "text", "author", "creation_time"]
PRODUCT_COMPONENT_CSV_REPORT_URL = "https://bugzilla.mozilla.org/report.cgi"
from bugbug.utils import (
download_check_etag,
download_model,
get_secret,
zstd_compress,
zstd_decompress,
)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
TEST_INFOS_DB = "data/test_info.json"
db.register(
TEST_INFOS_DB,
"https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.landings_risk_report.latest/artifacts/public/test_info.json.zst",
2,
)
PAST_REGRESSIONS_BY_URL = "https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.past_bugs_by_unit.latest/artifacts/public/past_regressions_by_{dimension}.json.zst"
PAST_FIXED_BUGS_BY_URL = "https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.past_bugs_by_unit.latest/artifacts/public/past_fixed_bugs_by_{dimension}.json.zst"
PAST_REGRESSION_BLOCKED_BUGS_BY_URL = "https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.past_bugs_by_unit.latest/artifacts/public/past_regression_blocked_bugs_by_{dimension}.json.zst"
PAST_FIXED_BUG_BLOCKED_BUGS_BY_URL = "https://community-tc.services.mozilla.com/api/index/v1/task/project.bugbug.past_bugs_by_unit.latest/artifacts/public/past_fixed_bug_blocked_bugs_by_{dimension}.json.zst"
def _deduplicate(bug_summaries: List[dict]) -> List[dict]:
seen = set()
results = []
for bug_summary in bug_summaries[::-1]:
if bug_summary["id"] in seen:
continue
