def __init__(self,
lemmatization=False,
granularity="label",
failures_skip=None):
Model.__init__(self, lemmatization)
self.granularity = granularity
self.failures_skip = failures_skip
self.training_dbs = [repository.COMMITS_DB]
self.eval_dbs[repository.COMMITS_DB] = (
repository.COMMITS_DB,
repository.COMMIT_EXPERIENCES_DB,
)
if granularity == "label":
self.training_dbs.append(test_scheduling.TEST_LABEL_SCHEDULING_DB)
self.eval_dbs[test_scheduling.TEST_LABEL_SCHEDULING_DB] = (
test_scheduling.PAST_FAILURES_LABEL_DB,
test_scheduling.FAILING_TOGETHER_LABEL_DB,
)
elif granularity == "group":
self.training_dbs.append(test_scheduling.TEST_GROUP_SCHEDULING_DB)
self.eval_dbs[test_scheduling.TEST_GROUP_SCHEDULING_DB] = (
test_scheduling.PAST_FAILURES_GROUP_DB,
test_scheduling.TOUCHED_TOGETHER_DB,
)
self.cross_validation_enabled = False
self.entire_dataset_training = True
self.sampler = RandomUnderSampler(random_state=0)
feature_extractors = [
test_scheduling_features.prev_failures(),
]
if granularity == "label":
feature_extractors += [
test_scheduling_features.platform(),
# test_scheduling_features.chunk(),
test_scheduling_features.suite(),
]
elif granularity == "group":
feature_extractors += [
test_scheduling_features.path_distance(),
test_scheduling_features.common_path_components(),
test_scheduling_features.touched_together(),
]
self.extraction_pipeline = Pipeline([
(
"commit_extractor",
commit_features.CommitExtractor(feature_extractors, []),
),
("union", ColumnTransformer([("data", DictVectorizer(), "data")])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor="cpu_predictor")
def __init__(self, lemmatization=False, granularity="label"):
Model.__init__(self, lemmatization)
self.granularity = granularity
self.required_dbs = [repository.COMMITS_DB]
if granularity == "label":
self.required_dbs.append(test_scheduling.TEST_LABEL_SCHEDULING_DB)
elif granularity == "group":
self.required_dbs.append(test_scheduling.TEST_GROUP_SCHEDULING_DB)
self.cross_validation_enabled = False
self.entire_dataset_training = True
self.sampler = RandomUnderSampler(random_state=0)
feature_extractors = [
commit_features.source_code_files_modified_num(),
commit_features.other_files_modified_num(),
commit_features.test_files_modified_num(),
commit_features.source_code_file_size(),
commit_features.other_file_size(),
commit_features.test_file_size(),
commit_features.source_code_added(),
commit_features.other_added(),
commit_features.test_added(),
commit_features.source_code_deleted(),
commit_features.other_deleted(),
commit_features.test_deleted(),
test_scheduling_features.name(),
test_scheduling_features.prev_failures(),
]
if granularity == "label":
feature_extractors += [
test_scheduling_features.platform(),
test_scheduling_features.chunk(),
test_scheduling_features.suite(),
]
elif granularity == "group":
feature_extractors += [
test_scheduling_features.path_distance(),
test_scheduling_features.common_path_components(),
]
self.extraction_pipeline = Pipeline([
(
"commit_extractor",
commit_features.CommitExtractor(feature_extractors, []),
),
("union", ColumnTransformer([("data", DictVectorizer(), "data")])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor="cpu_predictor")
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
self.sampler = RandomUnderSampler(random_state=0)
feature_extractors = [
bug_features.has_str(),
bug_features.has_regression_range(),
bug_features.severity(),
bug_features.keywords({"dev-doc-needed", "dev-doc-complete"}),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
bug_features.product(),
bug_features.component(),
bug_features.commit_added(),
bug_features.commit_deleted(),
bug_features.commit_types(),
]
cleanup_functions = [
bug_features.cleanup_fileref,
bug_features.cleanup_url,
bug_features.cleanup_synonyms,
]
self.extraction_pipeline = Pipeline([
(
"bug_extractor",
bug_features.BugExtractor(
feature_extractors,
cleanup_functions,
rollback=True,
rollback_when=self.rollback,
commit_data=True,
),
),
(
"union",
ColumnTransformer([
("data", DictVectorizer(), "data"),
("title", self.text_vectorizer(), "title"),
("comments", self.text_vectorizer(), "comments"),
]),
),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor="cpu_predictor")
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
self.sampler = InstanceHardnessThreshold(random_state=0)
feature_extractors = [
bug_features.has_str(),
bug_features.has_regression_range(),
bug_features.severity(),
bug_features.keywords(),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
bug_features.product(),
bug_features.component(),
bug_features.is_mozillian(),
bug_features.bug_reporter(),
bug_features.blocked_bugs_number(),
bug_features.priority(),
bug_features.has_cve_in_alias(),
bug_features.comment_count(),
bug_features.comment_length(),
bug_features.reporter_experience(),
bug_features.number_of_bug_dependencies()
]
cleanup_functions = [
bug_features.cleanup_url,
bug_features.cleanup_fileref,
bug_features.cleanup_hex,
bug_features.cleanup_dll,
bug_features.cleanup_synonyms,
bug_features.cleanup_crash,
]
self.extraction_pipeline = Pipeline([
('bug_extractor', bug_features.BugExtractor(feature_extractors, cleanup_functions, rollback=True, rollback_when=self.rollback)),
('union', ColumnTransformer([
('data', DictVectorizer(), 'data'),
('title', self.text_vectorizer(min_df=0.0001), 'title'),
('comments', self.text_vectorizer(min_df=0.0001), 'comments'),
])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor='cpu_predictor')
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
self.cross_validation_enabled = False
self.calculate_importance = False
feature_extractors = [
bug_features.has_str(),
bug_features.severity(),
bug_features.keywords(),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
]
cleanup_functions = [
bug_features.cleanup_fileref,
bug_features.cleanup_url,
bug_features.cleanup_synonyms,
]
self.extraction_pipeline = Pipeline([
(
"bug_extractor",
bug_features.BugExtractor(
feature_extractors,
cleanup_functions,
rollback=True,
rollback_when=self.rollback,
),
),
(
"union",
ColumnTransformer([
("data", DictVectorizer(), "data"),
("title", self.text_vectorizer(min_df=0.0001), "title"),
(
"comments",
self.text_vectorizer(min_df=0.0001),
"comments",
),
]),
),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor="cpu_predictor")
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
feature_extractors = [
bug_features.has_str(),
bug_features.has_regression_range(),
bug_features.severity(),
bug_features.keywords({'dev-doc-needed', 'dev-doc-complete'}),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
bug_features.product(),
bug_features.component(),
bug_features.commit_added(),
bug_features.commit_deleted(),
bug_features.commit_types(),
]
cleanup_functions = [
bug_features.cleanup_fileref,
bug_features.cleanup_url,
bug_features.cleanup_synonyms,
]
self.extraction_pipeline = Pipeline([
('bug_extractor',
bug_features.BugExtractor(feature_extractors,
cleanup_functions,
rollback=True,
rollback_when=self.rollback,
commit_data=True)),
('union',
ColumnTransformer([
('data', DictVectorizer(), 'data'),
('title', self.text_vectorizer(stop_words='english'),
'title'),
('comments', self.text_vectorizer(stop_words='english'),
'comments'),
])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor='cpu_predictor')
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
self.sampler = BorderlineSMOTE(random_state=0)
feature_extractors = [
bug_features.has_str(),
bug_features.severity(),
# Ignore keywords that would make the ML completely skewed
# (we are going to use them as 100% rules in the evaluation phase).
bug_features.keywords(
{'regression', 'talos-regression', 'feature'}),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
bug_features.blocked_bugs_number(),
bug_features.ever_affected(),
bug_features.affected_then_unaffected(),
bug_features.product(),
bug_features.component(),
]
cleanup_functions = [
bug_features.cleanup_url,
bug_features.cleanup_fileref,
bug_features.cleanup_synonyms,
]
self.extraction_pipeline = Pipeline([
('bug_extractor',
bug_features.BugExtractor(feature_extractors, cleanup_functions)),
('union',
ColumnTransformer([
('data', DictVectorizer(), 'data'),
('title', self.text_vectorizer(min_df=0.001), 'title'),
('first_comment', self.text_vectorizer(min_df=0.001),
'first_comment'),
('comments', self.text_vectorizer(min_df=0.001), 'comments'),
])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor='cpu_predictor')
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
self.undersampling_enabled = False
self.cross_validation_enabled = False
feature_extractors = [
bug_features.has_str(),
bug_features.severity(),
bug_features.keywords(),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
]
cleanup_functions = [
bug_features.cleanup_fileref,
bug_features.cleanup_url,
bug_features.cleanup_synonyms,
]
self.extraction_pipeline = Pipeline([
('bug_extractor',
bug_features.BugExtractor(feature_extractors, cleanup_functions)),
(
'union',
ColumnTransformer([
# TODO: Re-enable when we'll support bug snapshotting (#5).
# ('data', DictVectorizer(), 'data'),
('title', self.text_vectorizer(stop_words='english'),
'title'),
# TODO: Re-enable when we'll support bug snapshotting (#5).
# ('comments', self.text_vectorizer(stop_words='english'), 'comments'),
('first_comment',
self.text_vectorizer(stop_words='english'),
'first_comment'),
])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor='cpu_predictor')
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
self.sampler = RandomUnderSampler(random_state=0)
feature_extractors = [
bug_features.has_str(),
bug_features.has_regression_range(),
bug_features.severity(),
bug_features.keywords({'qawanted'}),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
]
cleanup_functions = [
bug_features.cleanup_fileref,
bug_features.cleanup_url,
bug_features.cleanup_synonyms,
]
self.extraction_pipeline = Pipeline([
('bug_extractor',
bug_features.BugExtractor(feature_extractors,
cleanup_functions,
rollback=True,
rollback_when=self.rollback)),
('union',
ColumnTransformer([
('data', DictVectorizer(), 'data'),
('title', self.text_vectorizer(), 'title'),
('comments', self.text_vectorizer(), 'comments'),
])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor='cpu_predictor')
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
self.required_dbs = [
repository.COMMITS_DB, test_scheduling.TEST_SCHEDULING_DB
]
self.calculate_importance = False
self.cross_validation_enabled = False
self.sampler = RandomUnderSampler(random_state=0)
feature_extractors = [
commit_features.source_code_files_modified_num(),
commit_features.other_files_modified_num(),
commit_features.test_files_modified_num(),
commit_features.source_code_file_size(),
commit_features.other_file_size(),
commit_features.test_file_size(),
commit_features.source_code_added(),
commit_features.other_added(),
commit_features.test_added(),
commit_features.source_code_deleted(),
commit_features.other_deleted(),
commit_features.test_deleted(),
test_scheduling_features.name(),
test_scheduling_features.platform(),
test_scheduling_features.chunk(),
test_scheduling_features.suite(),
test_scheduling_features.prev_failures(),
]
self.extraction_pipeline = Pipeline([
(
"commit_extractor",
commit_features.CommitExtractor(feature_extractors, []),
),
("union", ColumnTransformer([("data", DictVectorizer(), "data")])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor="cpu_predictor")
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
self.cross_validation_enabled = False
self.calculate_importance = False
feature_extractors = [
bug_features.has_str(),
bug_features.severity(),
bug_features.keywords(),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
]
cleanup_functions = [
bug_features.cleanup_fileref,
bug_features.cleanup_url,
bug_features.cleanup_synonyms,
]
self.extraction_pipeline = Pipeline([
('bug_extractor', bug_features.BugExtractor(feature_extractors, cleanup_functions, rollback=True)),
('union', ColumnTransformer([
('data', DictVectorizer(), 'data'),
('title', self.text_vectorizer(min_df=0.0001), 'title'),
('comments', self.text_vectorizer(min_df=0.0001), 'comments'),
])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor='cpu_predictor')
self.CONFLATED_COMPONENTS_INVERSE_MAPPING = {v: k for k, v in self.CONFLATED_COMPONENTS_MAPPING.items()}
def __init__(self,
lemmatization=False,
granularity="label",
use_subset=False):
Model.__init__(self, lemmatization)
self.granularity = granularity
# This is useful for development purposes, it avoids using too much memory
# by using a subset of the dataset (dropping some passing runnables).
self.use_subset = use_subset
self.training_dbs = [repository.COMMITS_DB]
self.eval_dbs[repository.COMMITS_DB] = (
repository.COMMITS_DB,
repository.COMMIT_EXPERIENCES_DB,
)
if granularity == "label":
self.training_dbs.append(test_scheduling.TEST_LABEL_SCHEDULING_DB)
self.eval_dbs[test_scheduling.TEST_LABEL_SCHEDULING_DB] = (
test_scheduling.PAST_FAILURES_LABEL_DB,
test_scheduling.FAILING_TOGETHER_LABEL_DB,
)
elif granularity == "group":
self.training_dbs.append(test_scheduling.TEST_GROUP_SCHEDULING_DB)
self.eval_dbs[test_scheduling.TEST_GROUP_SCHEDULING_DB] = (
test_scheduling.PAST_FAILURES_GROUP_DB,
test_scheduling.TOUCHED_TOGETHER_DB,
)
self.cross_validation_enabled = False
self.entire_dataset_training = True
self.sampler = RandomUnderSampler(random_state=0)
feature_extractors = [
test_scheduling_features.prev_failures(),
]
if granularity == "label":
feature_extractors += [
test_scheduling_features.platform(),
# test_scheduling_features.chunk(),
test_scheduling_features.suite(),
]
elif granularity == "group":
feature_extractors += [
commit_features.source_code_files_modified_num(),
commit_features.other_files_modified_num(),
commit_features.test_files_modified_num(),
commit_features.source_code_file_size(),
commit_features.other_file_size(),
commit_features.test_file_size(),
commit_features.source_code_added(),
commit_features.other_added(),
commit_features.test_added(),
commit_features.source_code_deleted(),
commit_features.other_deleted(),
commit_features.test_deleted(),
test_scheduling_features.path_distance(),
test_scheduling_features.common_path_components(),
test_scheduling_features.touched_together(),
]
self.extraction_pipeline = Pipeline([
(
"commit_extractor",
commit_features.CommitExtractor(feature_extractors, []),
),
("union", ColumnTransformer([("data", DictVectorizer(), "data")])),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor="cpu_predictor")
def __init__(self, lemmatization=False, historical=False):
Model.__init__(self, lemmatization)
self.sampler = BorderlineSMOTE(random_state=0)
feature_extractors = [
bug_features.has_str(),
bug_features.severity(),
# Ignore keywords that would make the ML completely skewed
# (we are going to use them as 100% rules in the evaluation phase).
bug_features.keywords(
{"regression", "talos-regression", "feature"}),
bug_features.is_coverity_issue(),
bug_features.has_crash_signature(),
bug_features.has_url(),
bug_features.has_w3c_url(),
bug_features.has_github_url(),
bug_features.whiteboard(),
bug_features.patches(),
bug_features.landings(),
bug_features.title(),
bug_features.blocked_bugs_number(),
bug_features.ever_affected(),
bug_features.affected_then_unaffected(),
bug_features.product(),
bug_features.component(),
]
if historical:
feature_extractors.append(bug_features.had_severity_enhancement())
cleanup_functions = [
bug_features.cleanup_url,
bug_features.cleanup_fileref,
bug_features.cleanup_synonyms,
]
self.extraction_pipeline = Pipeline([
(
"bug_extractor",
bug_features.BugExtractor(feature_extractors,
cleanup_functions),
),
(
"union",
ColumnTransformer([
("data", DictVectorizer(), "data"),
("title", self.text_vectorizer(min_df=0.001), "title"),
(
"first_comment",
self.text_vectorizer(min_df=0.001),
"first_comment",
),
(
"comments",
self.text_vectorizer(min_df=0.001),
"comments",
),
]),
),
])
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor="cpu_predictor")