def generate_file_fixes(self, data_service: DataService, changes: Sequence[Change],
) -> Iterator[FileFix]:
"""
Generate all data required for any type of further processing.
Next processing can be comment generation or performance report generation.
:param data_service: Connection to the Lookout data retrieval service.
:param changes: The list of changes in the pointed state.
:return: Iterator with unrendered data per comment.
"""
log = self._log
base_files_by_lang = files_by_language(c.base for c in changes)
head_files_by_lang = files_by_language(c.head for c in changes)
processed_files_counter = defaultdict(int)
processed_fixes_counter = defaultdict(int)
for lang, head_files in head_files_by_lang.items():
if lang not in self.model:
log.warning("skipped %d written in %s. Rules for %s do not exist in model",
len(head_files), lang, lang)
continue
rules = self.model[lang]
config = self.analyze_config[lang]
rules = rules.filter_by_confidence(config["confidence_threshold"]) \
.filter_by_support(config["support_threshold"])
for file in filter_files(head_files, rules.origin_config["line_length_limit"],
rules.origin_config["overall_size_limit"], log=log):
processed_files_counter[lang] += 1
try:
prev_file = base_files_by_lang[lang][file.path]
except KeyError:
prev_file = None
lines = None
else:
lines = sorted(chain.from_iterable((
find_new_lines(prev_file, file),
find_deleted_lines(prev_file, file),
)))
log.debug("%s %s", file.path, lines)
fe = FeatureExtractor(language=lang, **rules.origin_config["feature_extractor"])
feature_extractor_output = fe.extract_features([file], [lines])
if feature_extractor_output is None:
submit_event("%s.analyze.%s.parse_failures" % (self.name, lang), 1)
if config["report_parse_failures"]:
log.warning("Failed to parse %s", file.path)
yield FileFix(error="Failed to parse", head_file=file, language=lang,
feature_extractor=fe, base_file=prev_file, file_vnodes=[],
line_fixes=[], y_pred_pure=None, y=None)
else:
fixes, file_vnodes, y_pred_pure, y = self._generate_token_fixes(
file, fe, feature_extractor_output, data_service.get_bblfsh(), rules)
log.debug("%s %d fixes", file.path, len(fixes))
processed_fixes_counter[lang] += len(fixes)
yield FileFix(error="", head_file=file, language=lang, feature_extractor=fe,
base_file=prev_file, file_vnodes=file_vnodes, line_fixes=fixes,
y_pred_pure=y_pred_pure, y=y)
for key, val in processed_files_counter.items():
submit_event("%s.analyze.%s.files" % (self.name, key), val)
for key, val in processed_fixes_counter.items():
submit_event("%s.analyze.%s.fixes" % (self.name, key), val)
def generate_local_test(mcs, case_name, uast, contents):
fe_config = FormatAnalyzer._load_config(
get_config())["train"]["javascript"]
feature_extractor = FeatureExtractor(language="javascript",
label_composites=label_composites,
**fe_config["feature_extractor"])
file = UnicodeFile(content=contents, uast=uast, path="", language="")
_, _, (vnodes_y, _, _, _) = feature_extractor.extract_features([file])
offsets, y_pred, result = cases[case_name]
def _test(self):
y_cur = deepcopy(self.y)
for offset, yi in zip(offsets, y_pred):
i = None
for i, vnode in enumerate(vnodes_y): # noqa: B007
if offset == vnode.start.offset:
break
y_cur[i] = yi
code_generator = CodeGenerator(self.feature_extractor)
pred_vnodes = code_generator.apply_predicted_y(
self.vnodes, self.vnodes_y, list(range(len(self.vnodes_y))),
FakeRules(y_cur))
generated_file = code_generator.generate(pred_vnodes)
self.assertEqual(generated_file, result)
return _test
def get_class_sequences_from_code(code: str) -> Sequence[Tuple[int, ...]]:
uast = client.parse(filename="", language="javascript", contents=code.encode()).uast
extractor = FeatureExtractor(language="javascript", **config)
result = extractor.extract_features([UnicodeFile(content=code, uast=uast, path="",
language="javascript")])
if result is None:
self.fail("Could not parse test code.")
_, _, (vnodes_y, _, _, _) = result
return [vnode.y for vnode in vnodes_y]
def return_features() -> Response:
"""Featurize the given code."""
body = request.get_json()
code = body["code"]
babelfish_address = body["babelfish_address"]
language = body["language"]
client = BblfshClient(babelfish_address)
res = client.parse(filename="", contents=code.encode(), language=language)
if res.status != 0:
abort(500)
model = FormatModel().load(str(Path(__file__).parent / "models" / "model.asdf"))
if language not in model:
raise NotFittedError()
rules = model[language]
file = UnicodeFile(content=code, uast=res.uast, language="javascript", path="path")
config = rules.origin_config["feature_extractor"]
config["return_sibling_indices"] = True
fe = FeatureExtractor(language=language, **config)
res = fe.extract_features([file])
if res is None:
abort(500)
X, y, (vnodes_y, vnodes, vnode_parents, node_parents, sibling_indices) = res
y_pred, rule_winners, rules, grouped_quote_predictions = rules.predict(
X=X, vnodes_y=vnodes_y, vnodes=vnodes, feature_extractor=fe)
refuse_to_predict = y_pred < 0
checker = UASTStabilityChecker(fe)
_, _, _, _, safe_preds = checker.check(
y=y, y_pred=y_pred, vnodes_y=vnodes_y, vnodes=vnodes, files=[file], stub=client._stub,
vnode_parents=vnode_parents, node_parents=node_parents, rule_winners=rule_winners,
grouped_quote_predictions=grouped_quote_predictions)
break_uast = [False] * X.shape[0]
for wrong_pred in set(range(X.shape[0])).difference(safe_preds):
break_uast[wrong_pred] = True
labeled_indices = {id(vnode): i for i, vnode in enumerate(vnodes_y)}
app.logger.info("returning features of shape %d, %d" % X.shape)
app.logger.info("length of rules: %d", len(rules))
return jsonify({
"code": code,
"features": _input_matrix_to_descriptions(X, fe),
"ground_truths": y.tolist(),
"predictions": y_pred.tolist(),
"refuse_to_predict": refuse_to_predict.tolist(),
"sibling_indices": sibling_indices,
"rules": _rules_to_jsonable(rules, fe),
"winners": rule_winners.tolist(),
"break_uast": break_uast,
"feature_names": fe.feature_names,
"class_representations": fe.composite_class_representations,
"class_printables": fe.composite_class_printables,
"vnodes": list(map(partial(_vnode_to_jsonable, labeled_indices=labeled_indices), vnodes)),
"config": _mapping_to_jsonable(rules.origin_config)})
def files2vnodes(
files: Iterable[str],
feature_extractor: FeatureExtractor,
client: str,
) -> Iterable[VirtualNode]:
"""
Return the `VirtualNode`-s extracted from a list of files.
:param files: List of files to get `Misprediction`-s and `VirtualNode`-s from.
:param feature_extractor: FeatureExtractor to use.
:param client: Babelfish client. Babelfish server should be started accordingly.
:return: List of `VirtualNode`-s extracted from a given list of files.
"""
files = prepare_files(files, client, feature_extractor.language)
_, _, (vnodes_y, _, _, _) = feature_extractor.extract_features(files)
return vnodes_y
def files2vnodes(filepaths: Iterable[str], feature_extractor: FeatureExtractor, rules: Rules,
client: BblfshClient) -> Iterable[VirtualNode]:
"""
Return the `VirtualNode`-s extracted from a list of files.
:param filepaths: List of files to get `Misprediction`-s and `VirtualNode`-s from.
:param feature_extractor: FeatureExtractor to use.
:param rules: Rules to use for prediction.
:param client: Babelfish client. Babelfish server should be started accordingly.
:return: List of `VirtualNode`-s extracted from a given list of files.
"""
files = parse_files(filepaths=filepaths,
line_length_limit=rules.origin_config["line_length_limit"],
overall_size_limit=rules.origin_config["overall_size_limit"],
client=client, language=feature_extractor.language)
_, _, (vnodes_y, _, _, _) = feature_extractor.extract_features(files)
return vnodes_y
class FeaturesTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
base = Path(__file__).parent
# str() is needed for Python 3.5
with lzma.open(str(base / "benchmark.js.xz"), mode="rt") as fin:
cls.contents = fin.read()
with lzma.open(str(base / "benchmark.uast.xz")) as fin:
cls.uast = bblfsh.Node.FromString(fin.read())
def setUp(self):
config = FormatAnalyzer._load_config(get_config())
self.final_config = config["train"]["javascript"]
self.extractor = FeatureExtractor(language="javascript",
**self.final_config["feature_extractor"])
def test_parse_file_exact_match(self):
test_js_code_filepath = Path(__file__).parent / "for_parse_test.js.xz"
with lzma.open(str(test_js_code_filepath), mode="rt") as f:
code = f.read()
uast = bblfsh.BblfshClient("0.0.0.0:9432").parse(
filename="", language="javascript", contents=code.encode()).uast
nodes, parents = self.extractor._parse_file(code, uast, test_js_code_filepath)
self.assertEqual("".join(n.value for n in nodes), code)
def test_extract_features_exact_match(self):
file = UnicodeFile(content=self.contents,
uast=self.uast, path="test.js", language="javascript")
files = [file]
X, y, (vnodes_y, vnodes, _, _) = self.extractor.extract_features(files)
self.assertEqual("".join(vnode.value for vnode in vnodes), self.contents)
def test_parse_file_comment_after_regexp(self):
code = "x = // comment\n/<regexp>/;"
uast = bblfsh.BblfshClient("0.0.0.0:9432").parse(
filename="", language="javascript", contents=code.encode()).uast
nodes, parents = self.extractor._parse_file(code, uast, "")
self.assertEqual("".join(n.value for n in nodes), code)
def test_parse_file(self):
nodes, parents = self.extractor._parse_file(self.contents, self.uast, "test_file")
text = []
offset = line = col = 0
for n in nodes:
if line == n.start.line - 1:
line += 1
col = 1
self.assertEqual((offset, line, col), n.start, n.value)
text.append(n.value)
if n.node is not None:
self.assertIsNotNone(parents.get(id(n.node)), n)
offset, line, col = n.end
self.assertEqual(len(self.contents), offset)
# New line ends on the next line
self.assertEqual(len(self.contents.splitlines()) + 1, line)
self.assertEqual("".join(text), self.contents)
def test_parse_file_with_trailing_space(self):
contents = self.contents + " "
nodes, parents = self.extractor._parse_file(contents, self.uast, "test_file")
offset, line, col = nodes[-1].end
self.assertEqual(len(contents), offset)
# Space token always ends on the same line
self.assertEqual(len(contents.splitlines()), line)
self.assertEqual("".join(n.value for n in nodes), contents)
def test_classify_vnodes(self):
nodes, _ = self.extractor._parse_file(self.contents, self.uast, "test_file")
nodes = list(self.extractor._classify_vnodes(nodes, "test_file"))
text = "".join(n.value for n in nodes)
self.assertEqual(text, self.contents)
cls_counts = Counter()
offset = line = col = 0
for n in nodes:
if line == n.start.line - 1:
line += 1
col = 1
self.assertEqual((offset, line, col), n.start, n.value)
if n.y is not None:
cls_counts.update(map(CLASSES.__getitem__, n.y))
offset, line, col = n.end
self.assertEqual(len(self.contents), offset)
# New line ends on the next line
self.assertEqual(len(self.contents.splitlines()) + 1, line)
self.assertEqual(cls_counts[CLS_SPACE_INC], cls_counts[CLS_SPACE_DEC])
self.assertGreater(cls_counts[CLS_SPACE_INC], 0)
self.assertGreater(cls_counts[CLS_SPACE], 0)
self.assertGreater(cls_counts[CLS_NEWLINE], 0)
self.assertGreater(cls_counts[CLS_SINGLE_QUOTE], 0)
self.assertTrue(cls_counts[CLS_SINGLE_QUOTE] % 2 == 0)
def test_classify_vnodes_with_trailing_space(self):
contents = self.contents + " "
nodes, _ = self.extractor._parse_file(contents, self.uast, "test_file")
nodes = list(self.extractor._classify_vnodes(nodes, "test_file"))
text = "".join(n.value for n in nodes)
self.assertEqual(text, contents)
cls_counts = Counter()
offset = line = col = 0
for n in nodes:
if line == n.start.line - 1:
line += 1
col = 1
self.assertEqual((offset, line, col), n.start, n.value)
if n.y is not None:
cls_counts.update(map(CLASSES.__getitem__, n.y))
offset, line, col = n.end
self.assertEqual(len(contents), offset)
# Space token always ends on the same line
self.assertEqual(len(contents.splitlines()), line)
self.assertEqual(cls_counts[CLS_SPACE_INC], cls_counts[CLS_SPACE_DEC] + 1)
self.assertGreater(cls_counts[CLS_SPACE_INC], 0)
self.assertGreater(cls_counts[CLS_SPACE], 0)
self.assertGreater(cls_counts[CLS_NEWLINE], 0)
self.assertGreater(cls_counts[CLS_SINGLE_QUOTE], 0)
self.assertTrue(cls_counts[CLS_SINGLE_QUOTE] % 2 == 0)
def test_compute_labels_mappings(self):
pos1, pos2 = Position(1, 1, 1), Position(10, 2, 1)
files = [VirtualNode("", pos1, pos2, y=(1,))] * 2 + \
[VirtualNode("", pos1, pos2), VirtualNode("", pos1, pos2, y=(2,)),
VirtualNode("", pos1, pos2, y=(3,))]
self.extractor.cutoff_label_support = 2
self.extractor._compute_labels_mappings(files)
self.assertEqual(self.extractor.labels_to_class_sequences, [(1,)])
self.assertEqual(self.extractor.class_sequences_to_labels, {(1,): 0})
def test_extract_features(self):
file = UnicodeFile(content=self.contents,
uast=self.uast, path="test.js", language="javascript")
files = [file, file]
res = self.extractor.extract_features(files)
self.assertIsNotNone(res, "Failed to parse files.")
self.check_X_y(*res)
def check_X_y(self, X_csr, y, secondary_features):
X = X_csr.toarray()
vnodes_y, vnodes, vnode_parents, node_parents = secondary_features
self.assertEqual(X.shape[0], y.shape[0])
self.assertEqual(X.shape[0], len(vnodes_y))
self.assertEqual(len(vnodes), len(vnode_parents))
for vn in vnodes_y:
self.assertIsInstance(vn, VirtualNode)
self.assertEqual(type(vnode_parents[id(vnodes[0])]).__module__, bblfsh.Node.__module__)
for _, node in node_parents.items():
self.assertEqual(type(node).__module__, bblfsh.Node.__module__)
self.assertEqual(X.shape[1], self.extractor.count_features())
not_set = X == -1
unset_rows = numpy.nonzero(numpy.all(not_set, axis=1))[0]
unset_columns = numpy.nonzero(numpy.all(not_set, axis=0))[0]
self.assertEqual(len(unset_rows), 0, "%d rows are unset" % len(unset_rows))
self.assertEqual(len(unset_columns), 0,
"columns %s are unset" % ", ".join(map(str, unset_columns)))
def test_extract_features_all_lines(self):
file = UnicodeFile(content=self.contents, uast=self.uast, path="test.js",
language="javascript")
files = [file, file]
self.check_X_y(*self.extractor.extract_features(
files, [list(range(1, self.contents.count("\n") + 1))] * 2))
def test_empty_strings(self):
config = deepcopy(self.final_config["feature_extractor"])
config["cutoff_label_support"] = 0
client = bblfsh.BblfshClient("0.0.0.0:9432")
def get_class_sequences_from_code(code: str) -> Sequence[Tuple[int, ...]]:
uast = client.parse(filename="", language="javascript", contents=code.encode()).uast
extractor = FeatureExtractor(language="javascript", **config)
result = extractor.extract_features([UnicodeFile(content=code, uast=uast, path="",
language="javascript")])
if result is None:
self.fail("Could not parse test code.")
_, _, (vnodes_y, _, _, _) = result
return [vnode.y for vnode in vnodes_y]
self.assertEqual(get_class_sequences_from_code("var a = '';"),
get_class_sequences_from_code("var a = 'a';"))
def test_extract_features_some_lines(self):
file = UnicodeFile(content=self.contents,
uast=self.uast, path="test.js", language="javascript")
files = [file]
X1_csr, y1, (vn1_y, vn1, vn1_parents, n1_parents) = self.extractor.extract_features(
files, [list(range(1, self.contents.count("\n") // 2 + 1))] * 2)
self.check_X_y(X1_csr, y1, (vn1_y, vn1, vn1_parents, n1_parents))
X2_csr, y2, (vn2_y, vn2, _, _) = self.extractor.extract_features(files)
X1, X2 = X1_csr.toarray(), X2_csr.toarray()
self.assertTrue((X1 == X2[:len(X1)]).all())
self.assertTrue((y1 == y2[:len(y1)]).all())
self.assertTrue(vn1_y == vn2_y[:len(vn1_y)])
self.assertLess(len(y1), len(y2))
def test_noop_vnodes(self):
vnodes, parents = self.extractor._parse_file(self.contents, self.uast, "test_file")
vnodes = self.extractor._classify_vnodes(vnodes, "test_file")
vnodes = self.extractor._merge_classes_to_composite_labels(
vnodes, "test_file", index_labels=True)
vnodes = self.extractor._add_noops(list(vnodes), "test_file", index_labels=True)
for vnode1, vnode2, vnode3 in zip(vnodes,
islice(vnodes, 1, None),
islice(vnodes, 2, None)):
if vnode1.y is not None or vnode3.y is not None:
self.assertNotIn(CLASS_INDEX[CLS_NOOP], vnode2.y if vnode2.y else set(),
"\n".join(map(repr, [vnode1, vnode2, vnode3])))
def train(cls, ptr: ReferencePointer, config: Mapping[str, Any], data_service: DataService,
files: Iterator[File], **data) -> FormatModel:
"""
Train a model given the files available.
:param ptr: Git repository state pointer.
:param config: configuration dict.
:param data: contains "files" - the list of files in the pointed state.
:param data_service: connection to the Lookout data retrieval service.
:param files: iterator of File records from the data service.
:return: AnalyzerModel containing the learned rules, per language.
"""
_log = logging.getLogger(cls.__name__)
train_config = cls._load_config(config)["train"]
_log.info("train %s %s %s %s", __version__, ptr.url, ptr.commit,
pformat(train_config, width=4096, compact=True))
model = FormatModel().generate(cls, ptr)
for language, files in files_by_language(files).items():
try:
lang_config = train_config[language]
except KeyError:
_log.warning("language %s is not supported, skipped", language)
continue
_log.info("effective train config for %s:\n%s", language,
pformat(lang_config, width=120, compact=True))
random_state = lang_config["random_state"]
files = filter_files(
files, lang_config["line_length_limit"], lang_config["overall_size_limit"],
random_state, _log)
submit_event("%s.train.%s.files" % (cls.name, language), len(files))
if len(files) == 0:
_log.info("zero files after filtering, language %s is skipped.", language)
continue
try:
fe = FeatureExtractor(language=language, **lang_config["feature_extractor"])
except ImportError:
_log.warning("skipped %d %s files - not supported", len(files), language)
continue
else:
_log.info("training on %d %s files", len(files), language)
train_files, test_files = FormatAnalyzer.split_train_test(
files, lang_config["test_dataset_ratio"], random_state=random_state)
# ensure that the features are reproducible
train_files = sorted(train_files, key=lambda x: x.path)
test_files = sorted(test_files, key=lambda x: x.path)
X_train, y_train, _ = fe.extract_features(train_files)
X_train, selected_features = fe.select_features(X_train, y_train)
if test_files:
X_test, y_test, _ = fe.extract_features(test_files)
if lang_config["test_dataset_ratio"]:
_log.debug("Real test ratio is %.3f",
X_test.shape[0] / (X_test.shape[0] + X_train.shape[0])
if test_files else 0)
lang_config["feature_extractor"]["selected_features"] = selected_features
lang_config["feature_extractor"]["label_composites"] = fe.labels_to_class_sequences
lower_bound_instances = lang_config["lower_bound_instances"]
if X_train.shape[0] < lower_bound_instances:
_log.warning("skipped %d %s files: too few samples (%d/%d)",
len(files), language, X_train.shape[0], lower_bound_instances)
continue
_log.info("extracted %d samples to train, searching for the best hyperparameters",
X_train.shape[0])
optimizer = Optimizer(**lang_config["optimizer"], random_state=random_state)
best_score, best_params = optimizer.optimize(X_train, y_train)
if _log.isEnabledFor(logging.DEBUG):
_log.debug("score of the best estimator found: %.6f", best_score)
_log.debug("params of the best estimator found: %s", str(best_params))
_log.debug("training the model with complete data")
else:
_log.info("finished hyperopt at %.6f, training the full model", -best_score)
lang_config["trainable_rules"].update(best_params)
trainable_rules = TrainableRules(**lang_config["trainable_rules"],
random_state=random_state,
origin_config=lang_config)
trainable_rules.fit(X_train, y_train)
importances = trainable_rules.feature_importances_
_log.debug(
"feature importances from %s:\n\t%s",
lang_config["trainable_rules"]["base_model_name"],
"\n\t".join("%-55s %.5E" % (fe.feature_names[i], importances[i])
for i in numpy.argsort(-importances)[:25] if importances[i] > 1e-5))
trainable_rules.prune_categorical_attributes(fe)
_log.info("obtained %d rules, generating the classification report",
len(trainable_rules.rules))
trainable_rules.rules.generate_classification_report(
X_train, y_train, "train", fe.composite_class_representations)
if test_files:
trainable_rules.rules.generate_classification_report(
X_test, y_test, "test", fe.composite_class_representations)
submit_event("%s.train.%s.rules" % (cls.name, language), len(trainable_rules.rules))
if trainable_rules.rules.rules:
model[language] = trainable_rules.rules
else:
_log.warning("model for %s has 0 rules. Skipped.", language)
_log.info("trained %s", model)
return model
class PostprocessingTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
slogging_setup("DEBUG", False)
cls.language = "javascript"
cls.bblfsh_client = bblfsh.BblfshClient("0.0.0.0:9432")
cls.data_service = FakeDataService(cls.bblfsh_client,
files=None,
changes=None)
cls.stub = cls.data_service.get_bblfsh()
cls.config = FormatAnalyzer._load_config({
"train": {
"language_defaults": {
"feature_extractor": {
"cutoff_label_support": 0
}
}
},
})["train"][cls.language]["feature_extractor"]
@classmethod
def tearDownClass(cls):
cls.bblfsh_client._channel.close()
def setUp(self):
self.fe = FeatureExtractor(language=self.language, **self.config)
def _to_label(self, classes: Sequence[str]) -> Tuple[int, ...]:
return self.fe.class_sequences_to_labels[tuple(CLASS_INDEX[cls]
for cls in classes)]
@staticmethod
def _grouped_predictions_mapping(vnodes: Sequence[VirtualNode],
indices: Optional[Sequence[int]]):
result = OrderedDict()
if indices is None:
return result
y_index = [i for i, vnode in enumerate(vnodes) if vnode.y is not None]
for i in indices:
y_i = y_index[i]
result[id(vnodes[y_i])] = (vnodes[y_i], vnodes[y_i + 1],
vnodes[y_i + 2])
result[id(vnodes[y_i + 2])] = None
return result
def edit_and_test(self,
code: str,
modifs: Mapping[int, Sequence[str]],
*,
quote_indices: Optional[Tuple[int, ...]] = None,
bad_indices: Optional[FrozenSet[int]] = None) -> None:
uast, errors = parse_uast(self.stub,
code,
filename="",
language=self.language)
if errors:
self.fail("Could not parse the testing code.")
file = File(content=code.encode(), uast=uast, path="test_file")
X, y, (vnodes_y, vnodes, vnode_parents,
node_parents) = self.fe.extract_features([file])
y_pred = y.copy()
rule_winners = numpy.zeros(y.shape)
for index, classes in modifs.items():
y_pred[index] = self._to_label(classes)
checker = UASTStabilityChecker(self.fe)
grouped_quote_predictions = self._grouped_predictions_mapping(
vnodes, quote_indices)
new_y, new_y_pred, new_vnodes_y, new_rule_winners, safe_preds = checker.check(
y,
y_pred,
vnodes_y,
vnodes, [file],
self.stub,
vnode_parents,
node_parents,
rule_winners,
grouped_quote_predictions=grouped_quote_predictions)
bad_preds = set(range(y.shape[0])) - set(safe_preds)
bad = modifs.keys() if bad_indices is None else bad_indices
self.assertEqual(bad_preds, bad)
self.assertEqual(len(y) - len(bad), len(new_y))
self.assertEqual(len(y_pred) - len(bad), len(new_y_pred))
self.assertEqual(len(vnodes_y) - len(bad), len(new_vnodes_y))
self.assertEqual(len(rule_winners) - len(bad), len(new_rule_winners))
def test_posprocess(self):
self.edit_and_test("var a = 0", {1: (CLS_NOOP, )})
def test_bad_and_good_quotes(self):
self.edit_and_test("""var a = '"0"'; var c = "0";""", {
4: (CLS_DOUBLE_QUOTE, ),
5: (CLS_DOUBLE_QUOTE, ),
10: (CLS_SINGLE_QUOTE, ),
11: (CLS_SINGLE_QUOTE, )
},
quote_indices=(4, 10),
bad_indices=frozenset((4, 5)))
def test_lonely_quote(self):
self.edit_and_test("var a = 0; var b = 'c';", {2: (CLS_SINGLE_QUOTE)},
quote_indices=(9, ))
def test_multiple_files(self):
data = [
("var a = 0", {
1: (CLS_NOOP, )
}),
("var b = 123", {
4: (CLS_NOOP, )
}),
]
files = []
for i, (code, _) in enumerate(data):
uast, errors = parse_uast(self.stub,
code,
filename="",
language=self.language)
if errors:
self.fail("Could not parse the testing code.")
files.append(
File(content=code.encode(), uast=uast,
path="test_file_%d" % i))
X, y, (vnodes_y, vnodes, vnode_parents,
node_parents) = self.fe.extract_features(files)
y_pred = y.copy()
rule_winners = numpy.zeros(y.shape)
for (_, modif) in data:
for i in modif:
y_pred[i] = self._to_label(modif[i])
checker = UASTStabilityChecker(self.fe)
new_y, new_y_pred, new_vnodes_y, new_rule_winners, safe_preds = checker.check(
y,
y_pred,
vnodes_y,
vnodes,
files,
self.stub,
vnode_parents,
node_parents,
rule_winners,
grouped_quote_predictions={})
self.assertEqual(list(safe_preds), [0, 2, 3, 4, 5, 6, 7, 8])
def test_generate_new_line(self):
self.maxDiff = None
expected_res = {
"nothing changed": [],
"remove new line in the end of 4th line":
None,
"indentation in the beginning":
[" import { makeToast } from '../../common/app/Toasts/redux';"],
"remove indentation in the 4th line till the end":
[" return Object.keys(flash)", " }"],
"new line between 6th and 7th regular code lines":
["\n return messages.map(message => ({"],
"new line in the middle of the 7th code line with indentation increase":
[" return messages\n .map(message => ({", " })"],
"new line in the middle of the 7th code line with indentation decrease":
[" return messages\n .map(message => ({", " })"],
"new line in the middle of the 7th code line without indentation increase":
[" return messages\n .map(message => ({"],
"change quotes":
['import { makeToast } from "../../common/app/Toasts/redux";'],
"remove indentation decrease 11th line": [" }));"],
"change indentation decrease to indentation increase 11th line":
[" }));"],
"change indentation decrease to indentation increase 11th line but keep the rest":
[" }));", "})"],
}
base = Path(__file__).parent
# str() is needed for Python 3.5
with lzma.open(str(base / "benchmark_small.js.xz"), mode="rt") as fin:
contents = fin.read()
with lzma.open(str(base / "benchmark_small.js.uast.xz")) as fin:
uast = bblfsh.Node.FromString(fin.read())
config = FormatAnalyzer._load_config(get_config())
fe_config = config["train"]["javascript"]
for case in expected_res:
offsets, y_pred, _ = cases[case]
feature_extractor = FeatureExtractor(
language="javascript",
label_composites=label_composites,
**fe_config["feature_extractor"])
file = UnicodeFile(content=contents,
uast=uast,
path="",
language="")
X, y, (vnodes_y, vnodes, vnode_parents, node_parents) = \
feature_extractor.extract_features([file])
y_cur = deepcopy(y)
for offset, yi in zip(offsets, y_pred):
i = None
for i, vnode in enumerate(vnodes_y): # noqa: B007
if offset == vnode.start.offset:
break
y_cur[i] = yi
code_generator = CodeGenerator(feature_extractor)
pred_vnodes = code_generator.apply_predicted_y(
vnodes, vnodes_y, list(range(len(vnodes_y))), FakeRules(y_cur))
res = []
for gln in FormatAnalyzer._group_line_nodes(
y, y_cur, vnodes_y, pred_vnodes, [1] * len(y)):
line, (line_y, line_y_pred, line_vnodes_y, line_vnodes,
line_rule_winners) = gln
new_code_line = code_generator.generate_new_line(line_vnodes)
res.append(new_code_line)
if expected_res[case] is not None:
# None means that we delete some lines. We are not handle this properly now.
self.assertEqual(res, expected_res[case], case)
def visualize(input_filename: str, bblfsh: str, language: str,
model_path: str) -> None:
"""Visualize the errors made on a single file."""
model = FormatModel().load(model_path)
rules = model[language]
print("Model parameters: %s" % rules.origin)
print("Stats about rules: %s" % rules)
client = BblfshClient(bblfsh)
file = prepare_file(input_filename, client, language)
fe = FeatureExtractor(language=language,
**rules.origin_config["feature_extractor"])
X, y, vnodes_y, vnodes = fe.extract_features([file])
y_pred, _, _ = rules.predict(X, vnodes_y, vnodes, fe)
# collect lines with mispredictions - could be removed
mispred_lines = set()
lines = set()
for gt, pred, node in zip(y, y_pred, vnodes_y):
lines.add((node.path, node.start.line))
if gt != pred:
mispred_lines.add((node.path, node.start.line))
print("Number of lines with mispredictions: %s out of %s mispredicted" %
(len(mispred_lines), len(lines)))
# collect mispredictions and all other predictions for each line with mistake
mispred = defaultdict(list)
for gt, pred, node in zip(y, y_pred, vnodes_y):
if (node.path, node.start.line) in mispred_lines:
mispred[(node.path,
node.start.line)].append(Misprediction(gt, pred, node))
# sort each line
for value in mispred.values():
value.sort(key=lambda k: k.node.start.offset)
# final mispredictions
final_mispred = []
for line in sorted(mispred):
gt = [m.y for m in mispred[line]]
pred = [m.pred for m in mispred[line]]
s = SequenceMatcher(None, gt, pred)
blocks = s.get_matching_blocks()
if blocks[0].a != 0:
# mispredictions before the first matching block
final_mispred.extend(mispred[line][:blocks[0].a])
for i in range(len(blocks) - 1):
final_mispred.extend(mispred[line][blocks[i].a:blocks[i + 1].a])
if blocks[-1].a != len(mispred[line]):
# mispredictions after the last matching block
final_mispred.extend(mispred[line][blocks[-1].a:])
mispred = sorted([misp for misp in final_mispred if misp.y != misp.pred],
key=lambda r: r.node.start.offset)
new_content = ENDC
old_content = file.content.decode("utf-8")
for i in range(len(mispred)):
wrong = mispred[i]
start = wrong.node.start.offset
end = wrong.node.end.offset
if end == start:
end = start + len(wrong.node.value)
if i == 0 and start != 0:
new_content += old_content[:start]
new_content += GREEN + CLASSES[wrong.y] + RED + CLASSES[
wrong.pred] + ENDC
if i == len(mispred) - 1:
if end != len(old_content):
new_content += old_content[end:]
else:
new_content += old_content[end:mispred[i + 1].node.start.offset]
print("Visualization:\n" + new_content)
def train(cls, ptr: ReferencePointer, config: Mapping[str, Any],
data_service: DataService, **data) -> FormatModel:
"""
Train a model given the files available.
:param ptr: Git repository state pointer.
:param config: configuration dict.
:param data: contains "files" - the list of files in the pointed state.
:param data_service: connection to the Lookout data retrieval service.
:return: AnalyzerModel containing the learned rules, per language.
"""
_log = logging.getLogger(cls.__name__)
_log.info("train %s %s %s", ptr.url, ptr.commit,
pformat(config, width=4096, compact=True))
model = FormatModel().construct(cls, ptr)
config = cls._load_train_config(config)
for language, files in files_by_language(data["files"]).items():
try:
lang_config = config[language]
except KeyError:
_log.warning("language %s is not supported, skipped", language)
continue
files = filter_files(files, lang_config["line_length_limit"], _log)
submit_event("%s.train.%s.files" % (cls.name, language),
len(files))
if len(files) == 0:
_log.info(
"zero files after filtering, language %s is skipped.",
language)
continue
try:
fe = FeatureExtractor(language=language,
**lang_config["feature_extractor"])
except ImportError:
_log.warning("skipped %d %s files - not supported", len(files),
language)
continue
else:
_log.info("training on %d %s files", len(files), language)
# we sort to make the features reproducible
X, y, _ = fe.extract_features(sorted(files, key=lambda x: x.path))
X, selected_features = fe.select_features(X, y)
lang_config["feature_extractor"][
"selected_features"] = selected_features
lang_config["feature_extractor"][
"label_composites"] = fe.labels_to_class_sequences
lower_bound_instances = lang_config["lower_bound_instances"]
if X.shape[0] < lower_bound_instances:
_log.warning("skipped %d %s files: too few samples (%d/%d)",
len(files), language, X.shape[0],
lower_bound_instances)
continue
_log.debug("training the rules model")
optimizer = Optimizer(
n_jobs=lang_config["n_jobs"],
n_iter=lang_config["n_iter"],
cv=lang_config["cv"],
random_state=lang_config["trainable_rules"]["random_state"])
best_score, best_params = optimizer.optimize(X, y)
_log.debug("score of the best estimator found: %.6f", best_score)
_log.debug("params of the best estimator found: %s",
str(best_params))
_log.debug("training the model with complete data")
lang_config["trainable_rules"].update(best_params)
trainable_rules = TrainableRules(**lang_config["trainable_rules"],
origin_config=lang_config)
trainable_rules.fit(X, y)
importances = trainable_rules.feature_importances_
_log.debug(
"feature importances from %s:\n\t%s",
lang_config["trainable_rules"]["base_model_name"], "\n\t".join(
"%-55s %.5E" % (fe.feature_names[i], importances[i])
for i in numpy.argsort(-importances)[:25]
if importances[i] > 1e-5))
submit_event("%s.train.%s.rules" % (cls.name, language),
len(trainable_rules.rules))
# TODO(vmarkovtsev): save the achieved precision, recall, etc. to the model
# throw away imprecise classes
if trainable_rules.rules.rules:
model[language] = trainable_rules.rules
else:
_log.warning("model for %s has 0 rules. Skipping.", language)
_log.info("trained %s", model)
return model