def analyze(self, ptr_from: ReferencePointer, ptr_to: ReferencePointer, # noqa: D
data_service: DataService, changes: Iterable[Change]) -> [Comment]:
self._log.info("analyze %s %s", ptr_from.commit, ptr_to.commit)
comments = []
parser = TokenParser(stem_threshold=100, single_shot=True)
words = autocorrect.word.KNOWN_WORDS.copy()
try:
for name in self.model.names:
if len(name) >= 3:
autocorrect.word.KNOWN_WORDS.add(name)
for change in changes:
suggestions = defaultdict(list)
new_lines = set(find_new_lines(change.base, change.head))
for node in bblfsh.filter(change.head.uast, "//*[@roleIdentifier]"):
if node.start_position is not None and node.start_position.line in new_lines:
for part in parser.split(node.token):
if part not in self.model.names:
fixed = autocorrect.spell(part)
if fixed != part:
suggestions[node.start_position.line].append(
(node.token, part, fixed))
for line, s in suggestions.items():
comment = Comment()
comment.file = change.head.path
comment.text = "\n".join("`%s`: %s > %s" % fix for fix in s)
comment.line = line
comment.confidence = 100
comments.append(comment)
finally:
autocorrect.word.KNOWN_WORDS = words
return comments
def __init__(self, model: AnalyzerModel, url: str, config: Mapping[str,
Any]):
"""
Initialize a new instance of IdTyposAnalyzer.
:param model: The instance of the model loaded from the repository or freshly trained.
:param url: The analyzed project's Git remote.
:param config: Configuration of the analyzer of unspecified structure.
"""
super().__init__(model, url, config)
self.model = self.corrector_manager.get(config.get("model"))
self.n_candidates = config.get("n_candidates",
self.DEFAULT_N_CANDIDATES)
self.confidence_threshold = config.get(
"confidence_threshold", self.DEFAULT_CONFIDENCE_THRESHOLD)
self.parser = TokenParser(stem_threshold=40, single_shot=True)
def __init__(self, token2index=None, token_parser=None):
"""
:param token2index: The mapping from tokens to bag keys. If None, no mapping is performed.
:param token_parser: Specify token parser if you want to use a custom one. \
:class:'TokenParser' is used if it is not specified.
"""
token_parser = TokenParser() if token_parser is None else token_parser
super().__init__(token2index, token_parser)
def create_token_parser() -> TokenParser:
"""
Create instance of TokenParser that should be used by IdTyposAnalyzer.
:return: TokenParser.
"""
return TokenParser(stem_threshold=1000,
single_shot=True,
min_split_length=1)
def train(cls, ptr: ReferencePointer, config: Dict[str, Any], data_service: DataService, # noqa: D
files: Iterable[File]) -> AnalyzerModel:
cls._log.info("train %s %s", ptr.url, ptr.commit)
model = cls.construct_model(ptr)
uast2ids = UastIds2Bag(token_parser=TokenParser(stem_threshold=100))
for file in files:
model.names.update(uast2ids(file.uast))
cls._log.info("Parsed %d names", len(model.names))
return model
def test_process_token_with_attach_upper(self):
tp = TokenParser(stem_threshold=100, single_shot=True, max_token_length=100,
min_split_length=1)
tokens = [
("ONLYCAPS", ["onlycaps"]),
("nocaps", ["nocaps"]),
("UpperCamelCase", ["upper", "camel", "case"]),
("camelCase", ["camel", "case"]),
("FRAPScase", ["frap", "scase"]),
("SQLThing", ["sql", "thing"]),
("_Astra", ["astra"]),
("CAPS_CONST", ["caps", "const"]),
("_something_SILLY_", ["something", "silly"]),
("blink182", ["blink"]),
("FooBar100500Bingo", ["foo", "bar", "bingo"]),
("Man45var", ["man", "var"]),
("method_name", ["method", "name"]),
("Method_Name", ["method", "name"]),
("101dalms", ["dalms"]),
("101_dalms", ["dalms"]),
("101_DalmsBug", ["dalms", "bug"]),
("101_Dalms45Bug7", ["dalms", "bug"]),
("wdSize", ["wd", "size"]),
("Glint", ["glint"]),
("foo_BAR", ["foo", "bar"]),
("sourced.ml.algorithms.uast_ids_to_bag",
["sourced", "ml", "algorithms", "uast", "ids", "to", "bag"]),
("WORSTnameYOUcanIMAGINE", ["wors", "tname", "yo", "ucan", "imagine"]),
# Another bad example. Parser failed to parse it correctly
("SmallIdsToFoOo", ["small", "ids", "to", "fo", "oo"]),
("SmallIdFooo", ["small", "id", "fooo"]),
("ONE_M0re_.__badId.example", ["one", "m", "re", "bad",
"id", "example"]),
("never_use_Such__varsableNames", ["never", "use", "such", "varsable", "names"]),
("a.b.c.d", ["a", "b", "c", "d"]),
("A.b.Cd.E", ["a", "b", "cd", "e"]),
("looong_sh_loooong_sh", ["looong", "sh", "loooong", "sh"]),
("sh_sh_sh_sh", ["sh", "sh", "sh", "sh"]),
("loooong_loooong_loooong", ["loooong", "loooong", "loooong"])
]
for token, correct in tokens:
res = list(tp.process_token(token))
self.assertEqual(res, correct)
def process(self, rows: RDD):
"""
Process rows to gather identifier without frequencies.
"""
return rows \
.keys() \
.distinct() \
.map(lambda x: Row(
token=x,
token_split=" ".join(TokenParser(min_split_length=1).split(x))))
def remove_non_typos(dataset: str, filtered_dataset: str) -> None:
"""
Remove non-typo-ed identifiers from the dataset.
1. Remove examples, where token splits of the wrong and the correct identifiers are equal \
(they differ in non-alpha chars or casing).
2. Remove examples, where wrong and correct identifiers are equal on lemmas level.
:param dataset: Path to the dataset.
:param filtered_dataset: Path to save the filtered dataset to.
"""
data = pandas.read_csv(dataset, header=0, usecols=[0, 1], names=["wrong", "correct"],
keep_default_na=False)
# Filter examples with equal splits
tp = TokenParser(min_split_length=1, stem_threshold=400, single_shot=True,
max_token_length=400, attach_upper=True)
data["wrong_split"] = data["wrong"].apply(lambda x: " ".join(tp.split(x)))
data["correct_split"] = data["correct"].apply(lambda x: " ".join(tp.split(x)))
data = data[data["wrong_split"] != data["correct_split"]]
os.system("python3 -m spacy download en")
nlp = spacy.load("en", disable=["parser", "ner"])
# Filter examples with equal lemmas
def _lemmatize(token):
lemm = nlp(token)
if len(lemm) > 1 or lemm[0].lemma_ == "-PRON-" or (
token[-2:] == "ss" and lemm[0].lemma_ == token[:-1]):
return token
return lemm[0].lemma_
data["wrong_lem"] = data["wrong_split"].apply(
lambda x: " ".join(_lemmatize(token) for token in x.split()))
data["correct_lem"] = data["correct_split"].apply(
lambda x: " ".join(_lemmatize(token) for token in x.split()))
data = data[(data["wrong_lem"] != data["correct_lem"]) &
(data["wrong_lem"] != data["correct_split"]) &
(data["correct_lem"] != data["wrong_split"])]
# Save new dataset
whole_data = pandas.read_csv(dataset, header=0, keep_default_na=False)
whole_data = whole_data.loc[data.index]
whole_data.to_csv(filtered_dataset, compression="xz", index=False)
def process_tokens(self, tokens: Generator):
"""
Filter tokens of type "Name" and which are splittable
according to :class: 'TokenParser' rules
"""
for _type, token in tokens:
if _type[0] == "Name":
if self.split:
if sum(1 for _ in TokenParser(min_split_length=1).split(token)) > 1:
self.names.append(token)
else:
self.names.append(token)
def reconstruct_identifier(tokenizer: TokenParser, pred_tokens: List[str], identifier: str) \
-> str:
"""
Reconstruct identifier given predicted tokens and initial identifier.
:param tokenizer: tokenizer - instance of TokenParser.
:param pred_tokens: list of predicted tokens.
:param identifier: identifier.
:return: reconstructed identifier based on predicted tokens.
"""
identifier_l = identifier.lower()
# check required parameters
assert tokenizer._single_shot, "TokenParser should be initialized with " \
"`single_shot=True` for IdTyposAnalyzer"
# sanity checking
initial_tokens = list(tokenizer.split(identifier))
err = "Number of predicted tokens (%s) not equal to the number of tokens in the " \
"identifier (%s) for identifier '%s', predicted_tokens '%s', tokens in identifier " \
"'%s'"
assert len(initial_tokens) == len(pred_tokens), \
err % (len(initial_tokens), len(pred_tokens), identifier, pred_tokens, initial_tokens)
# reconstruction
res = []
prev_end = 0
for token, pred_token in zip(initial_tokens, pred_tokens):
curr = identifier_l.find(token, prev_end)
assert curr != -1, "TokenParser is broken, the subtoken `%s` was not found in the " \
"identifier `%s`" % (token, identifier)
if curr != prev_end:
# delimiter found
res.append(identifier[prev_end:curr])
if identifier[curr:curr + len(token)].isupper():
# upper case
res.append(pred_token.upper())
elif identifier[curr:curr + len(token)][0].isupper():
# capitalized
res.append(pred_token[0].upper() + pred_token[1:])
else:
res.append(pred_token)
prev_end = curr + len(token)
if prev_end != len(identifier):
# suffix
res.append(identifier[prev_end:])
return "".join(res)
def process_stats(self, rows: RDD):
"""
Process rows to gather identifier frequencies.
num_repos is the number of repositories where the identifier appears in.
num_files is the number of files where the identifier appears in.
num_occ is the total number of occurrences of the identifier.
"""
list_RDDs = []
for i in (0, 1):
# initial structure of x: (identifier, (repositoryId, filepath))
freq_processed = rows.map(lambda x: (x[0], x[1][i])).distinct()
list_RDDs.append(self.reduce_rows(freq_processed))
list_RDDs.append(self.reduce_rows(rows))
return rows \
.context.union(list_RDDs) \
.groupByKey() \
.mapValues(list) \
.map(lambda x: Row(
token=x[0],
token_split=" ".join(TokenParser(min_split_length=1).split(x[0])),
num_repos=x[1][0],
num_files=x[1][1],
num_occ=x[1][2]))
def setUp(self):
self.tp = TokenParser(stem_threshold=4, max_token_length=20)
self.tp._single_shot = False
class TokenParserTests(unittest.TestCase):
def setUp(self):
self.tp = TokenParser(stem_threshold=4, max_token_length=20)
self.tp._single_shot = False
def test_process_token(self):
self.tp.max_token_length = 100
tokens = [
("UpperCamelCase", ["upper", "camel", "case"]),
("camelCase", ["camel", "case"]),
("FRAPScase", ["frap", "case"]),
("SQLThing", ["sqlt", "hing"]),
("_Astra", ["astra"]),
("CAPS_CONST", ["caps", "const"]),
("_something_SILLY_", ["someth", "silli"]),
("blink182", ["blink"]),
("FooBar100500Bingo", ["foo", "bar", "bingo"]),
("Man45var", ["man", "var"]),
("method_name", ["method", "name"]),
("Method_Name", ["method", "name"]),
("101dalms", ["dalm"]),
("101_dalms", ["dalm"]),
("101_DalmsBug", ["dalm", "bug"]),
("101_Dalms45Bug7", ["dalm", "bug"]),
("wdSize", ["wd", "size", "wdsize"]),
("Glint", ["glint"]),
("foo_BAR", ["foo", "bar"]),
("sourced.ml.algorithms.uast_ids_to_bag", [
"sourc", "sourcedml", "algorithm", "mlalgorithm", "uast",
"ids", "idsto", "bag", "tobag"
]),
("WORSTnameYOUcanIMAGINE",
["worst", "name", "you", "can", "imagin"]),
# Another bad example. Parser failed to parse it correctly
("SmallIdsToFoOo", ["small", "ids", "idsto", "fo", "oo"]),
("SmallIdFooo", ["small", "smallid", "fooo", "idfooo"]),
("ONE_M0re_.__badId.example", [
"one", "onem", "re", "bad", "rebad", "badid", "exampl",
"idexampl"
]),
("never_use_Such__varsableNames",
["never", "use", "such", "varsabl", "name"]),
("a.b.c.d", ["a", "b", "c", "d"]),
("A.b.Cd.E", ["a", "b", "cd", "e"]),
("looong_sh_loooong_sh",
["looong", "looongsh", "loooong", "shloooong", "loooongsh"]),
("sh_sh_sh_sh", ["sh", "sh", "sh", "sh"]),
("loooong_loooong_loooong", ["loooong", "loooong", "loooong"])
]
for token, correct in tokens:
res = list(self.tp.process_token(token))
self.assertEqual(res, correct)
def test_process_token_single_shot(self):
self.tp.max_token_length = 100
self.tp._single_shot = True
self.tp.min_split_length = 1
tokens = [
("UpperCamelCase", ["upper", "camel", "case"]),
("camelCase", ["camel", "case"]),
("FRAPScase", ["frap", "case"]),
("SQLThing", ["sqlt", "hing"]),
("_Astra", ["astra"]),
("CAPS_CONST", ["caps", "const"]),
("_something_SILLY_", ["someth", "silli"]),
("blink182", ["blink"]),
("FooBar100500Bingo", ["foo", "bar", "bingo"]),
("Man45var", ["man", "var"]),
("method_name", ["method", "name"]),
("Method_Name", ["method", "name"]),
("101dalms", ["dalm"]),
("101_dalms", ["dalm"]),
("101_DalmsBug", ["dalm", "bug"]),
("101_Dalms45Bug7", ["dalm", "bug"]),
("wdSize", ["wd", "size"]),
("Glint", ["glint"]),
("foo_BAR", ["foo", "bar"]),
("sourced.ml.algorithms.uast_ids_to_bag",
["sourc", "ml", "algorithm", "uast", "ids", "to", "bag"]),
("WORSTnameYOUcanIMAGINE",
["worst", "name", "you", "can", "imagin"]),
# Another bad example. Parser failed to parse it correctly
("SmallIdsToFoOo", ["small", "ids", "to", "fo", "oo"]),
("SmallIdFooo", ["small", "id", "fooo"]),
("ONE_M0re_.__badId.example",
["one", "m", "re", "bad", "id", "exampl"]),
("never_use_Such__varsableNames",
["never", "use", "such", "varsabl", "name"]),
("a.b.c.d", ["a", "b", "c", "d"]),
("A.b.Cd.E", ["a", "b", "cd", "e"]),
("looong_sh_loooong_sh", ["looong", "sh", "loooong", "sh"]),
("sh_sh_sh_sh", ["sh", "sh", "sh", "sh"]),
("loooong_loooong_loooong", ["loooong", "loooong", "loooong"])
]
for token, correct in tokens:
res = list(self.tp.process_token(token))
self.assertEqual(res, correct)
min_split_length = 3
self.tp.min_split_length = min_split_length
for token, correct in tokens:
res = list(self.tp.process_token(token))
self.assertEqual(
res, [c for c in correct if len(c) >= min_split_length])
def test_split(self):
self.assertEqual(list(self.tp.split("set for")), ["set", "for"])
self.assertEqual(list(self.tp.split("set /for.")), ["set", "for"])
self.assertEqual(list(self.tp.split("NeverHav")), ["never", "hav"])
self.assertEqual(list(self.tp.split("PrintAll")), ["print", "all"])
self.assertEqual(list(self.tp.split("PrintAllExcept")),
["print", "all", "except"])
self.assertEqual(
list(self.tp.split("print really long line")),
# 'longli' is expected artifact due to edge effects
["print", "really", "long", "longli"])
self.assertEqual(list(self.tp.split("set /for. *&PrintAll")),
["set", "for", "print", "all"])
self.assertEqual(list(self.tp.split("JumpDown not Here")),
["jump", "down", "not", "here"])
self.assertEqual(list(self.tp.split("a b c d")), ["a", "b", "c", "d"])
self.assertEqual(list(self.tp.split("a b long c d")),
["a", "b", "long", "blong", "longc", "d"])
self.assertEqual(list(self.tp.split("AbCd")), ["ab", "cd"])
def test_split_single_shot(self):
self.tp._single_shot = True
self.tp.min_split_length = 1
self.assertEqual(
list(self.tp.split("print really long line")),
# 'longli' is expected artifact due to edge effects
["print", "really", "long", "li"])
self.assertEqual(list(self.tp.split("a b c d")), ["a", "b", "c", "d"])
self.assertEqual(list(self.tp.split("a b long c d")),
["a", "b", "long", "c", "d"])
self.assertEqual(list(self.tp.split("AbCd")), ["ab", "cd"])
def test_stem(self):
self.assertEqual(self.tp.stem("lol"), "lol")
self.assertEqual(self.tp.stem("apple"), "appl")
self.assertEqual(self.tp.stem("orange"), "orang")
self.assertEqual(self.tp.stem("embedding"), "embed")
self.assertEqual(self.tp.stem("Alfred"), "Alfred")
self.assertEqual(self.tp.stem("Pluto"), "Pluto")
def test_pickle(self):
tp = pickle.loads(pickle.dumps(self.tp))
self.assertEqual(tp.stem("embedding"), "embed")
class FunctionNameAnalyzer(Analyzer):
log = logging.getLogger("FunctionNameAnalyzer")
model_type = FunctionNameModel
version = "1"
description = "Analyzer that suggests function names."
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.token_parser = TokenParser(min_split_length=1, single_shot=True)
@with_changed_uasts_and_contents
def analyze(self, ptr_from: ReferencePointer, ptr_to: ReferencePointer,
data_request_stub: DataStub, **data) -> [Comment]:
comments = []
changes = list(data["changes"])
base_files = files_by_language(c.base for c in changes)
head_files = files_by_language(c.head for c in changes)
for lang, lang_head_files in head_files.items():
if lang.lower() != "java":
continue
self.log.info("Working on %d java files", len(lang_head_files))
for i, (path, file) in enumerate(lang_head_files.items(), start=1):
try:
prev_file = base_files[lang][path]
except KeyError:
lines = None
else:
lines = find_new_lines(prev_file, file)
names_file, tokens_file, line_numbers = self._extract_features(
file, lines)
for prediction, target, score, line_number, type_hint in self.translate(
tokens_file, names_file, line_numbers):
comment = Comment()
comment.line = line_number
comment.file = path
comment.confidence = int(round(score * 100))
comments.append(comment)
if type_hint == FunctionNameAnalyzer.TranslationTypes.LESS_DETAILED:
comment.text = "Consider a more generic name: %s instead of %s" % (
prediction, target)
else:
comment.text = "Consider a more specific name: %s instead of %s" % (
prediction, target)
comments.append(comment)
self.log.info("Processed %d files", i)
return comments
@classmethod
@with_uasts_and_contents
def train(cls, ptr: ReferencePointer, config: Dict[str, Any],
data_request_stub: DataStub, **data) -> FunctionNameModel:
"""
Dummy train.
:param ptr: Git repository state pointer.
:param config: configuration dict.
:param data: contains "files" - the list of files in the pointed state.
:param data_request_stub: connection to the Lookout data retrieval service, not used.
:return: FunctionNameModel dummy model.
"""
return FunctionNameModel().construct(cls, ptr)
def process_node(self, node, last_position):
if IDENTIFIER in node.roles and node.token and FUNCTION not in node.roles:
for x in self.token_parser(node.token):
yield x, last_position
def process_uast(self, uast):
stack = [(uast, [0, 0])]
while stack:
node, last_position = stack.pop()
if node.start_position.line != 0:
# A lot of Nodes do not have position
# It is good heuristic to take the last Node in tree with a position.
last_position[0] = node.start_position.line
last_position[1] = 0
if node.start_position.col != 0:
last_position[1] = node.start_position.col
yield from self.process_node(node, last_position)
stack.extend([(child, list(last_position))
for child in node.children])
def extract_functions_from_uast(self, uast: bblfsh.Node):
for node in uast2sequence(uast):
if node.internal_type != "MethodDeclaration":
continue
for subnode in node.children:
if FUNCTION not in subnode.roles and NAME not in subnode.roles:
continue
name = subnode.token
break
tokens = list(self.process_uast(node))
if len(tokens) < 5:
continue
yield (name, node.start_position.line, node.end_position.line, [
token for token, pos in sorted(tokens, key=lambda x: x[1])
])
def get_affected_functions(self, uast, lines: Optional[Sequence[int]]):
functions_info = list(self.extract_functions_from_uast(uast))
i = 0
res = []
for line in sorted(lines):
while i < len(functions_info):
if functions_info[i][1] <= line <= functions_info[i][2]:
res.append(functions_info[i])
i += 1
break
elif line < functions_info[i][1]:
break
elif line > functions_info[i][2]:
i += 1
return res
@staticmethod
def to_nmt_files(functions_info):
func_start = []
with tempfile.NamedTemporaryFile(delete=False, mode="w") as func_names:
with tempfile.NamedTemporaryFile(delete=False,
mode="w") as func_tokens:
for name, start_line, end_line, tokens in functions_info:
func_names.write(" ".join(list(name)) + "\n")
func_tokens.write(" ".join(tokens) + "\n")
func_start.append(start_line)
return func_names.name, func_tokens.name, func_start
def _extract_features(self, file, lines: Optional[Sequence[int]]):
if file.language.lower() != "java":
raise ValueError("Only java language is supported now")
if lines:
affected_functions = self.get_affected_functions(file.uast, lines)
else:
# all function are affected because the file is new
affected_functions = self.extract_functions_from_uast(file.uast)
return self.to_nmt_files(affected_functions)
class TranslationTypes(Enum):
NOOP = 0
MORE_DETAILED = 1
LESS_DETAILED = 2
CLASS_TO_FUNCTION = 3
FUNCTION_TO_CLASS = 4
CLASS_TO_CLASS = 5
OTHER = 6
def classify_translation(self, prediction, target):
split_prediction = set(self.token_parser.split(prediction))
split_target = set(self.token_parser.split(target))
if prediction[0].isupper():
if target[0].isupper():
return FunctionNameAnalyzer.TranslationTypes.CLASS_TO_CLASS
return FunctionNameAnalyzer.TranslationTypes.FUNCTION_TO_CLASS
elif target[0].isupper():
return FunctionNameAnalyzer.TranslationTypes.CLASS_TO_FUNCTION
if split_prediction == split_target:
return FunctionNameAnalyzer.TranslationTypes.NOOP
elif split_prediction > split_target:
return FunctionNameAnalyzer.TranslationTypes.MORE_DETAILED
elif split_prediction < split_target:
return FunctionNameAnalyzer.TranslationTypes.LESS_DETAILED
return FunctionNameAnalyzer.TranslationTypes.OTHER
def translate(self, source_file, target_file, line_numbers):
model = str(Path(__file__).parent.parent / "models" / "model.pt")
with open(target_file) as fh:
targets = [
"".join(line.strip().split(" ")) for line in fh.readlines()
]
command = "translate.py -model %s -src %s -tgt %s" % (
model, source_file, target_file)
with patch("sys.argv", command.split(" ")):
scores, gold_scores, translations = onmt.infer.main()
for [score], gold_score, [translation], target, line_number \
in zip(scores, gold_scores, translations, targets, line_numbers):
prediction = "".join(translation.split(" "))
gold_score = gold_score / len(target)
pred_score = score / len(prediction)
score = 1 / (1 + math.exp(-pred_score - gold_score))
hint_type = self.classify_translation(prediction, target)
if hint_type in [
FunctionNameAnalyzer.TranslationTypes.MORE_DETAILED,
FunctionNameAnalyzer.TranslationTypes.LESS_DETAILED
]:
yield prediction, target, score, line_number, hint_type
class IdTyposAnalyzer(Analyzer):
"""
Identifier typos analyzer.
"""
log = logging.getLogger("IdTyposAnalyzer")
model_type = None
version = 1
description = "Corrector of typos in source code identifiers."
corrector_manager = TyposCorrectorManager()
DEFAULT_LINE_LENGTH_LIMIT = 500
DEFAULT_N_CANDIDATES = 3
DEFAULT_CONFIDENCE_THRESHOLD = 0.1
INDEX_COLUMN = "index"
def __init__(self, model: AnalyzerModel, url: str, config: Mapping[str,
Any]):
"""
Initialize a new instance of IdTyposAnalyzer.
:param model: The instance of the model loaded from the repository or freshly trained.
:param url: The analyzed project's Git remote.
:param config: Configuration of the analyzer of unspecified structure.
"""
super().__init__(model, url, config)
self.model = self.corrector_manager.get(config.get("model"))
self.n_candidates = config.get("n_candidates",
self.DEFAULT_N_CANDIDATES)
self.confidence_threshold = config.get(
"confidence_threshold", self.DEFAULT_CONFIDENCE_THRESHOLD)
self.parser = TokenParser(stem_threshold=40, single_shot=True)
@with_changed_uasts_and_contents
def analyze(self, ptr_from: ReferencePointer, ptr_to: ReferencePointer,
data_service: DataService, **data) -> [Comment]:
"""
Return the list of `Comment`-s - found typo corrections.
:param ptr_from: The Git revision of the fork point. Exists in both the original and \
the forked repositories.
:param ptr_to: The Git revision to analyze. Exists only in the forked repository.
:param data_service: The channel to the data service in Lookout server to query for \
UASTs, file contents, etc.
:param data: Extra data passed into the method. Used by the decorators to simplify \
the data retrieval.
:return: List of found review suggestions. Refer to \
lookout/core/server/sdk/service_analyzer.proto.
"""
log = self.log
comments = []
changes = list(data["changes"])
base_files_by_lang = files_by_language(c.base for c in changes)
head_files_by_lang = files_by_language(c.head for c in changes)
line_length = self.config.get("line_length_limit",
self.DEFAULT_LINE_LENGTH_LIMIT)
for lang, head_files in head_files_by_lang.items():
for file in filter_files(head_files, line_length, log):
try:
prev_file = base_files_by_lang[lang][file.path]
except KeyError:
lines = []
old_identifiers = set()
else:
lines = find_new_lines(prev_file, file)
old_identifiers = {
node.token
for node in uast2sequence(prev_file.uast)
if bblfsh.role_id("IDENTIFIER") in node.roles
and bblfsh.role_id("IMPORT") not in node.roles
and node.token
}
changed_nodes = extract_changed_nodes(file.uast, lines)
new_identifiers = [
node for node in changed_nodes
if bblfsh.role_id("IDENTIFIER") in node.roles
and bblfsh.role_id("IMPORT") not in node.roles
and node.token and node.token not in old_identifiers
]
if not new_identifiers:
continue
suggestions = self.check_identifiers(
[n.token for n in new_identifiers])
for index in suggestions.keys():
corrections = suggestions[index]
for token in corrections.keys():
comment = Comment()
comment.file = file.path
corrections_line = " " + ", ".join(
"%s (%d%%)" %
(candidate[0], int(candidate[1] * 100))
for candidate in corrections[token])
comment.text = """
Possible typo in \"%s\". Suggestions:
""".strip(
) % new_identifiers[index].token + corrections_line
comment.line = new_identifiers[
index].start_position.line
comment.confidence = int(corrections[token][0][1] *
100)
comments.append(comment)
return comments
@classmethod
@with_uasts_and_contents
def train(cls, ptr: ReferencePointer, config: dict,
data_service: DataService, **data) -> AnalyzerModel:
"""
Generate a new model on top of the specified source code.
:param ptr: Git repository state pointer.
:param config: Configuration of the training of unspecified structure.
:param data_service: The channel to the data service in Lookout server to query for \
UASTs, file contents, etc.
:param data: Extra data passed into the method. Used by the decorators to simplify \
the data retrieval.
:return: Instance of `AnalyzerModel` (`model_type`, to be precise).
"""
return DummyAnalyzerModel()
def check_identifiers(
self,
identifiers: List[str],
) -> Dict[int, Dict[str, List[Tuple[str, float]]]]:
"""
Check tokens from identifiers for typos.
:param identifiers: List of identifiers to check.
:return: Dictionary of corrections grouped by ids of corresponding identifier \
in 'identifiers' and typoed tokens which have correction suggestions.
"""
df = pandas.DataFrame(columns=[self.INDEX_COLUMN, SPLIT_COLUMN])
df[self.INDEX_COLUMN] = range(len(identifiers))
df[SPLIT_COLUMN] = [
" ".join(self.parser.split(i)) for i in identifiers
]
df = flatten_data(df, new_column_name=TYPO_COLUMN)
suggestions = self.model.suggest(df,
n_candidates=self.n_candidates,
return_all=False)
suggestions = self.filter_suggestions(df, suggestions)
grouped_suggestions = defaultdict(dict)
for index, row in df.iterrows():
if index in suggestions.keys():
grouped_suggestions[row[self.INDEX_COLUMN]][
row[TYPO_COLUMN]] = suggestions[index]
return grouped_suggestions
def filter_suggestions(
self,
test_df: pandas.DataFrame,
suggestions: Dict[int, List[Tuple[str, float]]],
) -> Dict[int, List[Tuple[str, float]]]:
"""
Filter suggestions based on the repo specifics and confidence threshold.
:param test_df: DataFrame with info about tested tokens.
:param suggestions: Dictionary of correction suggestions grouped by \
typoed token index in test_df.
:return: Dictionary of filtered suggestions grouped by typoed token index in test_df.
"""
filtered_suggestions = {}
tokens = test_df.typo
for index, candidates in suggestions.items():
filtered_candidates = []
for candidate in candidates:
if candidate[0] == tokens[
index] or candidate[1] < self.confidence_threshold:
break
filtered_candidates.append(candidate)
if filtered_candidates:
filtered_suggestions[index] = filtered_candidates
return filtered_suggestions
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.token_parser = TokenParser(min_split_length=1, single_shot=True)
def setUp(self):
self.tp = TokenParser(stem_threshold=4, max_token_length=20)
class TokenParserTests(unittest.TestCase):
def setUp(self):
self.tp = TokenParser(stem_threshold=4, max_token_length=20)
def test_process_token(self):
_max_token_length = self.tp.max_token_length
self.tp.max_token_length = 100
tokens = [
("sourced.ml.algorithms.uast_ids_to_bag", [
"sourc", "sourcedml", "algorithm", "mlalgorithm", "uast",
"ids", "idsto", "bag", "tobag"
]),
("WORSTnameYOUcanIMAGINE",
['worst', 'name', 'you', 'can', 'imagin']),
# Another bad example. Parser failed to parse it correctly
("SmallIdsToFoOo", ["small", "ids", 'idsto', 'fo', 'oo']),
("SmallIdFooo", ["small", "smallid", 'fooo', 'idfooo']),
("ONE_M0re_.__badId.example", [
'one', 'onem', 're', 'bad', 'rebad', 'badid', 'exampl',
'idexampl'
]),
("never_use_Such__varsableNames",
['never', 'use', 'such', 'varsabl', 'name']),
("a.b.c.d", ["a", "b", "c", "d"]),
("A.b.Cd.E", ['a', 'b', 'cd', 'e']),
("looong_sh_loooong_sh",
['looong', 'looongsh', 'loooong', 'shloooong', 'loooongsh']),
("sh_sh_sh_sh", ['sh', 'sh', 'sh', 'sh']),
("loooong_loooong_loooong", ['loooong', 'loooong', 'loooong'])
]
for token, correct in tokens:
res = list(self.tp.process_token(token))
self.assertEqual(res, correct)
self.tp.max_token_length = _max_token_length
def test_split(self):
self.assertEqual(list(self.tp.split("set for")), ["set", "for"])
self.assertEqual(list(self.tp.split("set /for.")), ["set", "for"])
self.assertEqual(list(self.tp.split("NeverHav")), ["never", "hav"])
self.assertEqual(list(self.tp.split("PrintAll")), ["print", "all"])
self.assertEqual(list(self.tp.split("PrintAllExcept")),
["print", "all", "except"])
self.assertEqual(
list(self.tp.split("print really long line")),
# 'longli' is expected artifact due to edge effects
["print", "really", "long", "longli"])
self.assertEqual(list(self.tp.split("set /for. *&PrintAll")),
["set", "for", "print", "all"])
self.assertEqual(list(self.tp.split("JumpDown not Here")),
["jump", "down", "not", "here"])
def test_stem(self):
self.assertEqual(self.tp.stem("lol"), "lol")
self.assertEqual(self.tp.stem("apple"), "appl")
self.assertEqual(self.tp.stem("orange"), "orang")
self.assertEqual(self.tp.stem("embedding"), "embed")
self.assertEqual(self.tp.stem("Alfred"), "Alfred")
self.assertEqual(self.tp.stem("Pluto"), "Pluto")
def test_pickle(self):
tp = pickle.loads(pickle.dumps(self.tp))
self.assertEqual(tp.stem("embedding"), "embed")
