def get_full_context(example):
context = get_context(example)
if ADD_OPERATION_NAMES:
context += ' ' + get_operations(example)
if ADD_OPERATION_DOCSTRINGS:
raise NotImplementedError()
return ' '.join(tokenizer.tokenize(context))
def score_description(self, description: Text) -> Dict[Text, float]:
description_as_context = ' '.join(tokenizer.tokenize(description))
vectorized = self.vectorizer.transform([description_as_context])
probas = {}
for op_name, predict_fn in zip(self.all_names, self.predict_fns):
proba = predict_fn(vectorized)
probas[op_name] = proba
return probas
def get_context(example: Dict[Text, List[Text]]) -> Text:
"""Gets the textual context provided in a single example."""
docstring = example['docstring'][0]
comments = example['comments']
names = example['names']
strings = example['strings']
tokens = (tokenizer.tokenize(docstring) +
tokenizer.tokens_from_text_list(comments) +
tokenizer.tokens_from_text_list(names) +
tokenizer.tokens_from_text_list(strings))
return ' '.join(tokens)