def get_full_context(example): context = get_context(example) if ADD_OPERATION_NAMES: context += ' ' + get_operations(example) if ADD_OPERATION_DOCSTRINGS: raise NotImplementedError() return ' '.join(tokenizer.tokenize(context))
def score_description(self, description: Text) -> Dict[Text, float]: description_as_context = ' '.join(tokenizer.tokenize(description)) vectorized = self.vectorizer.transform([description_as_context]) probas = {} for op_name, predict_fn in zip(self.all_names, self.predict_fns): proba = predict_fn(vectorized) probas[op_name] = proba return probas
def get_context(example: Dict[Text, List[Text]]) -> Text: """Gets the textual context provided in a single example.""" docstring = example['docstring'][0] comments = example['comments'] names = example['names'] strings = example['strings'] tokens = (tokenizer.tokenize(docstring) + tokenizer.tokens_from_text_list(comments) + tokenizer.tokens_from_text_list(names) + tokenizer.tokens_from_text_list(strings)) return ' '.join(tokens)