Пример #1
0
def create_fake_python_semantics_common_error_vocabulary(
        begin_tokens, end_tokens, unk_token, addition_tokens=None):
    vocab = load_vocabulary(
        load_vocabulary_fn=get_fake_python_semantics_train_token_vocabulary_set,
        load_vocabulary_id_dict=
        get_fake_python_semantics_train_token_vocabulary_id_map,
        begin_tokens=begin_tokens,
        end_tokens=end_tokens,
        unk_token=unk_token,
        addition_tokens=addition_tokens)
    return vocab
def create_common_error_vocabulary(begin_tokens,
                                   end_tokens,
                                   unk_token,
                                   addition_tokens=None):
    vocab = load_vocabulary(get_common_error_vocabulary_set,
                            get_common_error_vocabulary_id_map,
                            begin_tokens=begin_tokens,
                            end_tokens=end_tokens,
                            unk_token=unk_token,
                            addition_tokens=addition_tokens)
    return vocab
Пример #3
0
def create_summarization_method_name_vocabulary(begin_tokens, end_tokens, unk_token, pad_token, addition_tokens=None):
    vocab = load_vocabulary(get_summarization_method_name_vocabulary_set, get_summarization_method_name_vocabulary_id_map, begin_tokens=begin_tokens, end_tokens=end_tokens, unk_token=unk_token, pad_token=pad_token, hole_token=None, addition_tokens=addition_tokens)
    return vocab