def create_fake_python_semantics_common_error_vocabulary(
begin_tokens, end_tokens, unk_token, addition_tokens=None):
vocab = load_vocabulary(
load_vocabulary_fn=get_fake_python_semantics_train_token_vocabulary_set,
load_vocabulary_id_dict=
get_fake_python_semantics_train_token_vocabulary_id_map,
begin_tokens=begin_tokens,
end_tokens=end_tokens,
unk_token=unk_token,
addition_tokens=addition_tokens)
return vocab
def create_common_error_vocabulary(begin_tokens,
end_tokens,
unk_token,
addition_tokens=None):
vocab = load_vocabulary(get_common_error_vocabulary_set,
get_common_error_vocabulary_id_map,
begin_tokens=begin_tokens,
end_tokens=end_tokens,
unk_token=unk_token,
addition_tokens=addition_tokens)
return vocab
def create_summarization_method_name_vocabulary(begin_tokens, end_tokens, unk_token, pad_token, addition_tokens=None):
vocab = load_vocabulary(get_summarization_method_name_vocabulary_set, get_summarization_method_name_vocabulary_id_map, begin_tokens=begin_tokens, end_tokens=end_tokens, unk_token=unk_token, pad_token=pad_token, hole_token=None, addition_tokens=addition_tokens)
return vocab