import os
import string
import re
import ast
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torchvision import transforms
from entity.data import NERProcessor
from pytorch_pretrained_bert.tokenization import BertTokenizer
from utils import com_utils
from entity.data import DataSet, NELToTensor
# init params
comConfig = config.ComConfig()
nerConfig = config.NERConfig()
nelConfig = config.NELConfig()
fileConfig = config.FileConfig()
fasttextConfig = config.FastTextConfig()
def init_params():
processors = {nerConfig.ner_task_name: NERProcessor}
task_name = nerConfig.ner_task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
tokenizer = BertTokenizer(vocab_file=fileConfig.dir_bert +
fileConfig.file_bert_vocab)
return processor, tokenizer
def create_ner_batch_iter(mode):
def load_file_config(config_file=None):
global FILE_CONFIG, CONFIG_FILE, CONFIG
CONFIG = load_config(config_file)
if FILE_CONFIG is None:
FILE_CONFIG = config.FileConfig(CONFIG)
return FILE_CONFIG