def __init__(self, opt): super().__init__(opt) # initialize from vocab path warn_once( 'WARNING: BERT uses a Hugging Face tokenizer; ParlAI dictionary args are ignored' ) download(opt['datapath']) vocab_path = os.path.join(opt['datapath'], 'models', 'bert_models', VOCAB_PATH) self.tokenizer = BertTokenizer.from_pretrained(vocab_path) self.start_token = '[CLS]' self.end_token = '[SEP]' self.null_token = '[PAD]' self.start_idx = self.tokenizer.convert_tokens_to_ids( ['[CLS]'])[0] # should be 101 self.end_idx = self.tokenizer.convert_tokens_to_ids( ['[SEP]'])[0] # should be 102 self.pad_idx = self.tokenizer.convert_tokens_to_ids( ['[PAD]'])[0] # should be 0 # set tok2ind for special tokens self.tok2ind[self.start_token] = self.start_idx self.tok2ind[self.end_token] = self.end_idx self.tok2ind[self.null_token] = self.pad_idx # set ind2tok for special tokens self.ind2tok[self.start_idx] = self.start_token self.ind2tok[self.end_idx] = self.end_token self.ind2tok[self.pad_idx] = self.null_token
def __init__(self, opt, shared=None): # download pretrained models download(opt['datapath']) self.pretrained_path = os.path.join(opt['datapath'], 'models', 'bert_models', MODEL_PATH) opt['pretrained_path'] = self.pretrained_path super().__init__(opt, shared)
def __init__(self, opt): super().__init__(opt) # initialize from vocab path download(opt['datapath']) vocab_path = os.path.join(opt['datapath'], 'models', 'bert_models', VOCAB_PATH) self.tokenizer = BertTokenizer.from_pretrained(vocab_path) self.start_token = '[CLS]' self.end_token = '[SEP]' self.null_token = '[PAD]' self.start_idx = self.tokenizer.convert_tokens_to_ids(['[CLS]'])[ 0 ] # should be 101 self.end_idx = self.tokenizer.convert_tokens_to_ids(['[SEP]'])[ 0 ] # should be 102 self.pad_idx = self.tokenizer.convert_tokens_to_ids(['[PAD]'])[0] # should be 0 # set tok2ind for special tokens self.tok2ind[self.start_token] = self.start_idx self.tok2ind[self.end_token] = self.end_idx self.tok2ind[self.null_token] = self.pad_idx # set ind2tok for special tokens self.ind2tok[self.start_idx] = self.start_token self.ind2tok[self.end_idx] = self.end_token self.ind2tok[self.pad_idx] = self.null_token
def __init__(self, opt): super().__init__(opt) # initialize from vocab path warn_once( "WARNING: BERT uses a Hugging Face tokenizer; ParlAI dictionary args are ignored" ) download(opt["datapath"]) vocab_path = PathManager.get_local_path( os.path.join(opt["datapath"], "models", "bert_models", VOCAB_PATH)) self.tokenizer = BertTokenizer.from_pretrained(vocab_path) self.start_token = "[CLS]" self.end_token = "[SEP]" self.null_token = "[PAD]" self.start_idx = self.tokenizer.convert_tokens_to_ids( ["[CLS]"])[0] # should be 101 self.end_idx = self.tokenizer.convert_tokens_to_ids( ["[SEP]"])[0] # should be 102 self.pad_idx = self.tokenizer.convert_tokens_to_ids( ["[PAD]"])[0] # should be 0 # set tok2ind for special tokens self.tok2ind[self.start_token] = self.start_idx self.tok2ind[self.end_token] = self.end_idx self.tok2ind[self.null_token] = self.pad_idx # set ind2tok for special tokens self.ind2tok[self.start_idx] = self.start_token self.ind2tok[self.end_idx] = self.end_token self.ind2tok[self.pad_idx] = self.null_token
def __init__(self, opt, shared=None): # download pretrained models download(opt['datapath']) self.pretrained_path = os.path.join(opt['datapath'], 'models', 'bert_models', MODEL_PATH) opt['pretrained_path'] = self.pretrained_path self.add_cls_token = opt.get('add_cls_token', True) self.sep_last_utt = opt.get('sep_last_utt', False) super().__init__(opt, shared)
def __init__(self, opt, shared=None): # download pretrained models download(opt["datapath"]) self.pretrained_path = PathManager.get_local_path( os.path.join(opt["datapath"], "models", "bert_models", MODEL_PATH)) opt["pretrained_path"] = self.pretrained_path self.add_cls_token = opt.get("add_cls_token", True) self.sep_last_utt = opt.get("sep_last_utt", False) super().__init__(opt, shared)
def __init__(self, opt, shared=None): # download pretrained models download(opt['datapath']) self.pretrained_path = os.path.join(opt['datapath'], 'models', 'bert_models', MODEL_PATH) super().__init__(opt, shared) # it's easier for now to use DataParallel when self.clip = -1 self.NULL_IDX = self.dict.pad_idx self.START_IDX = self.dict.start_idx self.END_IDX = self.dict.end_idx
def __init__(self, opt, shared=None): # download pretrained models download(opt['datapath']) self.pretrained_path = os.path.join(opt['datapath'], 'models', 'bert_models', MODEL_PATH) opt['pretrained_path'] = self.pretrained_path self.clip = -1 super().__init__(opt, shared) # it's easier for now to use DataParallel when self.NULL_IDX = self.dict.pad_idx self.START_IDX = self.dict.start_idx self.END_IDX = self.dict.end_idx # default one does not average self.rank_loss = torch.nn.CrossEntropyLoss(reduce=True, size_average=True)
def __init__(self, opt, shared=None): # download pretrained models download(opt['datapath']) self.pretrained_path = os.path.join(opt['datapath'], 'models', 'bert_models', MODEL_PATH) super().__init__(opt, shared) # it's easier for now to use DataParallel when self.data_parallel = opt.get('data_parallel') and self.use_cuda if self.data_parallel: self.model = torch.nn.DataParallel(self.model) if is_distributed(): raise ValueError( 'Cannot combine --data-parallel and distributed mode') self.clip = -1 self.NULL_IDX = self.dict.pad_idx self.START_IDX = self.dict.start_idx self.END_IDX = self.dict.end_idx
def __init__(self, opt, shared=None): # download pretrained models download(opt['datapath']) self.pretrained_path = os.path.join( opt['datapath'], 'models', 'bert_models', MODEL_PATH ) opt['pretrained_path'] = self.pretrained_path self.clip = -1 super().__init__(opt, shared) # it's easier for now to use DataParallel when self.data_parallel = opt.get('data_parallel') and self.use_cuda if self.data_parallel and shared is None: self.model = torch.nn.DataParallel(self.model) if is_distributed(): raise ValueError('Cannot combine --data-parallel and distributed mode') self.NULL_IDX = self.dict.pad_idx self.START_IDX = self.dict.start_idx self.END_IDX = self.dict.end_idx # default one does not average self.rank_loss = torch.nn.CrossEntropyLoss(reduce=True, size_average=True)