def __init__(self, args): args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count() self.args = args margs = torch.load(os.path.join(args.model_name_or_path, "training_args.bin")) margs.no_cuda = args.no_cuda margs.model_name_or_path = args.model_name_or_path margs.overwrite_output_dir = True self.lm = TransLanguageModel(margs) self.lm.model_init() # lm.load_model(args.model_name_or_path) Data2tensor.set_randseed(args.seed)
def __init__(self, args): args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count() margs = torch.load( os.path.join(args.model_name_or_path, "training_args.bin")) margs.no_cuda = args.no_cuda margs.label_file = args.label_file margs.model_name_or_path = args.model_name_or_path self.tagger = TransLabelerModel(margs) self.tagger.model_init(args.model_name_or_path) Data2tensor.set_randseed(args.seed)
def __init__(self, args): args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count() self.args = args margs = torch.load(os.path.join(args.model_name_or_path, "training_args.bin")) margs.no_cuda = args.no_cuda margs.model_name_or_path = args.model_name_or_path margs.overwrite_output_dir = True self.lm = TransSeq2SeqModel(margs) self.lm.model_init(args.model_name_or_path) # lm.load_model(args.model_name_or_path) Data2tensor.set_randseed(args.seed) self.bos_token_id = self.lm.tokenizer.tw2i[SOT] # self.pad_token_id = self.lm.pad_id self.eos_token_id = self.lm.tokenizer.tw2i[EOT]
@staticmethod def decode_batch(pad_ids, i2t, level=2): return Tokenizer.idx2text(pad_ids=pad_ids, i2t=i2t, level=level) if __name__ == '__main__': import torch from mlmodels.utils.idx2tensor import Data2tensor, seqPAD from mlmodels.utils.dataset import IterDataset, collate_fn, tokens2ids from torch.utils.data import DataLoader, Dataset, IterableDataset, RandomSampler, SequentialSampler, TensorDataset from mlmodels.utils.BPEtonkenizer import BPE from mlmodels.utils.special_tokens import BPAD, PAD, NULL from mlmodels.utils.txtIO import TXT Data2tensor.set_randseed(12345) device = torch.device("cpu") dtype = torch.long use_cuda = False filename = "../../data/reviews/processed_csv/train_res4.csv" label_file = "../../data/reviews/processed_csv/labels.txt" labels_list = TXT.read(label_file, firstline=False) lb2id_dict = Tokenizer.list2dict(sys_tokens + labels_list) id2lb_dict = Tokenizer.reversed_dict(lb2id_dict) lb2ids = Tokenizer.lst2idx(tokenizer=Tokenizer.process_target, vocab_words=lb2id_dict, unk_words=False, sos=False, eos=False) tokenize_type = "bpe" if tokenize_type != "bpe":
logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN, ) logger.warning( "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16, ) # Set seed Data2tensor.set_randseed(args.seed, args.n_gpu) # Prepare CONLL-2003 task labels = TextDataset.get_labels(args.labels) num_labels = len(labels) # Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later pad_token_label_id = CrossEntropyLoss().ignore_index # Load pretrained model and tokenizer if args.local_rank not in [-1, 0]: torch.distributed.barrier( ) # Make sure only the first process in distributed training will download model & vocab args.model_type = args.model_type.lower() config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] config = config_class.from_pretrained(