def create_and_check_xlnet_token_classif( self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetForTokenClassification(config) model.to(torch_device) model.eval() logits, mems_1 = model(input_ids_1) loss, logits, mems_1 = model(input_ids_1, labels=token_labels) result = { "loss": loss, "mems_1": mems_1, "logits": logits, } self.parent.assertListEqual(list(result["loss"].size()), []) self.parent.assertListEqual(list(result["logits"].size()), [ self.batch_size, self.seq_length, self.type_sequence_label_size ]) self.parent.assertListEqual( list(list(mem.size()) for mem in result["mems_1"]), [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers)
def create_and_check_xlnet_token_classif( self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels, ): model = XLNetForTokenClassification(config) model.to(torch_device) model.eval() result = model(input_ids_1) result = model(input_ids_1, labels=token_labels) self.parent.assertEqual(result.loss.shape, ()) self.parent.assertEqual( result.logits.shape, (self.batch_size, self.seq_length, self.type_sequence_label_size)) self.parent.assertListEqual( [mem.shape for mem in result.mems], [(self.seq_length, self.batch_size, self.hidden_size)] * self.num_hidden_layers, )
for i, j in enumerate(sep_sentence[1:-2]): out[i + pad_num + 1] = max(char_label[current_idx:current_idx + len(j)]) if j == "<unk>": current_idx = current_idx + 1 else: current_idx = current_idx + len(j) return out.tolist() config = AutoConfig.from_pretrained(model_path) tokenizer = XLNetTokenizer.from_pretrained(model_path, unk_token=unk_token) model = XLNetForTokenClassification.from_pretrained(model_path, num_labels=13) if torch.cuda.is_available(): device = torch.device("cuda") print('There are %d GPU(s) available.' % torch.cuda.device_count()) print('We will use the GPU:', torch.cuda.get_device_name(0)) else: print('No GPU available, using the CPU instead.') # device = torch.device("cpu") model.to(device) train_input_ids = [] train_labels = [] train_masks = []
os.makedirs(output_path) args.output_file = os.path.join(output_path, suffix) if args.started_sentence_id==1 and os.path.exists(args.output_file): os.remove(args.output_file) print('The output file is ', args.output_file) args.input_file = os.path.join(args.input_file, f'''{args.dataset}/{args.keywords}keywords.txt''') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print("device:", device) if args.random==0: classifier_model_path = '../checkpoints/xlnet_classifier/{}'.format(args.dataset) args.classifier_model_path = classifier_model_path classifier_model = XLNetForTokenClassification.from_pretrained(classifier_model_path,num_labels=4) classifier_model_tokenizer = XLNetTokenizer.from_pretrained(classifier_model_path) logger.logger.info('Initialize backward XLNetForTokenClassification from checkpoint {}.'.format(classifier_model_path)) classifier_model = classifier_model.to(device) classifier_model.eval() else: classifier_model = None classifier_model_tokenizer = None if args.model_name == 'LSTMLMGenerate': forward_lm_path = '../checkpoints/forward_lstm_lm/{}/best.pt'.format(args.dataset) backward_lm_path = '../checkpoints/backward_lstm_lm/{}/best.pt'.format(args.dataset) args.forward_lm_path = forward_lm_path args.backward_lm_path = backward_lm_path
args.model_path = model_path args.log_path = log_path if not os.path.exists(log_path): os.makedirs(log_path) log_file = '{}/{}.log'.format(log_path, args.dataset) print('The log file is ', log_file) logger = Logger(log_file) logger.logger.info(args) if not os.path.exists(model_path): os.makedirs(model_path) try: # load the pre-trained model and tokenizer tokenizer = XLNetTokenizer.from_pretrained(args.model_path) model = XLNetForTokenClassification.from_pretrained( args.model_path, num_labels=args.num_labels) logger.logger.info('Initialize XLNet from checkpoint {}.'.format( args.model_path)) except: tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') model = XLNetForTokenClassification.from_pretrained( 'xlnet-base-cased', num_labels=args.num_labels) logger.logger.info('Initialize XLNet with default parameters.') model = XLNetClassifier(model) """ copy: 0 replace: 1 insert: 2 delete: 3 """