def test(config_args, ckpt_file): args = get_args() # Add the args from the config file args.__dict__.update(config_args) args.split = "dev" args.infer = True args.trained_model = os.path.join(cwd, args.EXPT_DIR, ckpt_file) args.data_dir = os.path.join(syspath, "hedwig-data", "datasets") _, _, predictions, labels = __main__.main(args) return {"predictions": predictions, "labels": labels}
def train(config_args, labeled, resume_from: int = 0, ckpt_file: str = ""): args = get_args() # Add the args from the config file args.__dict__.update(config_args) args.labeled = labeled args.snapshot_path = os.path.join(cwd, args.EXPT_DIR, ckpt_file) args.data_dir = os.path.join(syspath, "hedwig-data", "datasets") args.infer = False if not os.path.isdir(os.path.join(cwd, args.EXPT_DIR)): os.mkdir(os.path.join(cwd, args.EXPT_DIR)) __main__.main(args) return
def infer(config_args, unlabeled, ckpt_file): args = get_args() # Add the args from the config file args.__dict__.update(config_args) args.split = "train" args.infer = True args.trained_model = os.path.join(cwd, args.EXPT_DIR, ckpt_file) args.data_dir = os.path.join(syspath, "hedwig-data", "datasets") scores = __main__.main(args) d = dict(zip(range(args.trainsize), scores)) outputs = {} for l in unlabeled: outputs[l] = {"softmax": d[l]} return {"outputs": outputs}
def evaluate_split(model, processor, args, split='dev'): evaluator = BertEvaluator(model, processor, args, split) start_time = time.time() accuracy, precision, recall, f1, avg_loss = evaluator.get_scores( silent=True)[0] print("Inference time", time.time() - start_time) print('\n' + LOG_HEADER) print( LOG_TEMPLATE.format(split.upper(), accuracy, precision, recall, f1, avg_loss)) if __name__ == '__main__': # Set default configuration in args.py args = get_args() if args.local_rank == -1 or not args.cuda: device = torch.device( "cuda" if torch.cuda.is_available() and args.cuda else "cpu") n_gpu = torch.cuda.device_count() else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') print('Device:', str(device).upper()) print('Number of GPUs:', n_gpu) print('Distributed training:', bool(args.local_rank != -1))
def do_main(): # Set default configuration in args.py args = get_args() if args.local_rank == -1 or not args.cuda: device = torch.device( "cuda" if torch.cuda.is_available() and args.cuda else "cpu") n_gpu = torch.cuda.device_count() torch.cuda.set_device(args.gpu) else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') print('Device:', str(device).upper()) print('Number of GPUs:', n_gpu) print('Distributed training:', bool(args.local_rank != -1)) print('FP16:', args.fp16) # Set random seed for reproducibility random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) dataset_map = {'News_art': News_artProcessor, 'News': News_Processor} if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') args.batch_size = args.batch_size // args.gradient_accumulation_steps args.device = device args.n_gpu = n_gpu args.num_labels = dataset_map[args.dataset].NUM_CLASSES args.is_multilabel = dataset_map[args.dataset].IS_MULTILABEL if not args.trained_model: save_path = os.path.join(args.save_path, dataset_map[args.dataset].NAME) os.makedirs(save_path, exist_ok=True) processor = dataset_map[args.dataset]() args.is_lowercase = 'uncased' in args.model args.is_hierarchical = False tokenizer = BertTokenizer.from_pretrained(args.model, is_lowercase=args.is_lowercase) num_train_optimization_steps = None if args.trained_model: train_examples = processor.get_train_examples(args.data_dir, args.train_name) num_train_optimization_steps = int( math.ceil(len(train_examples) / args.batch_size) / args.gradient_accumulation_steps) * args.epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size( ) model_bert = BertForSequenceClassification.from_pretrained(args.model, num_labels=4) model_bert.to(device) if args.trained_model: model_ = torch.load(args.trained_model, map_location=lambda storage, loc: storage ) # load personality model state = {} for key in model_.state_dict().keys(): new_key = key.replace("module.", "") state[new_key] = model_.state_dict()[key] del state['classifier.weight'] # removing personality classifier! del state['classifier.bias'] model_bert.load_state_dict(state, strict=False) model_bert = model_bert.to(device) args.freez_bert = False evaluate(model_bert, processor, args, last_bert_layers=-1, ngram_range=(1, 1))
def do_main(): # Set default configuration in args.py args = get_args() if args.local_rank == -1 or not args.cuda: device = torch.device( "cuda" if torch.cuda.is_available() and args.cuda else "cpu") n_gpu = torch.cuda.device_count() torch.cuda.set_device(args.gpu) else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') print('Device:', str(device).upper()) print('Number of GPUs:', n_gpu) print('Distributed training:', bool(args.local_rank != -1)) print('FP16:', args.fp16) # Set random seed for reproducibility random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) dataset_map = { 'SST-2': SST2Processor, 'Reuters': ReutersProcessor, 'IMDB': IMDBProcessor, 'AAPD': AAPDProcessor, 'AGNews': AGNewsProcessor, 'Yelp2014': Yelp2014Processor, 'Sogou': SogouProcessor, 'Personality': PersonalityProcessor, 'News_art': News_artProcessor, 'News': News_Processor, 'UCI_yelp': UCI_yelpProcessor, 'Procon': ProconProcessor, 'Style': StyleProcessor, 'ProconDual': ProconDualProcessor, 'Pan15': Pan15_Processor, 'Pan14E': Pan14E_Processor, 'Pan14N': Pan14N_Processor, 'Perspectrum': PerspectrumProcessor } if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') args.batch_size = args.batch_size // args.gradient_accumulation_steps args.device = device args.n_gpu = n_gpu args.num_labels = dataset_map[args.dataset].NUM_CLASSES args.is_multilabel = dataset_map[args.dataset].IS_MULTILABEL if not args.trained_model: save_path = os.path.join(args.save_path, dataset_map[args.dataset].NAME) os.makedirs(save_path, exist_ok=True) processor = dataset_map[args.dataset]() args.is_lowercase = 'uncased' in args.model args.is_hierarchical = False tokenizer = BertTokenizer.from_pretrained(args.model, is_lowercase=args.is_lowercase) train_examples = None num_train_optimization_steps = None if not args.trained_model: train_examples = processor.get_train_examples(args.data_dir, args.train_name) num_train_optimization_steps = int( len(train_examples) / args.batch_size / args.gradient_accumulation_steps) * args.epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size( ) cache_dir = args.cache_dir if args.cache_dir else os.path.join( str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format( args.local_rank)) model = BertForSequenceClassification.from_pretrained( args.model, cache_dir=cache_dir, num_labels=args.num_labels) if args.fp16: model.half() model.to(device) if args.local_rank != -1: try: from apex.parallel import DistributedDataParallel as DDP except ImportError: raise ImportError( "Install NVIDIA Apex to use distributed and FP16 training.") model = DDP(model) '''elif n_gpu > 1: changed by marjan model = torch.nn.DataParallel(model)''' # Prepare optimizer param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] if args.fp16: try: from apex.optimizers import FP16_Optimizer from apex.optimizers import FusedAdam except ImportError: raise ImportError( "Please install NVIDIA Apex for distributed and FP16 training") optimizer = FusedAdam(optimizer_grouped_parameters, lr=args.lr, bias_correction=False, max_grad_norm=1.0) if args.loss_scale == 0: optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) else: optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale) else: optimizer = BertAdam(optimizer_grouped_parameters, lr=args.lr, warmup=args.warmup_proportion, t_total=num_train_optimization_steps) trainer = BertTrainer(model, optimizer, processor, args) if not args.trained_model: trainer.train() model = torch.load(trainer.snapshot_path) else: model = BertForSequenceClassification.from_pretrained( args.model, num_labels=args.num_labels) model_ = torch.load(args.trained_model, map_location=lambda storage, loc: storage) state = {} for key in model_.state_dict().keys(): new_key = key.replace("module.", "") state[new_key] = model_.state_dict()[key] model.load_state_dict(state) model = model.to(device) evaluate_split(model, processor, args, split=args.dev_name) evaluate_split(model, processor, args, split=args.test_name)
def do_main(): # Set default configuration in args.py args = get_args() if args.local_rank == -1 or not args.cuda: device = torch.device( "cuda" if torch.cuda.is_available() and args.cuda else "cpu") n_gpu = torch.cuda.device_count() torch.cuda.set_device(args.gpu) else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') print('Device:', str(device).upper()) print('Number of GPUs:', n_gpu) print('Distributed training:', bool(args.local_rank != -1)) print('FP16:', args.fp16) # Set random seed for reproducibility random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) dataset_map = {'News_art': News_artProcessor, 'News': News_Processor} if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') args.batch_size = args.batch_size // args.gradient_accumulation_steps args.device = device args.n_gpu = n_gpu args.num_labels = dataset_map[args.dataset].NUM_CLASSES args.is_multilabel = dataset_map[args.dataset].IS_MULTILABEL if not args.trained_model: save_path = os.path.join(args.save_path, dataset_map[args.dataset].NAME) os.makedirs(save_path, exist_ok=True) processor = dataset_map[args.dataset]() args.is_lowercase = 'uncased' in args.model args.is_hierarchical = False tokenizer = BertTokenizer.from_pretrained(args.model, is_lowercase=args.is_lowercase) train_examples = None num_train_optimization_steps = None if args.trained_model: train_examples = processor.get_train_examples(args.data_dir, args.train_name) num_train_optimization_steps = int( math.ceil(len(train_examples) / args.batch_size) / args.gradient_accumulation_steps) * args.epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size( ) cache_dir = args.cache_dir if args.cache_dir else os.path.join( str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format( args.local_rank)) model = BertForSequenceClassification.from_pretrained( args.model, num_labels=2) # creating news model! #model = BertForSequenceClassification.from_pretrained(args.model, cache_dir=cache_dir, num_labels=args.num_labels) if args.fp16: model.half() model.to(device) #model = BertForSequenceClassification.from_pretrained(args.model, num_labels=args.num_labels) model_ = torch.load( args.trained_model, map_location=lambda storage, loc: storage) # load personality model state = {} for key in model_.state_dict().keys(): new_key = key.replace("module.", "") state[new_key] = model_.state_dict()[key] del state['classifier.weight'] # removing personality classifier! del state['classifier.bias'] model.load_state_dict(state, strict=False) model = model.to(device) # Prepare optimizer param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] print('t_total :', num_train_optimization_steps) optimizer = BertAdam(optimizer_grouped_parameters, lr=args.lr, warmup=args.warmup_proportion, t_total=num_train_optimization_steps) args.freez_bert = False trainer = BertTrainer(model, optimizer, processor, args) trainer.train() model = torch.load(trainer.snapshot_path) evaluate_split(model, processor, args, split=args.dev_name) evaluate_split(model, processor, args, split=args.test_name)
def do_main(): # Set default configuration in args.py args = get_args() if args.local_rank == -1 or not args.cuda: device = torch.device( "cuda" if torch.cuda.is_available() and args.cuda else "cpu") n_gpu = torch.cuda.device_count() torch.cuda.set_device(args.gpu) else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') print('Device:', str(device).upper()) print('Number of GPUs:', n_gpu) print('Distributed training:', bool(args.local_rank != -1)) print('FP16:', args.fp16) # Set random seed for reproducibility random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) dataset_map = {'Personality': PersonalityProcessor} if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') args.batch_size = args.batch_size // args.gradient_accumulation_steps args.device = device args.n_gpu = n_gpu args.num_labels = dataset_map[args.dataset].NUM_CLASSES args.is_multilabel = dataset_map[args.dataset].IS_MULTILABEL if not args.trained_model: raise Exception('This method only works wit pre-trained models!') processor = dataset_map[args.dataset]() args.is_lowercase = 'uncased' in args.model args.is_hierarchical = False if args.trained_model: model = BertForSequenceClassification.from_pretrained( args.model, num_labels=args.num_labels) model_ = torch.load(args.trained_model, map_location=lambda storage, loc: storage) state = {} for key in model_.state_dict().keys(): new_key = key.replace("module.", "") state[new_key] = model_.state_dict()[key] model.load_state_dict(state) model = model.to(device) #evaluate_split(model, processor, args, split='dev') #evaluate_split(model, processor, args, split='test') evaluate_split(model, processor, args, split=args.analyze_split)