예제 #1
0
    if (args.task == 'abs'):
        if (args.mode == 'train'):
            train_abs(args, device_id)
        elif (args.mode == 'validate'):
            validate_abs(args, device_id)
        elif (args.mode == 'lead'):
            baseline(args, cal_lead=True)
        elif (args.mode == 'oracle'):
            baseline(args, cal_oracle=True)
        if (args.mode == 'test'):
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                step = 0
            test_abs(args, device_id, cp, step)
        elif (args.mode == 'test_text'):
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                step = 0
                test_text_abs(args, device_id, cp, step)

    elif (args.task == 'ext'):
        if (args.mode == 'train'):
            train_ext(args, device_id)
        elif (args.mode == 'validate'):
            validate_ext(args, device_id)
        if (args.mode == 'test'):
            cp = args.test_from
예제 #2
0
파일: train.py 프로젝트: ulwan/indolem
        if (args.mode == 'train'):
            train_abs(args, device_id, tokenizer)
        elif (args.mode == 'validate'):
            validate_abs(args, device_id, tokenizer)
        elif (args.mode == 'lead'):
            baseline(args, cal_lead=True)
        elif (args.mode == 'oracle'):
            baseline(args, cal_oracle=True)
        if (args.mode == 'test'):
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                step = 0
            test_abs(args, device_id, cp, step, tokenizer)
        elif (args.mode == 'dev'):
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                step = 0
            dev_abs(args, device_id, cp, step, tokenizer)
        elif (args.mode == 'test_text'):
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                step = 0
                test_text_abs(args, device_id, cp, step, tokenizer)
예제 #3
0
    args = parser.parse_args()
    config_path = args.config_path
    with open(config_path, "r") as f:
        config = yaml.safe_load(f)
    config["train"]["gpu_ranks"] = [
        int(i) for i in range(len(config["train"]["visible_gpus"].split(",")))
    ]
    config["train"]["world_size"] = len(config["train"]["gpu_ranks"])
    os.environ["CUDA_VISIBLE_DEVICES"] = config["train"]["visible_gpus"]

    init_logger(config["train"]["log_file"])
    DEVICE = "cpu" if config["train"]["visible_gpus"] == "-1" else "cuda"
    DEVICE_ID = 0 if DEVICE == "cuda" else -1
    kwargs_train = config["train"]
    kwargs_test = {**kwargs_train, **config["test"]}
    TrainArgs = recordtype("TrainArgs", kwargs_train)
    TestArgs = recordtype("TestArgs", kwargs_test)
    train_args = TrainArgs(**kwargs_train)
    test_args = TestArgs(**kwargs_test)
    if args.mode == "train":
        train_abs(args=train_args, device_id=DEVICE_ID)
    elif args.mode == "test":
        cp = test_args.test_from
        try:
            STEP = int(cp.split(".")[-2].split("_")[-1])
        except ValueError:
            STEP = 0
        test_abs(args=test_args, device_id=DEVICE_ID, pt=cp, step=STEP)
    else:
        raise ValueError("Unknown argument")
예제 #4
0
    if (args.task == 'abs'):
        if (args.mode == 'train'):
            train_abs(args, device_id)
        elif (args.mode == 'validate'):
            validate_abs(args, device_id)
        elif (args.mode == 'lead'):
            baseline(args, cal_lead=True)
        elif (args.mode == 'oracle'):
            baseline(args, cal_oracle=True)
        if (args.mode == 'test'):
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                step = 0
            test_abs(args, device_id, cp, step, calculate_rouge=args.dont_calculate_rouge)
        elif (args.mode == 'test_text'):
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                step = 0
                test_text_abs(args, device_id, cp, step)

    elif (args.task == 'ext'):
        if (args.mode == 'train'):
            train_ext(args, device_id)
        elif (args.mode == 'validate'):
            validate_ext(args, device_id)
        if (args.mode == 'test'):
            cp = args.test_from
예제 #5
0
 def run(args, hpspace):
     if (args.task == 'abs'):
         if (args.mode == 'train'):
             if (args.hyperopt):
                 t = time.time()
                 newT = str(t).split(".")
                 args.model_path = "../models/" + newT[0]
                 args.log_file = "../logs/abs_bert_abs_" + newT[0]
                 args.lr_bert = hpspace['lr_bert']
                 args.lr_dec = hpspace['lr_dec']
                 args.accum_count = int(hpspace['accum_count'])
                 args.beta1 = hpspace['beta1']
                 args.beta2 = hpspace['beta2']
                 args.visible_gpus = '0'
                 args.bert_model = '..temp/bert-base-danish-uncased-v2'
                 args.vocab = '..temp/bert-base-danish-uncased-v2'
             train_stats = train_abs(args, device_id)
             x = train_stats.x
             ppl = train_stats.perplexity
             acc = train_stats.acc
             print(x)
             return {
                 'loss': x,
                 'eval_time': time.time(),
                 'status': STATUS_OK,
                 'other_stuff': {
                     'ppl': ppl,
                     'acc': acc
                 }
             }
         elif (args.mode == 'validate'):
             validate_abs(args, device_id)
         elif (args.mode == 'lead'):
             baseline(args, cal_lead=True)
         elif (args.mode == 'oracle'):
             baseline(args, cal_oracle=True)
         if (args.mode == 'test'):
             cp = args.test_from
             try:
                 step = int(cp.split('.')[-2].split('_')[-1])
             except:
                 step = 0
             test_abs(args, device_id, cp, step)
         elif (args.mode == 'test_text'):
             cp = args.test_from
             try:
                 step = int(cp.split('.')[-2].split('_')[-1])
             except:
                 step = 0
                 test_text_abs(args, device_id, cp, step)
     elif (args.task == 'ext'):
         if (args.mode == 'train'):
             train_ext(args, device_id)
         elif (args.mode == 'validate'):
             validate_ext(args, device_id)
         if (args.mode == 'test'):
             cp = args.test_from
             try:
                 step = int(cp.split('.')[-2].split('_')[-1])
             except:
                 step = 0
             test_ext(args, device_id, cp, step)
         elif (args.mode == 'test_text'):
             cp = args.test_from
             try:
                 step = int(cp.split('.')[-2].split('_')[-1])
             except:
                 step = 0
                 test_text_abs(args, device_id, cp, step)
     if (args.mode == "sent_label"):
         step = 0
         sent_label_ext(args, device_id)
예제 #6
0
    os.makedirs(args.model_path, exist_ok=True)
    os.makedirs(args.log_path, exist_ok=True)
    os.makedirs(args.result_path, exist_ok=True)

    # Train/valid/test
    if args.task == 'abs':
        if args.mode == 'train':
            init_logger(os.path.join(args.log_path, 'train.log'))
            train_abs(args, DEVICE_ID)
        elif args.mode == 'validate':
            init_logger(os.path.join(args.log_path, 'valid.log'))
            validate_abs(args, DEVICE_ID)
        elif args.mode == 'test':
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                print("Not correct model name (EX: model_step_1200.pt)")
            init_logger(
                os.path.join(args.log_path, 'test.' + str(step) + '.log'))
            test_abs(args, DEVICE_ID, cp, step)
        elif args.mode == 'test_text':
            cp = args.test_from
            try:
                step = int(cp.split('.')[-2].split('_')[-1])
            except:
                print("Not correct model name (EX: model_step_1200.pt)")
            init_logger(
                os.path.join(args.log_path, 'test_text.' + str(step) + '.log'))
            test_text_abs(args, DEVICE_ID, cp, step)