def pick_model(args, dicts):
"""
Use args to initialize the appropriate model
"""
Y = get_num_labels(args.Y, args.version)
if args.model == "rnn":
model = models.VanillaRNN(Y, args.embed_file, dicts, args.rnn_dim,
args.cell_type, args.rnn_layers, args.gpu,
args.embed_size, args.bidirectional)
elif args.model == "cnn_vanilla":
filter_size = int(args.filter_size)
model = models.VanillaConv(Y, args.embed_file, filter_size,
args.num_filter_maps, args.gpu, dicts,
args.embed_size, args.dropout)
elif args.model == "conv_attn":
filter_size = int(args.filter_size)
model = models.ConvAttnPool(Y,
args.embed_file,
filter_size,
args.num_filter_maps,
args.lmbda,
args.gpu,
dicts,
embed_size=args.embed_size,
dropout=args.dropout)
elif args.model == "saved":
model = torch.load(args.test_model)
if args.gpu:
model.cuda()
return model
def pick_model(args, dicts):
"""
Use args to initialize the appropriate model
"""
filter_size = int(args["filter_size"])
args["embed_file"] = None
model = models.ConvAttnPool(8921,
args["embed_file"],
filter_size,
args["num_filter_maps"],
0,
args["gpu"],
dicts,
dropout=args["dropout"])
if args["test_model"]:
sd = torch.load(args["test_model"])
model.load_state_dict(sd)
if args["gpu"]:
model.cuda()
return model
def pick_model(args, dicts):
"""
Use args to initialize the appropriate model
"""
Y = len(dicts['ind2c'])
if args.model == "rnn":
model = models.VanillaRNN(Y, args.embed_file, dicts, args.rnn_dim, args.cell_type, args.rnn_layers, args.gpu, args.embed_size,
args.bidirectional)
elif args.model == "cnn_vanilla":
filter_size = int(args.filter_size)
model = models.VanillaConv(Y, args.embed_file, filter_size, args.num_filter_maps, args.gpu, dicts, args.embed_size, args.dropout)
elif args.model == "conv_attn":
filter_size = int(args.filter_size)
model = models.ConvAttnPool(Y, args.embed_file, filter_size, args.num_filter_maps, args.lmbda, args.gpu, dicts,
embed_size=args.embed_size, dropout=args.dropout, code_emb=args.code_emb)
elif args.model == "logreg":
model = models.BOWPool(Y, args.embed_file, args.lmbda, args.gpu, dicts, args.pool, args.embed_size, args.dropout, args.code_emb)
elif args.model == 'bert':
config = BertConfig.from_pretrained('./pretrained_weights/bert-base-uncased-config.json')
if args.Y == 'full':
config.Y = 8921
else:
config.Y = int(args.Y)
config.gpu = args.gpu
if args.redefined_tokenizer:
bert_tokenizer = BertTokenizer.from_pretrained(args.tokenizer_path, do_lower_case=True)
else:
bert_tokenizer = BertTokenizer.from_pretrained('./pretrained_weights/bert-base-uncased-vocab.txt', do_lower_case=True)
config.redefined_vocab_size = len(bert_tokenizer)
if args.max_sequence_length is None:
config.redefined_max_position_embeddings = MAX_LENGTH
else:
config.redefined_max_position_embeddings = args.max_sequence_length
config.last_module = args.last_module
config.model = args.model
if args.from_scratch and not args.pretrain:
model = BertForMedical(config=config)
elif args.pretrain:
model = BertForMedical.from_pretrained(args.pretrain_ckpt_dir)
else:
model = BertForMedical.from_pretrained('./pretrained_weights/bert-base-uncased-pytorch_model.bin', config=config)
elif args.model == 'biobert':
config = BertConfig.from_pretrained('./pretrained_weights/biobert_pretrain_output_all_notes_150000/bert_config.json')
if args.Y == 'full':
config.Y = 8921
else:
config.Y = int(args.Y)
config.gpu = args.gpu
if args.redefined_tokenizer:
bert_tokenizer = BertTokenizer.from_pretrained(args.tokenizer_path, do_lower_case=False)
else:
bert_tokenizer = BertTokenizer.from_pretrained('./pretrained_weights/biobert_pretrain_output_all_notes_150000/vocab.txt', do_lower_case=False)
config.redefined_vocab_size = len(bert_tokenizer)
if args.max_sequence_length is None:
config.redefined_max_position_embeddings = MAX_LENGTH
else:
config.redefined_max_position_embeddings = args.max_sequence_length
config.last_module = args.last_module
config.model = args.model
if args.from_scratch and not args.pretrain:
model = BertForMedical(config=config)
elif args.pretrain:
model = BertForMedical.from_pretrained(args.pretrain_ckpt_dir)
else:
model = BertForMedical.from_pretrained('./pretrained_weights/biobert_pretrain_output_all_notes_150000/pytorch_model.bin', config=config)
elif args.model == 'bert-tiny':
config = BertConfig.from_pretrained('./pretrained_weights/bert-tiny-uncased-config.json')
if args.Y == 'full':
config.Y = 8921
else:
config.Y = int(args.Y)
config.gpu = args.gpu
if args.redefined_tokenizer:
bert_tokenizer = BertTokenizer.from_pretrained(args.tokenizer_path, do_lower_case=True)
else:
bert_tokenizer = BertTokenizer.from_pretrained('./pretrained_weights/bert-tiny-uncased-vocab.txt', do_lower_case=True)
config.redefined_vocab_size = len(bert_tokenizer)
if args.max_sequence_length is None:
config.redefined_max_position_embeddings = MAX_LENGTH
else:
config.redefined_max_position_embeddings = args.max_sequence_length
config.last_module = args.last_module
config.model = args.model
if args.from_scratch and not args.pretrain:
model = BertForMedical(config=config)
elif args.pretrain:
model = BertForMedical.from_pretrained(args.pretrain_ckpt_dir)
else:
model = BertForMedical.from_pretrained('./pretrained_weights/bert-tiny-uncased-pytorch_model.bin', config=config)
elif args.model == 'bert-caml':
config = BertConfig.from_pretrained('./pretrained_weights/bert-base-uncased-config.json')
if args.Y == 'full':
config.Y = 8921
else:
config.Y = int(args.Y)
config.gpu = args.gpu
if args.redefined_tokenizer:
bert_tokenizer = BertTokenizer.from_pretrained(args.tokenizer_path, do_lower_case=True)
else:
bert_tokenizer = BertTokenizer.from_pretrained('./pretrained_weights/bert-base-uncased-vocab.txt', do_lower_case=True)
config.redefined_vocab_size = len(bert_tokenizer)
if args.max_sequence_length is None:
config.redefined_max_position_embeddings = MAX_LENGTH
else:
config.redefined_max_position_embeddings = args.max_sequence_length
config.last_module = args.last_module
config.embed_size = args.embed_size
config.embed_file = args.embed_file
config.dicts = dicts
config.model = args.model
if args.from_scratch:
model = BertWithCAMLForMedical(config=config)
else:
model = BertWithCAMLForMedical.from_pretrained('./pretrained_weights/bert-base-uncased-pytorch_model.bin', config=config)
elif args.model == 'bert-tiny-caml':
if args.from_prev_result:
config = BertConfig.from_pretrained(args.from_prev_result + '/config.json')
else:
config = BertConfig.from_pretrained('./pretrained_weights/bert-tiny-uncased-config.json')
if args.Y == 'full':
config.Y = 8921
else:
config.Y = int(args.Y)
config.gpu = args.gpu
if args.redefined_tokenizer:
bert_tokenizer = BertTokenizer.from_pretrained(args.tokenizer_path, do_lower_case=True)
else:
bert_tokenizer = BertTokenizer.from_pretrained('./pretrained_weights/bert-tiny-uncased-vocab.txt', do_lower_case=True)
config.redefined_vocab_size = len(bert_tokenizer)
if args.max_sequence_length is None:
config.redefined_max_position_embeddings = MAX_LENGTH
else:
config.redefined_max_position_embeddings = args.max_sequence_length
config.last_module = args.last_module
config.embed_size = args.embed_size
config.embed_file = args.embed_file
config.dicts = dicts
config.model = args.model
if args.from_scratch:
model = BertWithCAMLForMedical(config=config)
elif args.from_prev_result:
model = BertWithCAMLForMedical.from_pretrained(args.from_prev_result + '/pytorch_model.bin', config=config)
else:
model = BertWithCAMLForMedical.from_pretrained('./pretrained_weights/bert-tiny-uncased-pytorch_model.bin', config=config)
elif args.model == 'bert-tiny-parallel-caml':
config = BertConfig.from_pretrained('./pretrained_weights/bert-tiny-uncased-config.json')
if args.Y == 'full':
config.Y = 8921
else:
config.Y = int(args.Y)
config.gpu = args.gpu
if args.redefined_tokenizer:
bert_tokenizer = BertTokenizer.from_pretrained(args.tokenizer_path, do_lower_case=True)
else:
bert_tokenizer = BertTokenizer.from_pretrained('./pretrained_weights/bert-tiny-uncased-vocab.txt', do_lower_case=True)
config.redefined_vocab_size = len(bert_tokenizer)
if args.max_sequence_length is None:
config.redefined_max_position_embeddings = MAX_LENGTH
else:
config.redefined_max_position_embeddings = args.max_sequence_length
config.last_module = args.last_module
config.embed_size = args.embed_size
config.embed_file = args.embed_file
config.dicts = dicts
if args.bert_parallel_count:
config.bert_parallel_count = args.bert_parallel_count
else:
config.bert_parallel_count = 1
config.bert_parallel_final_layer = args.bert_parallel_final_layer
config.model = args.model
if args.from_scratch:
model = BertTinyParallelWithCAMLForMedical(config=config)
else:
model = BertTinyParallelWithCAMLForMedical.from_pretrained('./pretrained_weights/bert-tiny-uncased-pytorch_model.bin', config=config)
if args.test_model:
sd = torch.load(args.test_model)
model.load_state_dict(sd)
if args.gpu:
model.cuda()
return model
def init_model(args, dicts):
"""
Use args to initialize the appropriate model
"""
assert not (args.test_model is not None and args.resume is not None)
Y = len(dicts['ind2c'])
Y_coarse = len(dicts['ind2c_coarse']) if args.hier else None
if args.embed_file and not (args.test_model or args.resume):
print("loading pretrained embeddings (freeze={0}, normalize={1})...".
format(args.embed_freeze, args.embed_normalize))
word_embeddings_matrix = load_embeddings(args.embed_file,
dicts['ind2w'], args.dims[0],
args.embed_normalize)
else:
word_embeddings_matrix = None
vocab_size = len(dicts['ind2w'])
if args.model == "dummy":
model = models.DummyModel(Y, dicts, args.gpu)
elif args.model == "conv_dilated":
model = models.ConvDilated(Y,
args.dims,
args.filter_size,
args.dilation,
word_embeddings_matrix,
args.gpu,
vocab_size,
embed_freeze=args.embed_freeze,
dropout=args.dropout,
hier=args.hier,
Y_coarse=Y_coarse,
fine2coarse=dicts['fine2coarse'],
embed_desc=args.embed_desc)
elif args.model == "conv_attn":
model = models.ConvAttnPool(Y,
args.dims,
args.filter_size,
word_embeddings_matrix,
args.gpu,
vocab_size,
embed_freeze=args.embed_freeze,
dropout=args.dropout,
hier=args.hier,
Y_coarse=Y_coarse,
fine2coarse=dicts['fine2coarse'],
embed_desc=args.embed_desc,
layer_norm=args.layer_norm)
if args.test_model:
sd = torch.load(os.path.abspath(args.test_model))
model.load_state_dict(sd)
if args.resume:
sd = torch.load(os.path.abspath(args.resume))
model.load_state_dict(sd)
if args.gpu:
model.cuda()
if not args.test_model and not args.model == 'dummy':
optimizer = optim.Adam(model.parameters(),
weight_decay=args.weight_decay,
lr=args.lr)
if args.resume:
model_dir = os.path.dirname(os.path.abspath(args.resume))
model_file = os.path.basename(os.path.abspath(args.resume))
sd_opt = torch.load(
os.path.join(model_dir, model_file.replace('model', 'optim')))
args.epoch = sd_opt.pop('epoch')
optimizer.load_state_dict(sd_opt)
else:
optimizer = None
return model, optimizer