def decoder_wrapper(
model_file_path: str = "http://dmserv4.cs.illinois.edu/pner0.th",
configs: dict = {}):
"""
Wrapper for different decode functions.
Parameters
----------
model_file_path: ``str``, optional, (default = "http://dmserv4.cs.illinois.edu/pner0.th").
Path to loaded checkpoint.
configs: ``dict``, optional, (default = "{}").
Additional configs.
"""
pw = wrapper(configs.get("log_path", None))
logger.info(
"Loading model from {} (might download from source if not cached).".
format(model_file_path))
model_file = wrapper.restore_checkpoint(model_file_path)
model_type = model_file['config'].get("model_type", 'char-lstm-crf')
logger.info('Preparing the pre-trained {} model.'.format(model_type))
model_type_dict = { \
"char-lstm-crf": decoder_wc,
"char-lstm-two-level": decoder_tl}
return model_type_dict[model_type](model_file, pw, configs)
def decoder_wrapper(model_file_path: str, configs: dict = {}):
"""
Wrapper for different decode functions.
Parameters
----------
model_file_path: ``str``, required.
Path to loaded checkpoint.
configs: ``dict``, optional, (default = "{}").
Additional configs.
"""
pw = wrapper(configs.get("log_path", None))
pw.set_level(configs.get("log_level", 'info'))
pw.info(
"Loading model from {} (might download from source if not cached).".
format(model_file_path))
model_file = wrapper.restore_checkpoint(model_file_path)
model_type = configs.get("model_type", 'char-lstm-crf')
pw.info('Preparing the pre-trained {} model.'.format(model_type))
model_type_dict = {"char-lstm-crf": decoder_wc}
return model_type_dict[model_type](model_file, pw, configs)
parser.add_argument('--add_relu', action='store_true')
parser.add_argument('--layer_drop', type=float, default=0.5)
parser.add_argument('--epoch', type=int, default=400)
parser.add_argument('--clip', type=float, default=5)
parser.add_argument('--update', choices=['Adam', 'Adagrad', 'Adadelta'], default='Adam', help='adam is the best')
parser.add_argument('--rnn_layer', choices=['Basic', 'DenseNet', 'LDNet'], default='LDNet')
parser.add_argument('--rnn_unit', choices=['gru', 'lstm', 'rnn'], default='lstm')
parser.add_argument('--lr', type=float, default=0.001)
parser.add_argument('--lr_decay', type=float, default=0.1)
parser.add_argument('--cut_off', nargs='+', default=[4000,40000,200000])
parser.add_argument('--interval', type=int, default=100)
parser.add_argument('--epoch_size', type=int, default=4000)
parser.add_argument('--patience', type=float, default=10)
args = parser.parse_args()
pw = wrapper(os.path.join(args.cp_root, args.checkpoint_name), args.checkpoint_name, enable_git_track=args.git_tracking)
pw.set_level('info')
gpu_index = pw.auto_device() if 'auto' == args.gpu else int(args.gpu)
device = torch.device("cuda:" + str(gpu_index) if gpu_index >= 0 else "cpu")
if gpu_index >= 0:
torch.cuda.set_device(gpu_index)
pw.info('Loading dataset.')
dataset = pickle.load(open(args.dataset_folder + 'test.pk', 'rb'))
w_map, test_data, range_idx = dataset['w_map'], dataset['test_data'], dataset['range']
train_loader = LargeDataset(args.dataset_folder, range_idx, args.batch_size, args.sequence_length)
test_loader = EvalDataset(test_data, args.batch_size)
pw.info('Building models.')
rnn_map = {'Basic': BasicRNN, 'DenseNet': DenseRNN, 'LDNet': functools.partial(LDRNN, layer_drop = args.layer_drop)}
from abnlp.optim import Nadam
logger = logging.getLogger(__name__)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--cp_root', default="./checkpoint")
parser.add_argument('--config', default="./config/ner0.json")
parser.add_argument('--train_file', default="./config/ner0.json")
conf = parser.parse_args()
with open(conf.config, 'r') as fin:
args = json.load(fin)
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]),
args["checkpoint_name"])
logger.info('Loading the data...')
train_data = strFromFileEncoderWrapper(args,
processed_file=conf.train_file)
logger.info('Checking the device...')
gpu_index = pw.auto_device() if 'auto' == args["gpu"] else int(args["gpu"])
device = torch.device("cuda:" +
str(gpu_index) if gpu_index >= 0 else "cpu")
if gpu_index >= 0:
torch.cuda.set_device(gpu_index)
logger.info("Exp: {}".format(args['checkpoint_name']))
logger.info("Config: {}".format(args))
from betanlp.model import seqLabel, seqLabelEvaluator
from betanlp.encoder import strFromFileEncoderWrapper
from betanlp.common.utils import adjust_learning_rate
from betanlp.optim import Nadam
logger = logging.getLogger(__name__)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--config', default="./config/twitter_ner.json")
conf = parser.parse_args()
with open(conf.config, 'r') as fin:
args = json.load(fin)
pw = wrapper(os.path.join(args["cp_root"], args["checkpoint_name"]), args["checkpoint_name"], enable_git_track=args["git_tracking"])
gpu_index = pw.auto_device() if 'auto' == args["gpu"] else int(args["gpu"])
device = torch.device("cuda:" + str(gpu_index) if gpu_index >= 0 else "cpu")
if gpu_index >= 0:
torch.cuda.set_device(gpu_index)
logger.info("Exp: {}".format(args['checkpoint_name']))
logger.info("Config: {}".format(args))
logger.info('Saving the configure...')
pw.save_configue(args)
logger.info('Building the model...')
model = seqLabel(args)
evaluator = seqLabelEvaluator(model.spDecoder.to_spans)
