def __init__(self, config):
super(SeqSupervisedNetwork, self).__init__()
self.base_path = config['base_path']
self.early_stopping = config['early_stopping']
self.lr = config.get('meta_lr', 1e-3)
self.weight_decay = config.get('meta_weight_decay', 0.0)
if 'seq' in config['learner_model']:
self.learner = RNNSequenceModel(config['learner_params'])
elif 'mlp' in config['learner_model']:
self.learner = MLPModel(config['learner_params'])
elif 'bert' in config['learner_model']:
self.learner = BERTSequenceModel(config['learner_params'])
self.dropout = nn.Dropout(config['learner_params']['dropout_ratio'])
self.classifier = nn.Linear(
config['learner_params']['embed_dim'],
config['learner_params']['num_outputs']['ner'])
self.num_outputs = config['learner_params']['num_outputs']
self.vectors = config.get('vectors', 'glove')
if self.vectors == 'elmo':
self.elmo = Elmo(
options_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json",
weight_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5",
num_output_representations=1,
dropout=0,
requires_grad=False)
elif self.vectors == 'glove':
self.glove = torchtext.vocab.GloVe(name='840B', dim=300)
elif self.vectors == 'bert':
self.bert_tokenizer = BertTokenizer.from_pretrained(
'bert-base-cased')
self.loss_fn = {}
for task in config['learner_params']['num_outputs']:
self.loss_fn[task] = nn.CrossEntropyLoss(ignore_index=-1)
if config.get('trained_learner', False):
self.learner.load_state_dict(
torch.load(
os.path.join(self.base_path, 'saved_models',
config['trained_learner'])))
self.classifier.load_state_dict(
torch.load(
os.path.join(self.base_path, 'saved_models',
config['trained_classifier'])))
logger.info('Loaded trained learner model {}'.format(
config['trained_learner']))
self.device = torch.device(config.get('device', 'cpu'))
self.to(self.device)
if self.vectors == 'elmo':
self.elmo.to(self.device)
self.initialize_optimizer_scheduler()
def __init__(self, config):
super(SeqMetaModel, self).__init__()
self.base_path = config['base_path']
self.learner_lr = config.get('learner_lr', 1e-3)
self.output_lr = config.get('output_lr', 0.1)
if 'seq' in config['learner_model']:
self.learner = RNNSequenceModel(config['learner_params'])
elif 'mlp' in config['learner_model']:
self.learner = MLPModel(config['learner_params'])
elif 'bert' in config['learner_model']:
self.learner = BERTSequenceModel(config['learner_params'])
self.proto_maml = config.get('proto_maml', False)
self.fomaml = config.get('fomaml', False)
self.vectors = config.get('vectors', 'glove')
if self.vectors == 'elmo':
self.elmo = Elmo(
options_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json",
weight_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5",
num_output_representations=1,
dropout=0,
requires_grad=False)
elif self.vectors == 'glove':
self.glove = torchtext.vocab.GloVe(name='840B', dim=300)
elif self.vectors == 'bert':
self.bert_tokenizer = BertTokenizer.from_pretrained(
'bert-base-cased')
self.learner_loss = {}
for task in config['learner_params']['num_outputs']:
self.learner_loss[task] = nn.CrossEntropyLoss(ignore_index=-1)
self.output_layer_weight = None
self.output_layer_bias = None
if config.get('trained_learner', False):
self.learner.load_state_dict(
torch.load(
os.path.join(self.base_path, 'saved_models',
config['trained_learner'])))
logger.info('Loaded trained learner model {}'.format(
config['trained_learner']))
self.device = torch.device(config.get('device', 'cpu'))
self.to(self.device)
if self.proto_maml:
logger.info(
'Initialization of output layer weights as per prototypical networks turned on'
)
params = [p for p in self.learner.parameters() if p.requires_grad]
self.learner_optimizer = optim.SGD(params, lr=self.learner_lr)
def __init__(self, config):
super(SeqBaselineModel, self).__init__()
self.base_path = config['base_path']
self.early_stopping = config['early_stopping']
self.learner_lr = config.get('learner_lr', 1e-3)
self.weight_decay = config.get('meta_weight_decay', 0.0)
if 'seq' in config['learner_model']:
self.learner = RNNSequenceModel(config['learner_params'])
elif 'mlp' in config['learner_model']:
self.learner = MLPModel(config['learner_params'])
self.vectors = config.get('vectors', 'glove')
if self.vectors == 'elmo':
self.elmo = Elmo(
options_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json",
weight_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5",
num_output_representations=1,
dropout=0,
requires_grad=False)
elif self.vectors == 'glove':
self.glove = torchtext.vocab.GloVe(name='840B', dim=300)
self.learner_loss = {}
for task in config['learner_params']['num_outputs']:
self.learner_loss[task] = nn.CrossEntropyLoss(ignore_index=-1)
self.output_layer = None
if config.get('trained_baseline', None):
self.learner.load_state_dict(
torch.load(
os.path.join(self.base, 'saved_models',
config['trained_baseline'])))
logger.info('Loaded trained baseline model {}'.format(
config['trained_baseline']))
self.device = torch.device(config.get('device', 'cpu'))
self.to(self.device)