def __init__(self, device, n_classes, **kwargs):
self.inner_lr = kwargs.get('inner_lr')
self.meta_lr = kwargs.get('meta_lr')
self.write_prob = kwargs.get('write_prob')
self.replay_rate = kwargs.get('replay_rate')
self.replay_every = kwargs.get('replay_every')
self.device = device
self.nm = TransformerNeuromodulator(model_name=kwargs.get('model'),
device=device)
self.pn = TransformerClsModel(model_name=kwargs.get('model'),
n_classes=n_classes,
max_length=kwargs.get('max_length'),
device=device)
self.memory = ReplayMemory(write_prob=self.write_prob, tuple_size=2)
self.loss_fn = nn.CrossEntropyLoss()
logger.info('Loaded {} as NM'.format(self.nm.__class__.__name__))
logger.info('Loaded {} as PN'.format(self.pn.__class__.__name__))
meta_params = [p for p in self.nm.parameters() if p.requires_grad] + \
[p for p in self.pn.parameters() if p.requires_grad]
self.meta_optimizer = AdamW(meta_params, lr=self.meta_lr)
inner_params = [p for p in self.pn.parameters() if p.requires_grad]
self.inner_optimizer = optim.SGD(inner_params, lr=self.inner_lr)
def __init__(self, device, **kwargs):
self.lr = kwargs.get('lr', 3e-5)
self.write_prob = kwargs.get('write_prob')
self.replay_rate = kwargs.get('replay_rate')
self.replay_every = kwargs.get('replay_every')
self.device = device
self.model = TransformerClsModel(model_name=kwargs.get('model'),
n_classes=1,
max_length=kwargs.get('max_length'),
device=device)
self.memory = ReplayMemory(write_prob=self.write_prob, tuple_size=3)
logger.info('Loaded {} as the model'.format(self.model.__class__.__name__))
params = [p for p in self.model.parameters() if p.requires_grad]
self.optimizer = AdamW(params, lr=self.lr)
self.loss_fn = nn.BCEWithLogitsLoss()
def __init__(self, device, n_classes, **kwargs):
self.lr = kwargs.get('lr', 3e-5)
self.write_prob = kwargs.get('write_prob')
self.replay_rate = kwargs.get('replay_rate')
self.replay_every = kwargs.get('replay_every')
self.device = device
self.model = TransformerClsModel(model_name=kwargs.get('model'),
n_classes=n_classes,
max_length=kwargs.get('max_length'),
device=device,
hebbian=kwargs.get('hebbian'))
self.memory = ReplayMemory(write_prob=self.write_prob, tuple_size=2)
logger.info('Loaded {} as model'.format(self.model.__class__.__name__))
self.loss_fn = nn.CrossEntropyLoss()
self.optimizer = AdamW(
[p for p in self.model.parameters() if p.requires_grad],
lr=self.lr)
def __init__(self, device, **kwargs):
self.inner_lr = kwargs.get('inner_lr')
self.meta_lr = kwargs.get('meta_lr')
self.write_prob = kwargs.get('write_prob')
self.replay_rate = kwargs.get('replay_rate')
self.replay_every = kwargs.get('replay_every')
self.device = device
self.rln = TransformerRLN(model_name=kwargs.get('model'),
max_length=kwargs.get('max_length'),
device=device)
self.pln = LinearPLN(in_dim=768, out_dim=1, device=device)
meta_params = [p for p in self.rln.parameters() if p.requires_grad] + \
[p for p in self.pln.parameters() if p.requires_grad]
self.meta_optimizer = AdamW(meta_params, lr=self.meta_lr)
self.memory = ReplayMemory(write_prob=self.write_prob, tuple_size=3)
self.loss_fn = nn.BCEWithLogitsLoss()
logger.info('Loaded {} as RLN'.format(self.rln.__class__.__name__))
logger.info('Loaded {} as PLN'.format(self.pln.__class__.__name__))
inner_params = [p for p in self.pln.parameters() if p.requires_grad]
self.inner_optimizer = optim.SGD(inner_params, lr=self.inner_lr)
def __init__(self, device, **kwargs):
self.inner_lr = kwargs.get('inner_lr')
self.meta_lr = kwargs.get('meta_lr')
self.write_prob = kwargs.get('write_prob')
self.replay_rate = kwargs.get('replay_rate')
self.replay_every = kwargs.get('replay_every')
self.device = device
self.pn = TransformerClsModel(model_name=kwargs.get('model'),
n_classes=1,
max_length=kwargs.get('max_length'),
device=device,
hebbian=kwargs.get('hebbian'))
logger.info('Loaded {} as PN'.format(self.pn.__class__.__name__))
meta_params = [p for p in self.pn.parameters() if p.requires_grad]
self.meta_optimizer = AdamW(meta_params, lr=self.meta_lr)
self.memory = ReplayMemory(write_prob=self.write_prob, tuple_size=3)
self.loss_fn = nn.BCEWithLogitsLoss()
inner_params = [p for p in self.pn.parameters() if p.requires_grad]
self.inner_optimizer = optim.SGD(inner_params, lr=self.inner_lr)