def finite_nin_caffe(data_opts: DataOpts, isrelu=True, isnormstoch=False) -> Tuple[NetOpts, OptimOpts]: model_string = '' isrelu = booltostr(isrelu) isnormstoch = booltostr(isnormstoch) nl = 'lnorm|s:{}'.format(isnormstoch) convparam = 'param:logdirich,stoch:0,isrelu:{},coef:4'.format(isrelu) d = '->' finish = 'fin' model_string += 'klconvb|r:5,f:192,pad:same,bias:1,{}'.format( convparam) + d + nl + d model_string += 'klconv|r:1,f:160,pad:same,bias:1,{}'.format( convparam) + d + nl + d model_string += 'klconv|r:1,f:96,pad:same,bias:1,{}'.format( convparam) + d + nl + d model_string += 'klavgpool|r:3,f:32,pad:valid,stride:2,bias:1' + d # model_string += 'dropout|p:0.5' + d model_string += 'klconv|r:5,f:192,pad:same,bias:1,{}'.format( convparam) + d + nl + d model_string += 'klconv|r:1,f:192,pad:same,bias:1,{}'.format( convparam) + d + nl + d model_string += 'klconv|r:1,f:192,pad:same,bias:1,{}'.format( convparam) + d + nl + d model_string += 'klavgpool|r:3,f:32,pad:valid,stride:2,bias:1' + d # model_string += 'dropout|p:0.5' + d model_string += 'klconv|r:3,f:192,pad:same,bias:1,{}'.format( convparam) + d + nl + d model_string += 'klconv|r:1,f:192,pad:same,bias:1,{}'.format( convparam) + d + nl + d model_string += 'klconv|r:1,f:' + str( data_opts.classnum) + ',pad:valid,bias:0,{}'.format( convparam) + d + nl + d model_string += 'klavgpool|r:7,f:32,pad:valid,stride:2,bias:1' + d + nl + d model_string += finish '''Data OPTs''' data_transforms = [] '''LR SCHED''' lr_sched = discrete_exp_decay_lr(init_lr=1, step=30, exp_decay_perstep=1) ''' Net Options''' opts_net = NetOpts( model_string, inputspatszvalidator=lambda x: x == 32, data_transforms=data_transforms, classicNet=False, weightinit=lambda x: x.normal_(0, 0.05), biasinit=lambda x: x.zero_(), ) opts_optim = OptimOpts(lr=1, lr_sched_lambda=lr_sched, type='SGD', momentum=0.9, weight_decay=0, dampening=0, nestrov=False, loss=NLLLoss(reduce=False)) '''Optimizer Options''' return opts_net, opts_optim
def vgg(data_opts: DataOpts) -> Tuple[NetOpts, OptimOpts]: model_string = '' nl = 'relu' d = '->' finish = 'fin' model_string += 'conv|r:3,f:64,pad:same,bias:1' + d + nl + d + 'bn' + d + 'dropout|p:0.3' + d model_string += 'conv|r:3,f:64,pad:same,bias:1' + d + nl + d model_string += 'maxpool|r:2,pad:valid,stride:2,bias:1' + d model_string += 'conv|r:3,f:128,pad:same,bias:1' + d + nl + d + 'bn' + d + 'dropout|p:0.4' + d model_string += 'conv|r:3,f:128,pad:same,bias:1' + d + nl + d model_string += 'maxpool|r:2,pad:valid,stride:2,bias:1' + d model_string += 'conv|r:3,f:256,pad:same,bias:1' + d + nl + d + 'bn' + d + 'dropout|p:0.4' + d model_string += 'conv|r:3,f:256,pad:same,bias:1' + d + nl + d + 'bn' + d + 'dropout|p:0.4' + d model_string += 'conv|r:3,f:256,pad:same,bias:1' + d + nl + d model_string += 'maxpool|r:2,pad:valid,stride:2,bias:1' + d model_string += 'conv|r:3,f:512,pad:same. 1 +,bias:1' + d + nl + d + 'bn' + d + 'dropout|p:0.4' + d model_string += 'conv|r:3,f:512,pad:same,bias:1' + d + nl + d + 'bn' + d + 'dropout|p:0.4' + d model_string += 'conv|r:3,f:512,pad:same,bias:1' + d + nl + d model_string += 'maxpool|r:2,pad:valid,stride:2,bias:1' + d model_string += 'conv|r:3,f:512,pad:same,bias:1' + d + nl + d + 'bn' + d + 'dropout|p:0.4' + d model_string += 'conv|r:3,f:512,pad:same,bias:1' + d + nl + d + 'bn' + d + 'dropout|p:0.4' + d model_string += 'conv|r:3,f:512,pad:same,bias:1' + d + nl + d + 'bn' + d model_string += 'maxpool|r:2,pad:valid,stride:2,bias:1' + d model_string += 'conv|r:1,f:512,pad:same,bias:1' + d + nl + d + 'dropout|p:0.5' + d model_string += 'conv|r:1,f:' + str( data_opts.classnum) + ',pad:valid,bias:1' + d model_string += 'lnorm|s:0' + d model_string += finish '''Data OPTs''' data_transforms = [transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])] '''LR SCHED''' lr_sched = vgg_lr ''' Net Options''' opts_net = NetOpts(model_string, inputspatszvalidator=lambda x: x == 32, data_transforms=data_transforms, classicNet=True, weightinit=lambda x: x.normal_(0, 0.01), biasinit=lambda x: x.zero_) '''Optimizer Options''' opts_optim = OptimOpts(lr=.001, lr_sched_lambda=lr_sched, type='SGD', momentum=0.9, weight_decay=1e-4, dampening=0, nestrov=False, loss=NLLLoss(reduce=True)) return opts_net, opts_optim
def __init__(self, char_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) # Initialize embedding vector. self.char_embeddings = char_embeddings # Initialize encode self.encoder = encoder # Initialize hidden-tag layer. # It outputs score for wach label. self.hidden2tag = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) # Initialize the average metric. self.accuracy = Average() # it’s faster and has better numerical properties compared to Softmax self.m = LogSoftmax() # The negative log likelihood loss. It is useful to train a # classification problem with `C` classes self.loss = NLLLoss()
def quick_cifar(data_opts: DataOpts) -> Tuple[NetOpts, OptimOpts]: model_string = '' nl = 'relu' d = '->' finish = 'fin' model_string += 'conv|r:5,f:32,pad:same,bias:1' + d model_string += 'maxpool|r:3,f:32,pad:same,stride:2,bias:1' + d model_string += 'conv|r:5,f:64,pad:same,bias:1' + d + nl + d model_string += 'avgpool|r:3,f:32,pad:same,stride:2,bias:1' + d model_string += 'conv|r:4,f:64,pad:same,bias:1' + d + nl + d model_string += 'conv|r:7,f:' + str( data_opts.classnum) + ',pad:valid,bias:1' + d + 'lnorm|s:0' + d model_string += finish '''Data OPTs''' data_transforms = [transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])] '''LR SCHED''' initlr = .01 lr_sched = exp_decay_lr(init_lr=initlr, step=20, exp_decay_perstep=1) ''' Net Options''' opts_net = NetOpts( model_string, inputspatszvalidator=lambda x: x == 32, data_transforms=data_transforms, classicNet=True, weightinit=lambda x: x.normal_(0, 0.05), biasinit=lambda x: x.zero_(), ) '''Optimizer Options''' opts_optim = OptimOpts(lr=initlr, lr_sched_lambda=lr_sched, type='SGD', momentum=0.9, weight_decay=1e-5, dampening=0, nestrov=False, loss=NLLLoss(reduce=True)) return opts_net, opts_optim