def simpleFlowTrial(strict=False):
def makeTrainer(config):
cfg = {
'dataset': CIFAR10,
'network': iCNN,
'net_config': {},
'loader_config': {
'amnt_dev': 5000,
'lab_BS': 32,
'pin_memory': True,
'num_workers': 3
},
'opt_config': {
'lr': .0003,
}, # 'momentum':.9, 'weight_decay':1e-4,'nesterov':True},
'num_epochs': 100,
'trainer_config': {},
'parallel': False,
}
recursively_update(cfg, config)
train_transforms = transforms.Compose(
[transforms.ToTensor(),
transforms.RandomHorizontalFlip()])
trainset = cfg['dataset'](
'~/datasets/{}/'.format(cfg['dataset']),
flow=True,
)
device = torch.device('cuda')
fullCNN = cfg['network'](num_classes=trainset.num_classes,
**cfg['net_config']).to(device)
if cfg['parallel']: fullCNN = multigpu_parallelize(fullCNN, cfg)
dataloaders = {}
dataloaders['train'], dataloaders['dev'] = getLabLoader(
trainset, **cfg['loader_config'])
dataloaders['Train'] = islice(dataloaders['train'],
10000 // cfg['loader_config']['lab_BS'])
if len(dataloaders['dev']) == 0:
testset = cfg['dataset']('~/datasets/{}/'.format(cfg['dataset']),
train=False,
flow=True)
dataloaders['test'] = DataLoader(
testset,
batch_size=cfg['loader_config']['lab_BS'],
shuffle=False)
dataloaders = {k: LoaderTo(v, device)
for k, v in dataloaders.items()} #LoaderTo(v,device)
opt_constr = lambda params: torch.optim.Adam(params, **cfg['opt_config'
])
lr_sched = cosLr(cfg['num_epochs'])
return Flow(fullCNN, dataloaders, opt_constr, lr_sched,
**cfg['trainer_config'])
return train_trial(makeTrainer, strict)
def simpleiClassifierTrial(strict=False):
def makeTrainer(config):
cfg = {
'dataset': CIFAR10,
'network': layer13s,
'net_config': {},
'loader_config': {
'amnt_dev': 5000,
'lab_BS': 20,
'pin_memory': True,
'num_workers': 2
},
'opt_config': {
'lr': .3e-4
}, #, 'momentum':.9, 'weight_decay':1e-4,'nesterov':True},
'num_epochs': 100,
'trainer_config': {},
}
recursively_update(cfg, config)
trainset = cfg['dataset']('~/datasets/{}/'.format(cfg['dataset']),
flow=True)
device = torch.device('cuda')
fullCNN = torch.nn.Sequential(
trainset.default_aug_layers(),
cfg['network'](num_classes=trainset.num_classes,
**cfg['net_config']).to(device))
dataloaders = {}
dataloaders['train'], dataloaders['dev'] = getLabLoader(
trainset, **cfg['loader_config'])
dataloaders['Train'] = islice(dataloaders['train'],
10000 // cfg['loader_config']['lab_BS'])
if len(dataloaders['dev']) == 0:
testset = cfg['dataset']('~/datasets/{}/'.format(cfg['dataset']),
train=False)
dataloaders['test'] = DataLoader(
testset,
batch_size=cfg['loader_config']['lab_BS'],
shuffle=False)
dataloaders = {k: LoaderTo(v, device) for k, v in dataloaders.items()}
opt_constr = lambda params: torch.optim.Adam(params, **cfg[
'opt_config']) #torch.optim.SGD(params, **cfg['opt_config'])
lr_sched = cosLr(cfg['num_epochs'])
return iClassifier(fullCNN, dataloaders, opt_constr, lr_sched,
**cfg['trainer_config'])
return train_trial(makeTrainer, strict)
}
# subsampled training dataloader for faster logging of training performance
dataloaders['Train'] = islice(dataloaders['train'],
len(dataloaders['train']) // 10)
# Initialize optimizer and learning rate schedule
opt_constr = functools.partial(Adam, lr=lr)
cos = cosLr(num_epochs)
lr_sched = lambda e: min(e / (.01 * num_epochs), 1) * cos(e)
return MoleculeTrainer(model,
dataloaders,
opt_constr,
lr_sched,
task=task,
ds_stats=ds_stats,
**trainer_config)
Trial = train_trial(makeTrainer)
if __name__ == '__main__':
defaults = copy.deepcopy(makeTrainer.__kwdefaults__)
defaults['trainer_config']['early_stop_metric'] = 'Train_MAE'
print(
Trial(
argupdated_config(defaults,
namespace=(moleculeTrainer, lieGroups))))
# thestudy = Study(simpleTrial,argupdated_config(config_spec,namespace=__init__),
# study_name="point2d",base_log_dir=log_dir)
# thestudy.run(ordered=False)
device = torch.device('cuda')
model = cfg['network'](num_classes=trainset.num_classes,dim_in=trainset.dim,**cfg['net_config']).to(device)
dataloaders = {}
dataloaders['lab'], dataloaders['dev'] = getLabLoader(trainset,**cfg['loader_config'])
dataloaders['train'] = dataloaders['Train'] = dataloaders['lab']
full_data_loader = DataLoader(trainset,shuffle=True,**cfg['unlab_loader_config'])
dataloaders['_unlab'] = imap(lambda z: z[0], full_data_loader)
dataloaders['test'] = DataLoader(testset,batch_size=cfg['loader_config']['lab_BS'],shuffle=False)
dataloaders = {k:LoaderTo(v,device) for k,v in dataloaders.items()}
opt_constr = lambda params: torch.optim.Adam(params, **cfg['opt_config'])
lr_sched = lambda e: 1.#cosLr(cfg['num_epochs'])
return cfg['trainer'](model,dataloaders,opt_constr,lr_sched,**cfg['trainer_config'])
PI_trial = train_trial(makeTabularTrainer,strict=True)
uci_pi_spec = {'network':SmallNN,'net_config': {},'dataset':[MINIBOONE,HEPMASS],
#'loader_config': {'lab_BS':200},
'opt_config': {'lr':[1e-3,3e-3,1e-4]},
'loader_config': {'amnt_labeled':20+5000,'amnt_dev':5000,'lab_BS':20},
'num_epochs':50,#100,#100,#5,#800,
#'unlab_loader_config':{'batch_size':2000},
'net_config':{'k':[256,512]},'trainer':PiModel,
'trainer_config':{'log_dir':os.path.expanduser('~/tb-experiments/UCI/t3layer_pi_uci3/'),
'cons_weight':[20,30,50]}#[1,.1,.3,3],}#'advEps':[10,3,1,.3]}
}
uci_pi_spec2 = {'network':SmallNN,'net_config': {},'dataset':[MINIBOONE],
#'loader_config': {'lab_BS':200},
'opt_config': {'lr':lambda cfg: 3e-3 if cfg['dataset']==HEPMASS else 3e-5},
'loader_config': {'amnt_labeled':20+5000,'amnt_dev':5000,'lab_BS':20},
model = network(num_classes=datasets['train'].num_classes,
dim_in=datasets['train'].dim,
**net_config).to(device)
dataloaders = {
k: LoaderTo(
DataLoader(v,
batch_size=min(bs, len(datasets[k])),
shuffle=(k == 'train'),
num_workers=0,
pin_memory=False), device)
for k, v in datasets.items()
}
dataloaders['Train'] = dataloaders['train']
opt_constr = partial(optim, lr=lr, **opt_config)
lr_sched = cosLr(num_epochs) #lambda e:1#
return trainer(model, dataloaders, opt_constr, lr_sched, **trainer_config)
tabularTrial = train_trial(makeTrainer)
if __name__ == '__main__':
defaults = copy.deepcopy(makeTrainer.__kwdefaults__)
cfg = argupdated_config(defaults,
namespace=(tabular_datasets, flows, archs,
trainers))
cfg.pop('local_rank')
trainer = makeTrainer(**cfg)
#tabularTrial()
trainer.train(cfg['num_epochs'])
