def test_b():
# -- second by simply freezing them
np.random.seed(0)
torch.manual_seed(0)
print('2', end='')
mod = make_module_cls()()
opt = torch.optim.SGD
cb = skorch.callbacks.Freezer(
['sequential.{}.weight'.format(i) for i in linear_idcs[:-1]] +
['sequential.{}.bias'.format(i) for i in linear_idcs[:-1]])
net = skorch.NeuralNetClassifier(mod,
verbose=0,
optimizer=opt,
callbacks=[cb])
net.fit(X, y)
for i in linear_idcs[:-1]:
assert not mod.sequential[i].weight.requires_grad
assert not net.module_.sequential[i].weight.requires_grad
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return rss, net.history[-1, 'valid_loss'], np.mean(net.history[:, 'dur'])
def build_model(lr=1e-4, max_epochs=2):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = skorch.NeuralNetClassifier(
module=ClassifierModule,
module__backbone=torchvision.models.resnet18(pretrained=True),
module__output_dim=10,
criterion=torch.nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
optimizer__param_groups=[
('conv1.*', {'lr': lr / 10}),
('bn1.*', {'lr': lr / 10}),
('layer1.*', {'lr': lr / 8}),
('layer2.*', {'lr': lr / 6}),
('layer3.*', {'lr': lr / 4}),
('layer4.*', {'lr': lr / 2}),
],
optimizer__lr=lr,
max_epochs=max_epochs,
batch_size=256,
iterator_train__shuffle=True,
iterator_valid__shuffle=False,
device=device,
callbacks=[
ReportParams()
],
)
return model
def test_a():
# -- first by stripping parameters explicitly
np.random.seed(0)
torch.manual_seed(0)
print('1', end='')
mod = make_module_cls()()
# freeze all params but last layer
for i in linear_idcs[:-1]:
skorch.utils.freeze_parameter(mod.sequential[i].weight)
skorch.utils.freeze_parameter(mod.sequential[i].bias)
opt = skorch.helper.filtered_optimizer(torch.optim.SGD,
skorch.helper.filter_requires_grad)
net = skorch.NeuralNetClassifier(mod,
verbose=0,
optimizer=opt,
warm_start=True)
for i in linear_idcs[:-1]:
assert not mod.sequential[i].weight.requires_grad
net.fit(X, y)
for i in linear_idcs[:-1]:
assert not mod.sequential[i].weight.requires_grad
assert not net.module_.sequential[i].weight.requires_grad
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return rss, net.history[-1, 'valid_loss'], np.mean(net.history[:, 'dur'])
def train_simple_nn(X, y, num_in, num_feat):
nnet = skorch.NeuralNetClassifier(
MySimpleNet,
module__num_in=num_in,
module__num_feat=num_feat,
max_epochs=10,
lr=0.1,
)
nnet.fit(X, y)
return nnet
def get_sklearn_compatible_estimator(self):
EPOCHS = 7
criterion = nn.CrossEntropyLoss
optimizer = optim.Adam
net_with_params = functools.partial(LSTMModel,
vector_size=self.vector_size)
return skorch.NeuralNetClassifier(net_with_params,
max_epochs=EPOCHS,
criterion=criterion,
optimizer=optimizer)
def objective(trial: optuna.trial.Trial) -> float:
net = skorch.NeuralNetClassifier(
ClassifierModule,
max_epochs=10,
lr=0.02,
callbacks=[SkorchPruningCallback(trial, "valid_acc")],
)
net.fit(X, y)
return 1.0
def objective(trial: optuna.Trial) -> float:
net = skorch.NeuralNetClassifier(
ClassifierModule(trial),
max_epochs=20,
lr=0.1,
device=device,
callbacks=[SkorchPruningCallback(trial, "valid_acc")],
)
net.fit(X_train, y_train)
return accuracy_score(y_test, net.predict(X_test))
def main():
# Load data
path_to_data = os.path.abspath('cifar10/cifar-10-batches-py/')
cifar10 = cifar10_utils.get_cifar10(data_dir=path_to_data,
one_hot=False,
validation_size=0)
train_set = cifar10['train']
# Initialize model
input_dim = train_set.images[0, :, :, :].size
n_classes = train_set.labels.max() + 1
net = skorch.NeuralNetClassifier(MLP,
criterion=nn.CrossEntropyLoss,
module__n_inputs=input_dim,
module__n_classes=n_classes,
optimizer=torch.optim.Adam)
# params = {
# 'lr' : [0.1, 0.01, 0.001],
# 'module__n_hidden' : [
# [100, 100],
# [1000],
# [50,30,20]
# ],
# 'batch_size' : [64, 128, 256, 512]
# }
params = {
'lr': [0.002],
'module__n_hidden': [[500, 500, 500, 500]],
'optimizer': [torch.optim.Adam]
}
gs = GridSearchCV(net,
params,
cv=5,
scoring='accuracy',
n_jobs=4,
refit=False,
verbose=2)
gs.fit(train_set.images.reshape(train_set.images.shape[0], -1),
train_set.labels)
print()
print('--------')
print()
print('Best params:\t', gs.best_params_)
print('Best score:\t', gs.best_score_)
print()
with open('gridsearch_full_training_set.dill', 'wb') as f:
dill.dump(gs, f)
def fit(self,X,y,**fit_params):
X=X.astype(np.float32)
y=y.astype(np.int64)
ClassifierModule.Xsize = X.shape[-1]
ClassifierModule.ysize = y.shape[-1]
l=ClassifierModule.ysize
nets = [None] * l
for i in range(l):
net = skorch.NeuralNetClassifier(
ClassifierModule,
max_epochs=self.max_epochs,
lr=self.lr
#train_split=skorch.dataset.CVSplit(5, stratified=False)
# device='cuda', # uncomment this to train with CUDA
)
nets[i] = net.fit(X, y[:, i],**fit_params)
self.nets=nets
def create_neural_classifier():
"""
Common neural net classifier.
"""
optimizer_parameters = {
'weight_decay': 1e-4,
'lr': 1e-3,
'eps': 1e-16,
'betas': (0.9, 0.999),
'weight_decouple': True,
'rectify': False
}
optimizer_parameters = {
"optimizer__" + key: v
for key, v in optimizer_parameters.items()
}
optimizer_parameters['optimizer'] = AdaBelief
checkpoint_callback = skorch.callbacks.Checkpoint(
dirname=DEEP_CACHE_DIR, monitor='train_loss_best')
callbacks = [checkpoint_callback, LoadEndState(checkpoint_callback)]
parameters = {
'callbacks': callbacks,
'device': 'cuda',
'max_epochs': 300,
'train_split': None,
'batch_size': 80,
'iterator_train': BalancedDataLoader,
'iterator_train__num_workers': 0,
'iterator_train__pin_memory': False,
'module__embedding_net': lmelloEmbeddingNet(8),
'module__n_classes': 5
}
parameters = {**parameters, **optimizer_parameters}
convnet = skorch.NeuralNetClassifier(ClassificationNet, **parameters)
return 'convnet', convnet, {}
num_inputs = 4
model = nn.Sequential(nn.Linear(num_inputs, hidden_units), nn.ReLU(),
nn.Dropout(), nn.Linear(hidden_units, num_classes),
nn.Softmax())
return model
iris = datasets.load_iris()
X = iris.data
Y = iris.target
# print(iris.DESCR)
clf = skorch.NeuralNetClassifier(
create_model(),
optimizer=torch.optim.Adam,
max_epochs=10,
)
print(X.shape, Y.shape)
clf.fit(X, Y)
def test_ws_sql_gen(pickle_data):
WS_URL = "https://sklearn2sql.herokuapp.com/model"
# WS_URL="http://localhost:1888/model"
b64_data = base64.b64encode(pickle_data).decode('utf-8')
data = {
"Name": "model1",
"PickleData": b64_data,
"SQLDialect": "postgresql"
net = skorch.NeuralNetClassifier(
module=wavetorch.WaveCell,
# Training configuration
max_epochs=cfg['training']['N_epochs'],
batch_size=cfg['training']['batch_size'],
lr=cfg['training']['lr'],
# train_split=skorch.dataset.CVSplit(cfg['training']['N_folds'], stratified=True, random_state=cfg['seed']),
optimizer=torch.optim.Adam,
criterion=torch.nn.CrossEntropyLoss,
callbacks=[
ClipDesignRegion,
skorch.callbacks.EpochScoring('accuracy',
lower_is_better=False,
on_train=True,
name='train_acc'),
skorch.callbacks.Checkpoint(monitor=None,
fn_prefix='1234_',
dirname='test',
f_params="params_{last_epoch[epoch]}.pt",
f_optimizer='optimizer.pt',
f_history='history.json')
],
callbacks__print_log__keys_ignored=None,
train_split=None,
# These al get passed as options to WaveCell
module__Nx=cfg['geom']['Nx'],
module__Ny=cfg['geom']['Ny'],
module__h=cfg['geom']['h'],
module__dt=cfg['geom']['dt'],
module__init=cfg['geom']['init'],
module__c0=cfg['geom']['c0'],
module__c1=cfg['geom']['c1'],
module__sigma=cfg['geom']['pml']['max'],
module__N=cfg['geom']['pml']['N'],
module__p=cfg['geom']['pml']['p'],
module__design_region=design_region,
module__output_probe=True,
module__probes=probes,
module__sources=source)
)
early_stopping = skorch.callbacks.EarlyStopping(
monitor='valid_loss',
patience=1,
threshold=0,
threshold_mode='rel',
lower_is_better=True,
)
val_dataset = Dataset(X=xs['val'], y=ys['val'])
clf = skorch.NeuralNetClassifier(
module=RNN,
device=device,
callbacks=[('early_stopping', early_stopping), ('checkpoint', checkpoint)],
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
train_split=predefined_split(val_dataset),
iterator_train__shuffle=True,
iterator_train__drop_last=True,
iterator_valid__drop_last=False,
iterator_train__batch_size=128,
iterator_valid__batch_size=-1, # use all examples
verbose=1
)
# FIT MODEL
clf.fit(X=xs['train'], y=ys['train'])
clf.initialize()
clf.load_params('challenges/spooky_author_identification/checkpoint/params.pt'.format(c=CHALLENGE))
# # CALIBRATE PROBABILITIES
name='train_acc')
callback_save_best = Checkpoint(monitor='valid_acc_best',
f_params=None,
f_optimizer=None,
f_criterion=None,
f_history=None,
f_pickle=best_model_file_path,
event_name='event_cp')
callback_early_stop = EarlyStopping(monitor='valid_loss',
patience=30,
threshold_mode='rel',
lower_is_better=True)
net = skorch.NeuralNetClassifier(
module=model,
criterion=criterion,
train_split=skorch.dataset.CVSplit(10, stratified=True),
optimizer=optimizer,
lr=lr,
max_epochs=150,
batch_size=32,
dataset=dataset,
device=device,
callbacks=[callback_train_acc, callback_early_stop, callback_save_best]
# optimizer__weight_decay = 3e-4,
)
if __name__ == '__main__':
x_train = np.array(X for X, y in iter(dataset))
y_train = np.array([y for X, y in iter(dataset)])
net.fit(X=x_train, y=y_train)
def torch_attack_model_fn():
model = skorch.NeuralNetClassifier(module=AttackNet,
max_epochs=5,
criterion=nn.BCELoss,
train_split=None)
return model
def torch_shadow_model_fn():
model = skorch.NeuralNetClassifier(module=ShadowNet,
max_epochs=5,
criterion=CrossEntropyOneHot,
train_split=None)
return model