def test_continuous_loss_fn():
"""
Demonstrate using a custom loss function with the continuous_loss_fn
option.
"""
from sklearn.metrics import log_loss
# Generate some random data
X = np.hstack([
np.vstack([
np.random.normal(0, 1, size=(1000, 10)),
np.random.normal(1, 1, size=(1000, 10)),
]),
np.random.normal(0, 1, size=(2000, 10)),
])
y = np.zeros(2000)
y[:1000] = 1
def loss_function(targ, pred):
# hyperopt_estimator flattens the prediction when saving it. This also
# affects multilabel classification.
pred = pred.reshape((-1, 2))
return log_loss(targ, pred[:, 1])
# Try to fit an SGD model using log_loss as the loss function
cls = hyperopt_estimator(
classifier=components.sgd('sgd', loss='log'),
preprocessing=[],
loss_fn=loss_function,
continuous_loss_fn=True,
)
cls.fit(X, y, cv_shuffle=True)
def test_sparse_input():
"""
Ensure the estimator can handle sparse X matrices.
"""
import scipy.sparse as ss
# Generate some random sparse data
nrows, ncols, nnz = 100, 50, 10
ntrue = nrows // 2
D, C, R = [], [], []
for r in range(nrows):
feats = np.random.choice(range(ncols), size=nnz, replace=False)
D.extend([1] * nnz)
C.extend(feats)
R.extend([r] * nnz)
X = ss.csr_matrix((D, (R, C)), shape=(nrows, ncols))
y = np.zeros(nrows)
y[:ntrue] = 1
# Try to fit an SGD model
cls = hyperopt_estimator(
classifier=components.sgd('sgd', loss='log'),
preprocessing=[],
)
cls.fit(X, y)
def test_continuous_loss_fn():
"""
Demonstrate using a custom loss function with the continuous_loss_fn
option.
"""
from sklearn.metrics import log_loss
# Generate some random data
X = np.hstack([
np.vstack([
np.random.normal(0,1,size=(1000,10)),
np.random.normal(1,1,size=(1000,10)),
]),
np.random.normal(0,1,size=(2000,10)),
])
y = np.zeros(2000)
y[:1000] = 1
def loss_function(targ, pred):
# hyperopt_estimator flattens the prediction when saving it. This also
# affects multilabel classification.
pred = pred.reshape( (-1, 2) )
return log_loss(targ, pred[:,1])
# Try to fit an SGD model using log_loss as the loss function
cls = hyperopt_estimator(
classifier=components.sgd('sgd', loss='log'),
preprocessing=[],
loss_fn = loss_function,
continuous_loss_fn=True,
)
cls.fit(X,y,cv_shuffle=True)
def test_sparse_input():
"""
Ensure the estimator can handle sparse X matrices.
"""
import scipy.sparse as ss
# Generate some random sparse data
nrows,ncols,nnz = 100,50,10
ntrue = nrows // 2
D,C,R = [],[],[]
for r in range(nrows):
feats = np.random.choice(range(ncols), size=nnz, replace=False)
D.extend([1]*nnz)
C.extend(feats)
R.extend([r]*nnz)
X = ss.csr_matrix( (D,(R,C)), shape=(nrows, ncols))
y = np.zeros( nrows )
y[:ntrue] = 1
# Try to fit an SGD model
cls = hyperopt_estimator(
classifier=components.sgd('sgd', loss='log'),
preprocessing=[],
)
cls.fit(X,y)
def select_classifier(cls, clf_type, name='clf', printout=False):
cls.check_translate(clf_type, translate_type=name) # check parameter
classifiers_dict = dict(
svc=svc(name + '.svc'),
knn=knn(name + '.knn'),
random_forest=random_forest(name + '.random_forest'),
extra_trees=extra_trees(name + '.extra_trees'),
ada_boost=ada_boost(name + '.ada_boost'),
gradient_boosting=gradient_boosting(name + '.grad_boosting', loss='deviance'),
sgd=sgd(name + '.sgd')
)
if xgboost:
classifiers_dict['xgboost'] = xgboost_classification(name + '.xgboost')
# if clf_type in classifiers_dict.keys():
if printout:
print([classifiers_dict[clf_type]])
return hp.choice('%s' % name, [classifiers_dict[clf_type]])
def test_crossvalidation():
"""
Demonstrate performing a k-fold CV using the fit() method.
"""
# Generate some random data
X = np.hstack([
np.vstack([
np.random.normal(0, 1, size=(1000, 10)),
np.random.normal(1, 1, size=(1000, 10)),
]),
np.random.normal(0, 1, size=(2000, 10)),
])
y = np.zeros(2000)
y[:1000] = 1
# Try to fit a model
cls = hyperopt_estimator(
classifier=components.sgd('sgd', loss='log'),
preprocessing=[],
)
cls.fit(X, y, cv_shuffle=True, n_folds=5)
def test_crossvalidation():
"""
Demonstrate performing a k-fold CV using the fit() method.
"""
# Generate some random data
X = np.hstack([
np.vstack([
np.random.normal(0,1,size=(1000,10)),
np.random.normal(1,1,size=(1000,10)),
]),
np.random.normal(0,1,size=(2000,10)),
])
y = np.zeros(2000)
y[:1000] = 1
# Try to fit a model
cls = hyperopt_estimator(
classifier=components.sgd('sgd', loss='log'),
preprocessing=[],
)
cls.fit(X,y,cv_shuffle=True, n_folds=5)
def main():
# Thanks to https://stackoverflow.com/questions/32612180/eliminating-warnings-from-scikit-learn
def warn(*args, **kwargs):
pass
warnings.warn = warn
parser.add_argument('--manual_seed', type=int, help='manual seed')
directory = "C:\\Users\\alan\\Desktop\\experiments\\"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
options = parser.parse_args()
# files = get_data(directory)
# models = []
# for f in files:
# if f.endswith(".pth"):
# models.append(f)
ngpu = options.num_gpu
channels = options.channels
num_disc_filters = options.num_disc_filters
if options.gan_type == "dcgan":
if options.gradient_penalty and options.wgan:
netD = GoodDiscriminator(32).to(device)
print(netD)
elif options.wgan:
netD = WganDiscriminator(3, 64).to(device)
print(netD)
else:
netD = DiscriminatorOrig(1, 3, 64).to(device)
netD.eval()
if options.gradient_penalty:
netD.apply(weight_init)
else:
netD.apply(weights_init)
if options.disc_path != '':
netD.load_state_dict(torch.load(options.disc_path))
if options.gradient_penalty:
layers = nn.Sequential(*list(netD.children()))
else:
layers = nn.Sequential(*list(netD.children()))[0]
conv_layers = []
# print(type(layers))
# print(layers)
# sys.exit(0)
if options.gradient_penalty:
for layer in layers:
if type(layer) == MyConvo2d:
conv_layers.append(*layer.children())
if type(layer) == ResidualBlock:
for item in list(layer.children()):
if type(item) == MyConvo2d:
conv_layers.append(*item.children())
else:
for layer in layers:
if type(layer) == torch.nn.modules.conv.Conv2d:
conv_layers.append(layer)
# conv_layers.append(layers[0])
# for layer in range(1,len(layers)+1):
# if type(layers[layer])==ResidualBlock:
# if type(item) == torch.nn.modules.conv.Conv2d:
# conv_layers.append(item)
# print(netD)
def get_conv_activations_wgangp(name, count):
pool = nn.AdaptiveMaxPool2d(4)
def hook(module, input, output):
activations[str(name) + str(count)] = pool(output).view(
output.size(0), -1)
return hook
def get_conv_activations(name):
pool = nn.AdaptiveMaxPool2d(4)
def hook(module, input, output):
activations[name] = pool(output).view(output.size(0), -1)
return hook
# self.conv1 = MyConvo2d(3, self.dim, 3, he_init = False)
# self.rb1 = ResidualBlock(self.dim, 2*self.dim, 3, resample = 'down', hw=DIM)
# self.rb2 = ResidualBlock(2*self.dim, 4*self.dim, 3, resample = 'down', hw=int(DIM/2))
# self.rb3 = ResidualBlock(4*self.dim, 8*self.dim, 3, resample = 'down', hw=int(DIM/4))
# self.rb4 = ResidualBlock(8*self.dim, 8*self.dim, 3, resample = 'down', hw=int(DIM/8))
# self.ln1 = nn.Linear(4*4*8*self.dim, 1)
# Register forward hooks to get activations
if options.gradient_penalty:
netD.conv1.register_forward_hook(get_conv_activations_wgangp(
'conv', 1))
netD.rb1.register_forward_hook(get_conv_activations_wgangp('conv', 2))
netD.rb2.register_forward_hook(get_conv_activations_wgangp('conv', 3))
netD.rb3.register_forward_hook(get_conv_activations_wgangp('conv', 4))
netD.rb4.register_forward_hook(get_conv_activations_wgangp('conv', 5))
else:
netD.main[0].register_forward_hook(get_conv_activations('conv1'))
netD.main[2].register_forward_hook(get_conv_activations('conv2'))
netD.main[5].register_forward_hook(get_conv_activations('conv3'))
netD.main[8].register_forward_hook(get_conv_activations('conv4'))
netD.main[11].register_forward_hook(get_conv_activations('conv5'))
if options.dataset == 'cifar10':
training_data = dset.CIFAR10(root=options.dataroot,
download=False,
train=True,
transform=transforms.Compose([
transforms.Resize(options.image_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5),
(0.5, 0.5, 0.5)),
]))
test_data = dset.CIFAR10(root=options.dataroot,
download=False,
train=False,
transform=transforms.Compose([
transforms.Resize(options.image_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5),
(0.5, 0.5, 0.5)),
]))
train_dataloader = torch.utils.data.DataLoader(training_data,
batch_size=1,
shuffle=True,
num_workers=int(
options.workers))
test_dataloader = torch.utils.data.DataLoader(test_data,
batch_size=1,
shuffle=True,
num_workers=int(
options.workers))
else:
# thanks to @Jordi_de_la_Torre https://discuss.pytorch.org/t/balanced-sampling-between-classes-with-torchvision-dataloader/2703/2
svhn_train = dset.SVHN(root="D:\svhn",
split="train",
download=False,
transform=transforms.Compose([
transforms.Resize(options.image_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5),
(0.5, 0.5, 0.5)),
]))
svhn_test = dset.SVHN(root="D:\svhn",
split="test",
download=False,
transform=transforms.Compose([
transforms.Resize(options.image_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5),
(0.5, 0.5, 0.5)),
]))
train_dataloader = torch.utils.data.DataLoader(svhn_train,
shuffle=True,
batch_size=1,
num_workers=int(
options.workers),
pin_memory=True)
test_dataloader = torch.utils.data.DataLoader(
svhn_test,
sampler=ImbalancedDatasetSampler(svhn_test),
batch_size=1,
num_workers=int(options.workers),
pin_memory=True)
training_iter = iter(train_dataloader)
test_iter = iter(test_dataloader)
# print(len(train_dataloader),len(test_dataloader))
i = 0
x_train = []
# Heavily inspired by https://github.com/pytorch/examples/blob/master/dcgan/main.py
# x=[]
y_train = []
while i < 50000:
data = training_iter.next()
data[0] = data[0].to(device)
if options.gradient_penalty:
data[0] = F.interpolate(data[0], 64)
activations = {}
output = netD(data[0])
get_conv_activations('conv1')
get_conv_activations('conv2')
get_conv_activations('conv3')
get_conv_activations('conv4')
get_conv_activations('conv5')
act_flat = []
y_train.append(data[1])
for key in activations:
act_flat.append(activations[key])
act_flat = torch.cat(act_flat, 1)[0]
x_train.append(act_flat.detach().cpu().numpy())
i += 1
x_train = np.asarray(x_train)
y_train = np.asarray(y_train)
j = 0
x_test = []
y_test = []
if options.dataset == 'svhn':
test_length = len(test_dataloader)
else:
test_length = 10000
while j < test_length:
data = test_iter.next()
data[0] = data[0].to(device)
if options.gradient_penalty:
data[0] = F.interpolate(data[0], 64)
activations = {}
output = netD(data[0])
get_conv_activations('conv1')
get_conv_activations('conv2')
get_conv_activations('conv3')
get_conv_activations('conv4')
get_conv_activations('conv5')
act_flat = []
y_test.append(data[1])
for key in activations:
act_flat.append(activations[key])
act_flat = torch.cat(act_flat, 1)[0]
x_test.append(act_flat.detach().cpu().numpy())
j += 1
x_test = np.asarray(x_test)
y_test = np.asarray(y_test)
title = str(options.gan_type + "_wgan-" + str(options.wgan) + "_" +
options.dataset + "_" + options.optimise + "_gradpen-" +
str(options.gradient_penalty) + ".txt")
output_file = open(title, "w")
if options.optimise != 'no_optimisation':
model = HyperoptEstimator(classifier=hpc.sgd('_sgd',
penalty='l2',
n_jobs=-1),
max_evals=25,
trial_timeout=1200)
else:
model = linear_model.SGDClassifier(penalty='l2', n_jobs=-1)
# model = HyperoptEstimator(classifier=any_classifier('my_clf'),
# algo=tpe.suggest,
# max_evals=10,
# trial_timeout=30)
#model = TPOTClassifier(generations=5, population_size=20, verbosity=2)
# model = ac.AutoSklearnClassifier()
#model = linear_model.
#model = linear_model.SGDClassifier(n_jobs=-1)
# grid = {
# 'alpha': [1e-4,1e-3,1e-2,1e-1,1e0,1e1,1e2,1e3],
# 'max_iter': [1000],
# 'loss': ['hinge'],
# 'penalty': ['l2'],
# 'n_jobs': [-1]
# }
# param_grid = ParameterGrid(grid)
# best_model, best_score, all_models, all_scores = pf.bestFit(linear_model.SGDClassifier,param_grid,x_train,y_train,
# x_test,y_test,metric=accuracy_score,scoreLabel='Acc')
# print(best_model,best_score)
model.fit(x_train, y_train)
# #print(model,"is the model")
# predictions = model.predict(x_test)
# print(predictions,"are the predictions")
score = model.score(x_test, y_test)
output_file.write("Score:\n" + str(score))
if options.optimise != 'no_optimisation':
best = str(model.best_model())
output_file.write("\nBest model:\n" + str(best))
else:
params = str(model.get_params())
output_file.write("\nModel parameters:\n" + str(params))
# score = model.score(x_test,y_test)
#output_file.write("\n"+str(best))
# output_file.write(str(np.asarray(predictions))+"\n")
# output_file.write(str(np.asarray(y_test))+"\n")
output_file.close()