def test_sgd_no_mon():
# tests that we can run the sgd algorithm
# wihout a monitoring dataset
# does not test for correctness at all, just
# that the algorithm runs without dying
dim = 3
m = 10
rng = np.random.RandomState([25, 9, 2012])
X = rng.randn(m, dim)
idx = rng.randint(0, dim, (m, ))
Y = np.zeros((m, dim))
for i in xrange(m):
Y[i, idx[i]] = 1
dataset = DenseDesignMatrix(X=X, y=Y)
m = 15
X = rng.randn(m, dim)
idx = rng.randint(0, dim, (m, ))
Y = np.zeros((m, dim))
for i in xrange(m):
Y[i, idx[i]] = 1
model = SoftmaxModel(dim)
learning_rate = 1e-3
batch_size = 5
cost = CrossEntropy()
# We need to include this so the test actually stops running at some point
termination_criterion = EpochCounter(5)
algorithm = SGD(learning_rate,
cost,
batch_size=5,
monitoring_dataset=None,
termination_criterion=termination_criterion,
update_callbacks=None,
init_momentum=None,
set_batch_size=False)
train = Train(dataset,
model,
algorithm,
save_path=None,
save_freq=0,
callbacks=None)
train.main_loop()
def test_sgd_unsup():
# tests that we can run the sgd algorithm
# on an supervised cost.
# does not test for correctness at all, just
# that the algorithm runs without dying
dim = 3
m = 10
rng = np.random.RandomState([25, 9, 2012])
X = rng.randn(m, dim)
dataset = DenseDesignMatrix(X=X)
m = 15
X = rng.randn(m, dim)
# including a monitoring datasets lets us test that
# the monitor works with supervised data
monitoring_dataset = DenseDesignMatrix(X=X)
model = SoftmaxModel(dim)
learning_rate = 1e-3
batch_size = 5
cost = DummyCost()
# We need to include this so the test actually stops running at some point
termination_criterion = EpochCounter(5)
algorithm = SGD(learning_rate,
cost,
batch_size=5,
monitoring_batches=3,
monitoring_dataset=monitoring_dataset,
termination_criterion=termination_criterion,
update_callbacks=None,
init_momentum=None,
set_batch_size=False)
train = Train(dataset,
model,
algorithm,
save_path=None,
save_freq=0,
callbacks=None)
train.main_loop()
def get_layer_trainer(layer):
# configs on sgd
config = {'learning_rate': 0.1,
'cost' : MeanSquaredReconstructionError(),
'batch_size': 10,
'monitoring_batches': 10,
'monitoring_dataset': ToyDataset(),
'termination_criterion': EpochCounter(max_epochs=100),
'update_callbacks': None
}
train_algo = UnsupervisedExhaustiveSGD(**config)
model = layer
callbacks = None
return LayerTrainer(model, train_algo, callbacks)
def test_sgd_sequential():
# tests that requesting train_iteration_mode = 'sequential'
# works
dim = 1
batch_size = 3
m = 5 * batch_size
dataset = ArangeDataset(m)
model = SoftmaxModel(dim)
learning_rate = 1e-3
batch_size = 5
visited = [False] * m
def visit(X):
assert X.shape[1] == 1
assert np.all(X[1:] == X[0:-1] + 1)
start = int(X[0, 0])
if start > 0:
assert visited[start - 1]
for i in xrange(batch_size):
assert not visited[start + i]
visited[start + i] = 1
cost = CallbackCost(visit)
# We need to include this so the test actually stops running at some point
termination_criterion = EpochCounter(5)
algorithm = SGD(learning_rate,
cost,
batch_size=5,
train_iteration_mode='sequential',
monitoring_dataset=None,
termination_criterion=termination_criterion,
update_callbacks=None,
init_momentum=None,
set_batch_size=False)
algorithm.setup(dataset=dataset, model=model)
algorithm.train(dataset)
assert all(visited)
def get_layer_trainer_sgd_rbm(layer, trainset):
train_algo = SGD(
learning_rate = 1e-1,
batch_size = 5,
#"batches_per_iter" : 2000,
monitoring_batches = 20,
monitoring_dataset = trainset,
cost = SMD(corruptor=GaussianCorruptor(stdev=0.4)),
termination_criterion = EpochCounter(max_epochs=MAX_EPOCHS),
# another option:
# MonitorBasedTermCrit(prop_decrease=0.01, N=10),
)
model = layer
callbacks = [MonitorBasedLRAdjuster(), ModelSaver()]
return Train(model = model, algorithm = train_algo,
callbacks = callbacks, dataset = trainset)
def get_layer_trainer_sgd_autoencoder(layer, trainset):
# configs on sgd
train_algo = SGD(
learning_rate=0.1,
cost=MeanSquaredReconstructionError(),
batch_size=10,
monitoring_batches=10,
monitoring_dataset=trainset,
termination_criterion=EpochCounter(max_epochs=MAX_EPOCHS_UNSUPERVISED),
update_callbacks=None)
model = layer
extensions = None
return Train(model=model,
algorithm=train_algo,
extensions=extensions,
dataset=trainset)
def get_layer_trainer_sparse_denoising_autoencoder(layer, trainset):
train_algo = SGD(
learning_rate = 0.1,
cost = SampledMeanSquaredReconstructionError(),
batch_size = 10,
monitoring_batches = 10,
monitoring_dataset = trainset,
termination_criterion = EpochCounter(max_epochs=MAX_EPOCHS),
update_callbacks = None
)
model = layer
callbacks = [ModelSaver()]
return Train(model = model,
algorithm = train_algo,
callbacks = callbacks,
dataset = trainset)
def get_layer_trainer_sgd_rbm(layer, trainset):
train_algo = SGD(
learning_rate=1e-1,
batch_size=5,
#"batches_per_iter" : 2000,
monitoring_batches=20,
monitoring_dataset=trainset,
cost=SMD(corruptor=GaussianCorruptor(stdev=0.4)),
termination_criterion=EpochCounter(max_epochs=MAX_EPOCHS_UNSUPERVISED),
)
model = layer
extensions = [MonitorBasedLRAdjuster()]
return Train(model=model,
algorithm=train_algo,
save_path='grbm.pkl',
save_freq=1,
extensions=extensions,
dataset=trainset)
def get_layer_trainer_logistic(layer, trainset):
# configs on sgd
config = {
'learning_rate': 0.1,
'cost': CrossEntropy(),
'batch_size': 10,
'monitoring_batches': 10,
'monitoring_dataset': None,
'termination_criterion': EpochCounter(max_epochs=10),
'update_callbacks': None
}
train_algo = ExhaustiveSGD(**config)
model = layer
callbacks = None
return Train(model=model,
dataset=trainset,
algorithm=train_algo,
callbacks=callbacks)
def get_layer_trainer_logistic(layer, trainset):
# configs on sgd
config = {
'learning_rate': 0.1,
'cost': Default(),
'batch_size': 10,
'monitoring_batches': 10,
'monitoring_dataset': trainset,
'termination_criterion':
EpochCounter(max_epochs=MAX_EPOCHS_SUPERVISED),
'update_callbacks': None
}
train_algo = SGD(**config)
model = layer
return Train(model=model,
dataset=trainset,
algorithm=train_algo,
extensions=None)
def test_determinism():
# Verifies that running SGD twice results in the same examples getting visited
# in the same order
for mode in _iteration_schemes:
dim = 1
batch_size = 3
num_batches = 5
m = num_batches * batch_size
dataset = ArangeDataset(m)
model = SoftmaxModel(dim)
learning_rate = 1e-3
batch_size = 5
visited = [[-1] * m]
def visit(X):
mx = max(visited[0])
counter = mx + 1
for i in X[:, 0]:
i = int(i)
assert visited[0][i] == -1
visited[0][i] = counter
counter += 1
cost = CallbackCost(visit)
# We need to include this so the test actually stops running at some point
termination_criterion = EpochCounter(5)
def run_algorithm():
unsupported_modes = ['random_slice', 'random_uniform']
algorithm = SGD(learning_rate,
cost,
batch_size=5,
train_iteration_mode=mode,
monitoring_dataset=None,
termination_criterion=termination_criterion,
update_callbacks=None,
init_momentum=None,
set_batch_size=False)
algorithm.setup(dataset=dataset, model=model)
raised = False
try:
algorithm.train(dataset)
except ValueError:
print mode
assert mode in unsupported_modes
raised = True
if mode in unsupported_modes:
assert raised
return True
return False
if run_algorithm():
continue
visited.insert(0, [-1] * m)
del model.monitor
run_algorithm()
for v in visited:
assert len(v) == m
for elem in range(m):
assert elem in v
assert len(visited) == 2
print visited[0]
print visited[1]
assert np.all(np.asarray(visited[0]) == np.asarray(visited[1]))
def test_sgd_topo():
# tests that we can run the sgd algorithm
# on data with topology
# does not test for correctness at all, just
# that the algorithm runs without dying
rows = 3
cols = 4
channels = 2
dim = rows * cols * channels
m = 10
rng = np.random.RandomState([25, 9, 2012])
X = rng.randn(m, rows, cols, channels)
idx = rng.randint(0, dim, (m, ))
Y = np.zeros((m, dim))
for i in xrange(m):
Y[i, idx[i]] = 1
dataset = DenseDesignMatrix(topo_view=X, y=Y)
m = 15
X = rng.randn(m, rows, cols, channels)
idx = rng.randint(0, dim, (m, ))
Y = np.zeros((m, dim))
for i in xrange(m):
Y[i, idx[i]] = 1
# including a monitoring datasets lets us test that
# the monitor works with supervised data
monitoring_dataset = DenseDesignMatrix(topo_view=X, y=Y)
model = TopoSoftmaxModel(rows, cols, channels)
learning_rate = 1e-3
batch_size = 5
cost = CrossEntropy()
# We need to include this so the test actually stops running at some point
termination_criterion = EpochCounter(5)
algorithm = SGD(learning_rate,
cost,
batch_size=5,
monitoring_batches=3,
monitoring_dataset=monitoring_dataset,
termination_criterion=termination_criterion,
update_callbacks=None,
init_momentum=None,
set_batch_size=False)
train = Train(dataset,
model,
algorithm,
save_path=None,
save_freq=0,
callbacks=None)
train.main_loop()
def main_train(work_dir="../results/avicenna/",
corruption_level=0.3,
nvis=75,
nhid=600,
tied_weights=True,
act_enc="sigmoid",
act_dec=None,
max_epochs=2,
learning_rate=0.001,
batch_size=20,
monitoring_batches=5,
save_freq=1,
n_components_trans_pca=7):
conf = {
'corruption_level': corruption_level,
'nvis': nvis,
'nhid': nhid,
'tied_weights': tied_weights,
'act_enc': act_enc,
'act_dec': act_dec,
'max_epochs': max_epochs,
'learning_rate': learning_rate,
'batch_size': batch_size,
'monitoring_batches': monitoring_batches,
'save_freq': save_freq,
'n_components_trans_pca': n_components_trans_pca
}
start = time.clock()
############### TRAIN THE DAE
train_file = work_dir + "train_pca" + str(conf['nvis']) + ".npy"
save_path = work_dir + "train_pca" + str(conf['nvis']) + "_dae" + str(
conf['nhid']) + "_model.pkl"
trainset = NpyDataset(file=train_file)
trainset.yaml_src = 'script'
corruptor = BinomialCorruptor(corruption_level=conf['corruption_level'])
dae = DenoisingAutoencoder(nvis=conf['nvis'],
nhid=conf['nhid'],
tied_weights=conf['tied_weights'],
corruptor=corruptor,
act_enc=conf['act_enc'],
act_dec=conf['act_dec'])
cost = MeanSquaredReconstructionError()
termination_criterion = EpochCounter(max_epochs=conf['max_epochs'])
algorithm = UnsupervisedExhaustiveSGD(
learning_rate=conf['learning_rate'],
batch_size=conf['batch_size'],
monitoring_batches=conf['monitoring_batches'],
monitoring_dataset=trainset,
cost=cost,
termination_criterion=termination_criterion)
train_obj = Train(dataset=trainset,
model=dae,
algorithm=algorithm,
save_freq=conf['save_freq'],
save_path=save_path)
train_obj.main_loop()
############### APPLY THE MODEL ON THE TRAIN DATASET
print("Applying the model on the train dataset...")
model = load(save_path)
save_train_path = work_dir + "train_pca" + str(
conf['nvis']) + "_dae" + str(conf['nhid']) + ".npy"
dump_obj = FeatureDump(encoder=model,
dataset=trainset,
path=save_train_path)
dump_obj.main_loop()
############### APPLY THE MODEL ON THE VALID DATASET
print("Applying the model on the valid dataset...")
valid_file = work_dir + "valid_pca" + str(conf['nvis']) + ".npy"
validset = NpyDataset(file=valid_file)
validset.yaml_src = 'script'
save_valid_path = work_dir + "valid_pca" + str(
conf['nvis']) + "_dae" + str(conf['nhid']) + ".npy"
dump_obj = FeatureDump(encoder=model,
dataset=validset,
path=save_valid_path)
dump_obj.main_loop()
############### APPLY THE MODEL ON THE TEST DATASET
print("Applying the model on the test dataset...")
test_file = work_dir + "test_pca" + str(conf['nvis']) + ".npy"
testset = NpyDataset(file=test_file)
testset.yaml_src = 'script'
save_test_path = work_dir + "test_pca" + str(conf['nvis']) + "_dae" + str(
conf['nhid']) + ".npy"
dump_obj = FeatureDump(encoder=model, dataset=testset, path=save_test_path)
dump_obj.main_loop()
############### COMPUTE THE ALC SCORE ON VALIDATION SET
valid_data = ift6266h12.load_npy(save_valid_path)
label_data = ift6266h12.load_npy(
'/data/lisa/data/UTLC/numpy_data/avicenna_valid_y.npy')
alc_1 = score(valid_data, label_data)
############### APPLY THE TRANSDUCTIVE PCA
test_data = ift6266h12.load_npy(save_test_path)
trans_pca = PCA(n_components=conf['n_components_trans_pca'])
final_valid = trans_pca.fit_transform(valid_data)
final_test = trans_pca.fit_transform(test_data)
save_valid_path = work_dir + "valid_pca" + str(
conf['nvis']) + "_dae" + str(conf['nhid']) + "_tpca" + str(
conf['n_components_trans_pca']) + ".npy"
save_test_path = work_dir + "test_pca" + str(conf['nvis']) + "_dae" + str(
conf['nhid']) + "_tpca" + str(conf['n_components_trans_pca']) + ".npy"
np.save(save_valid_path, final_valid)
np.save(save_test_path, final_test)
############### COMPUTE THE NEW ALC SCORE ON VALIDATION SET
alc_2 = score(final_valid, label_data)
############### OUTPUT AND RETURN THE RESULTS
timeSpent = ((time.clock() - start) / 60.)
print 'FINAL RESULTS (PCA-' + str(conf['nvis']) + ' DAE-' + str(conf['nhid']) + ' TransPCA-' + str(conf['n_components_trans_pca']) + ') ALC after DAE: ', alc_1, ' FINAL ALC: ', alc_2, \
' Computed in %5.2f min' % (timeSpent)
return timeSpent, alc_1, alc_2