class UseSDAE(AdaClassifier):
def __init__(self, param):
self.batch_size = param['batch_size']
self.iterations = param['iterations']
self.in_size = param['in_size']
self.out_size = param['out_size']
self.hid_sizes = param['hid_sizes']
self.learning_rate = param['learning_rate']
self.pre_epochs = param['pre_epochs']
self.finetune_epochs = param['fine_epochs']
self.lam = param['lam']
self.act = param['act']
self.sdae = SDAE(self.in_size, self.out_size, self.hid_sizes,
self.batch_size, self.learning_rate, self.lam,
self.act, self.iterations)
self.sdae.process()
self.theano_tr_ids, self.tr_pred, self.tr_act = [], [], []
def train(self, tr_all, v_all, weights):
from math import ceil
tr_ids, tr_x, tr_y = tr_all
v_ids, v_x, v_y = v_all
weights_shr = shared(value=np.asarray(weights, dtype=config.floatX),
borrow=True)
def get_shared_data(data_xy):
data_x, data_y = data_xy
shared_x = shared(value=np.asarray(data_x, dtype=config.floatX),
borrow=True)
shared_y = shared(value=np.asarray(data_y, dtype=config.floatX),
borrow=True)
return shared_x, T.cast(shared_y, 'int32')
train = get_shared_data((tr_x, tr_y))
valid = get_shared_data((v_x, v_y))
n_train_batches = ceil(train[0].get_value(borrow=True).shape[0] /
self.batch_size)
n_valid_batches = ceil(valid[0].get_value(borrow=True).shape[0] /
self.batch_size)
pretrain_func = self.sdae.pre_train(train[0], train[1])
finetune_func = self.sdae.fine_tune(train[0], train[1], weights_shr)
my_valid_id_tensor = shared(value=np.asarray(v_ids,
dtype=config.floatX),
borrow=True)
my_valid_id_int_tensor = T.cast(my_valid_id_tensor, 'int32')
validate_func = self.sdae.validate(valid[0], valid[1],
my_valid_id_int_tensor)
my_train_id_tensor = shared(value=np.asarray(tr_ids,
dtype=config.floatX),
borrow=True)
my_train_id_int_tensor = T.cast(my_train_id_tensor, 'int32')
tr_validate_func = self.sdae.validate(train[0], train[1],
my_train_id_int_tensor)
for epoch in range(self.pre_epochs):
pre_train_cost = []
for b in range(n_train_batches):
pre_train_cost.append(pretrain_func(b))
print('Pretrain cost ', '(epoch ', epoch, '): ',
np.mean(pre_train_cost))
min_valid_err = np.inf
for epoch in range(self.finetune_epochs):
from random import shuffle
finetune_cost = []
b_idx = [i for i in range(0, n_train_batches)]
shuffle(b_idx)
for b in b_idx:
cost = finetune_func(b)
finetune_cost.append(cost)
if epoch % 10 == 0:
print('Finetune cost: ', '(epoch ', epoch, '): ',
np.mean(finetune_cost))
valid_cost = []
for b in range(n_valid_batches):
ids, errs, pred_y, act_y = validate_func(b)
valid_cost.append(errs)
curr_valid_err = np.mean(valid_cost)
print('Validation error: ', np.mean(valid_cost))
if curr_valid_err * 0.99 > min_valid_err:
break
elif curr_valid_err < min_valid_err:
min_valid_err = curr_valid_err
for b in range(n_train_batches):
t_ids, t_errs, t_pred_y, t_act_y = tr_validate_func(b)
self.theano_tr_ids.extend(t_ids)
self.tr_pred.extend([np.argmax(arr) for arr in t_pred_y])
self.tr_act.extend([np.argmax(arr) for arr in t_act_y])
def get_labels(self):
return self.theano_tr_ids, self.tr_pred, self.tr_act
def get_test_results(self, ts_data):
ts_ids, test_x = ts_data
test_x = shared(value=np.asarray(test_x, dtype=config.floatX),
borrow=True)
test_func = self.sdae.test(test_x)
n_test_batches = (test_x.get_value(borrow=True).shape[0])
test_out_probs = []
for b in range(n_test_batches):
cls, probs = test_func(b)
test_out_probs.append(probs[0])
return ts_ids, test_out_probs
class MySDAE(object):
def __init__(self, param):
self.batch_size = param['batch_size']
self.iterations = param['iterations']
self.in_size = param['in_size']
self.out_size = param['out_size']
self.hid_sizes = param['hid_sizes']
self.learning_rate = param['learning_rate']
self.pre_epochs = param['pre_epochs']
self.finetune_epochs = param['fine_epochs']
self.lam = param['lam']
self.act = param['act']
self.denoise = param['denoise']
self.corr_level = param['corr_level']
self.sdae = SDAE(self.in_size, self.out_size, self.hid_sizes,
self.batch_size, self.learning_rate, self.lam,
self.act, self.iterations, self.denoise,
self.corr_level)
self.sdae.process()
self.theano_tr_ids, self.tr_pred, self.tr_act = [], [], []
def pretrain(self, tr_all, v_all, ts_all, weights):
tr_ids, tr_x, tr_y = tr_all
v_ids, v_x, v_y = v_all
ts_ids, ts_x = ts_all
all_x = []
all_x.extend(tr_x.get_value())
all_x.extend(v_x.get_value())
all_x.extend(ts_x.get_value())
all_theano_x = shared(value=np.asarray(all_x, dtype=config.floatX),
borrow=True)
if weights is not None:
use_layer_wise_w = False
if isinstance(weights[0], list):
use_layer_wise_w = True
th_all_weights = []
for w in weights:
w_tmp = np.asarray(w, dtype=config.floatX).reshape(-1, 1)
w_tmp = np.append(
w_tmp,
np.asarray([
1 for _ in range(v_x.get_value().shape[0])
]).reshape(-1, 1),
axis=0)
w_tmp = np.append(
w_tmp,
np.asarray([
1 for _ in range(ts_x.get_value().shape[0])
]).reshape(-1, 1),
axis=0)
th_all_weights.append(shared(w_tmp))
else:
all_weights = np.asarray(tr_weights)
all_weights = np.append(
all_weights,
np.asarray([1 for _ in range(v_x.get_value().shape[0])
]).reshape(-1, 1),
axis=0)
all_weights = np.append(
all_weights,
np.asarray([1 for _ in range(ts_x.get_value().shape[0])
]).reshape(-1, 1),
axis=0)
th_all_weights = shared(all_weights)
else:
th_all_weights = None
n_pretrain_batches = ceil(
all_theano_x.get_value(borrow=True).shape[0] / self.batch_size)
pretrain_func = self.sdae.pre_train(all_theano_x, th_all_weights)
for epoch in range(self.pre_epochs):
pre_train_cost = []
b_indices = [i for i in range(n_pretrain_batches)]
np.random.shuffle(b_indices)
for b in b_indices:
pre_train_cost.append(pretrain_func(b))
print('Pretrain cost (Layer-wise) ', '(epoch ', epoch, '): ',
np.mean(pre_train_cost))
def full_pretrain(self, tr_all, v_all, ts_all, weights):
tr_ids, tr_x, tr_y = tr_all
v_ids, v_x, v_y = v_all
ts_ids, ts_x = ts_all
all_x = []
all_x.extend(tr_x.get_value())
all_x.extend(v_x.get_value())
all_x.extend(ts_x.get_value())
all_theano_x = shared(value=np.asarray(all_x, dtype=config.floatX),
borrow=True)
if weights is not None:
all_weights = np.asarray(tr_weights)
all_weights = np.append(
all_weights,
np.asarray([1 for _ in range(v_x.get_value().shape[0])
]).reshape(-1, 1),
axis=0)
all_weights = np.append(
all_weights,
np.asarray([1 for _ in range(ts_x.get_value().shape[0])
]).reshape(-1, 1),
axis=0)
th_all_weights = shared(all_weights)
else:
th_all_weights = None
n_pretrain_batches = ceil(
all_theano_x.get_value(borrow=True).shape[0] / self.batch_size)
full_pretrain_func = self.sdae.full_pretrain(all_theano_x,
th_all_weights)
for epoch in range(self.pre_epochs // 2):
full_pre_cost = []
b_indices = [i for i in range(n_pretrain_batches)]
np.random.shuffle(b_indices)
for b in b_indices:
full_pre_cost.append(full_pretrain_func(b))
print('Pretrain cost (Full) ', '(epoch ', epoch, '): ',
np.mean(full_pre_cost))
def finetune(self, tr_all, v_all, weights):
tr_ids, tr_x, tr_y = tr_all
v_ids, v_x, v_y = v_all
if weights is not None:
th_weights = shared(np.array(weights, dtype=config.floatX))
else:
th_weights = None
finetune_func = self.sdae.fine_tune(tr_x, tr_y, th_weights)
n_pretrain_batches = ceil(
tr_x.get_value(borrow=True).shape[0] / self.batch_size)
validate_func = self.sdae.validate(v_x, v_y, v_ids)
n_valid_batches = ceil(
v_x.get_value(borrow=True).shape[0] / self.batch_size)
min_valid_err = np.inf
for epoch in range(self.finetune_epochs):
b_indices = [i for i in range(n_pretrain_batches)]
np.random.shuffle(b_indices)
finetune_cost = []
for b in b_indices:
finetune_cost.append(finetune_func(b))
if epoch % 1 == 0:
print('Finetune cost: ', '(epoch ', epoch, '): ',
np.mean(finetune_cost))
valid_cost = []
for b in range(n_valid_batches):
ids, errs, pred_y, act_y = validate_func(b)
valid_cost.append(errs)
curr_valid_err = np.mean(valid_cost)
print('Validation error: ', np.mean(valid_cost))
if curr_valid_err * 0.99 > min_valid_err:
break
elif curr_valid_err < min_valid_err:
min_valid_err = curr_valid_err
print('Fintune with logreg (epoch ', epoch, '): ',
np.mean(finetune_cost))
def get_features(self, tr_all, v_all, ts_all, layer_idx):
tr_ids, tr_x, tr_y = tr_all
v_ids, v_x, v_y = v_all
ts_ids, ts_x = ts_all
tr_feature_func = self.sdae.get_features(tr_x, tr_y, tr_ids, layer_idx,
False)
v_features_func = self.sdae.get_features(v_x, v_y, v_ids, layer_idx,
False)
ts_features_func = self.sdae.get_features(ts_x, None, ts_ids,
layer_idx, True)
n_train_batches = ceil(
tr_x.get_value(borrow=True).shape[0] / self.batch_size)
n_valid_batches = ceil(
v_x.get_value(borrow=True).shape[0] / self.batch_size)
n_test_batches = ceil(
ts_x.get_value(borrow=True).shape[0] / self.batch_size)
all_tr_features = []
all_tr_outputs = []
all_ts_features = []
for b in range(n_train_batches):
features, y, ids = tr_feature_func(b)
temp = np.concatenate((np.reshape(ids,
(ids.shape[0], 1)), features),
axis=1)
all_tr_features.extend(temp.tolist())
all_tr_outputs.extend(y)
print('Size train features: ', len(all_tr_features), ' x ',
len(all_tr_features[0]))
for b in range(n_valid_batches):
features, y, ids = v_features_func(b)
temp = [ids]
temp = np.concatenate((np.reshape(ids,
(ids.shape[0], 1)), features),
axis=1)
all_tr_features.extend(temp.tolist())
all_tr_outputs.extend(y)
print('Size train+valid features: ', len(all_tr_features), ' x ',
len(all_tr_features[0]))
for b in range(n_test_batches):
features, y, ids = ts_features_func(b)
temp = np.concatenate((np.reshape(ids,
(ids.shape[0], 1)), features),
axis=1)
all_ts_features.extend(temp.tolist())
print('Size test features: ', len(all_ts_features), ' x ',
len(all_ts_features[0]))
return (all_tr_features, all_tr_outputs), all_ts_features
def get_labels(self):
return self.theano_tr_ids, self.tr_pred, self.tr_act
def get_test_results(self, ts_data):
ts_ids, test_x = ts_data
test_x = shared(value=np.asarray(test_x, dtype=config.floatX),
borrow=True)
test_func = self.sdae.test(test_x)
n_test_batches = (test_x.get_value(borrow=True).shape[0])
test_out_probs = []
for b in range(n_test_batches):
cls, probs = test_func(b)
test_out_probs.append(probs[0])
return ts_ids, test_out_probs
class UseSDAE(AdaClassifier):
def __init__(self,param):
self.batch_size = param['batch_size']
self.iterations = param['iterations']
self.in_size = param['in_size']
self.out_size = param['out_size']
self.hid_sizes = param['hid_sizes']
self.learning_rate = param['learning_rate']
self.pre_epochs = param['pre_epochs']
self.finetune_epochs = param['fine_epochs']
self.lam = param['lam']
self.act = param['act']
self.sdae = SDAE(self.in_size,self.out_size,self.hid_sizes,self.batch_size,self.learning_rate,self.lam,self.act,self.iterations)
self.sdae.process()
self.theano_tr_ids, self.tr_pred, self.tr_act = [],[],[]
def train(self,tr_all,v_all,weights):
from math import ceil
tr_ids,tr_x,tr_y = tr_all
v_ids,v_x,v_y = v_all
weights_shr = shared(value=np.asarray(weights,dtype=config.floatX),borrow=True)
def get_shared_data(data_xy):
data_x,data_y = data_xy
shared_x = shared(value=np.asarray(data_x,dtype=config.floatX),borrow=True)
shared_y = shared(value=np.asarray(data_y,dtype=config.floatX),borrow=True)
return shared_x,T.cast(shared_y,'int32')
train = get_shared_data((tr_x,tr_y))
valid = get_shared_data((v_x,v_y))
n_train_batches = ceil(train[0].get_value(borrow=True).shape[0] / self.batch_size)
n_valid_batches = ceil(valid[0].get_value(borrow=True).shape[0] / self.batch_size)
pretrain_func = self.sdae.pre_train(train[0],train[1])
finetune_func = self.sdae.fine_tune(train[0],train[1],weights_shr)
my_valid_id_tensor = shared(value=np.asarray(v_ids,dtype=config.floatX),borrow=True)
my_valid_id_int_tensor = T.cast(my_valid_id_tensor,'int32')
validate_func = self.sdae.validate(valid[0],valid[1],my_valid_id_int_tensor)
my_train_id_tensor = shared(value=np.asarray(tr_ids,dtype=config.floatX),borrow=True)
my_train_id_int_tensor = T.cast(my_train_id_tensor,'int32')
tr_validate_func = self.sdae.validate(train[0],train[1],my_train_id_int_tensor)
for epoch in range(self.pre_epochs):
pre_train_cost = []
for b in range(n_train_batches):
pre_train_cost.append(pretrain_func(b))
print('Pretrain cost ','(epoch ', epoch,'): ',np.mean(pre_train_cost))
min_valid_err = np.inf
for epoch in range(self.finetune_epochs):
from random import shuffle
finetune_cost = []
b_idx =[i for i in range(0,n_train_batches)]
shuffle(b_idx)
for b in b_idx:
cost = finetune_func(b)
finetune_cost.append(cost)
if epoch%10==0:
print('Finetune cost: ','(epoch ', epoch,'): ',np.mean(finetune_cost))
valid_cost = []
for b in range(n_valid_batches):
ids,errs,pred_y,act_y = validate_func(b)
valid_cost.append(errs)
curr_valid_err = np.mean(valid_cost)
print('Validation error: ',np.mean(valid_cost))
if curr_valid_err*0.99>min_valid_err:
break
elif curr_valid_err<min_valid_err:
min_valid_err = curr_valid_err
for b in range(n_train_batches):
t_ids,t_errs,t_pred_y,t_act_y = tr_validate_func(b)
self.theano_tr_ids.extend(t_ids)
self.tr_pred.extend([np.argmax(arr) for arr in t_pred_y])
self.tr_act.extend([np.argmax(arr) for arr in t_act_y])
def get_labels(self):
return self.theano_tr_ids,self.tr_pred,self.tr_act
def get_test_results(self,ts_data):
ts_ids, test_x = ts_data
test_x = shared(value=np.asarray(test_x,dtype=config.floatX),borrow=True)
test_func = self.sdae.test(test_x)
n_test_batches = (test_x.get_value(borrow=True).shape[0])
test_out_probs = []
for b in range(n_test_batches):
cls,probs = test_func(b)
test_out_probs.append(probs[0])
return ts_ids,test_out_probs
class MySDAE(object):
def __init__(self,param):
self.batch_size = param['batch_size']
self.iterations = param['iterations']
self.in_size = param['in_size']
self.out_size = param['out_size']
self.hid_sizes = param['hid_sizes']
self.learning_rate = param['learning_rate']
self.pre_epochs = param['pre_epochs']
self.finetune_epochs = param['fine_epochs']
self.lam = param['lam']
self.act = param['act']
self.denoise = param['denoise']
self.corr_level = param['corr_level']
self.sdae = SDAE(self.in_size,self.out_size,self.hid_sizes,self.batch_size,self.learning_rate,self.lam,self.act,self.iterations,self.denoise,self.corr_level)
self.sdae.process()
self.theano_tr_ids, self.tr_pred, self.tr_act = [],[],[]
def pretrain(self,tr_all,v_all,ts_all,weights):
tr_ids,tr_x,tr_y = tr_all
v_ids,v_x,v_y = v_all
ts_ids,ts_x = ts_all
all_x = []
all_x.extend(tr_x.get_value())
all_x.extend(v_x.get_value())
all_x.extend(ts_x.get_value())
all_theano_x = shared(value=np.asarray(all_x,dtype=config.floatX),borrow=True)
if weights is not None:
use_layer_wise_w = False
if isinstance(weights[0],list):
use_layer_wise_w = True
th_all_weights = []
for w in weights:
w_tmp = np.asarray(w,dtype=config.floatX).reshape(-1,1)
w_tmp = np.append(w_tmp, np.asarray([1 for _ in range(v_x.get_value().shape[0])]).reshape(-1,1),axis=0)
w_tmp = np.append(w_tmp, np.asarray([1 for _ in range(ts_x.get_value().shape[0])]).reshape(-1,1),axis=0)
th_all_weights.append(shared(w_tmp))
else:
all_weights = np.asarray(tr_weights)
all_weights = np.append(all_weights, np.asarray([1 for _ in range(v_x.get_value().shape[0])]).reshape(-1,1),axis=0)
all_weights = np.append(all_weights, np.asarray([1 for _ in range(ts_x.get_value().shape[0])]).reshape(-1,1),axis=0)
th_all_weights = shared(all_weights)
else:
th_all_weights = None
n_pretrain_batches = ceil(all_theano_x.get_value(borrow=True).shape[0] / self.batch_size)
pretrain_func = self.sdae.pre_train(all_theano_x,th_all_weights)
for epoch in range(self.pre_epochs):
pre_train_cost = []
b_indices = [i for i in range(n_pretrain_batches)]
np.random.shuffle(b_indices)
for b in b_indices:
pre_train_cost.append(pretrain_func(b))
print('Pretrain cost (Layer-wise) ','(epoch ', epoch,'): ',np.mean(pre_train_cost))
def full_pretrain(self,tr_all,v_all,ts_all,weights):
tr_ids,tr_x,tr_y = tr_all
v_ids,v_x,v_y = v_all
ts_ids,ts_x = ts_all
all_x = []
all_x.extend(tr_x.get_value())
all_x.extend(v_x.get_value())
all_x.extend(ts_x.get_value())
all_theano_x = shared(value=np.asarray(all_x,dtype=config.floatX),borrow=True)
if weights is not None:
all_weights = np.asarray(tr_weights)
all_weights = np.append(all_weights,np.asarray([1 for _ in range(v_x.get_value().shape[0])]).reshape(-1,1),axis=0)
all_weights = np.append(all_weights,np.asarray([1 for _ in range(ts_x.get_value().shape[0])]).reshape(-1,1),axis=0)
th_all_weights = shared(all_weights)
else:
th_all_weights = None
n_pretrain_batches = ceil(all_theano_x.get_value(borrow=True).shape[0] / self.batch_size)
full_pretrain_func = self.sdae.full_pretrain(all_theano_x,th_all_weights)
for epoch in range(self.pre_epochs//2):
full_pre_cost = []
b_indices = [i for i in range(n_pretrain_batches)]
np.random.shuffle(b_indices)
for b in b_indices:
full_pre_cost.append(full_pretrain_func(b))
print('Pretrain cost (Full) ','(epoch ', epoch,'): ',np.mean(full_pre_cost))
def finetune(self,tr_all,v_all,weights):
tr_ids,tr_x,tr_y = tr_all
v_ids,v_x,v_y = v_all
if weights is not None:
th_weights = shared(np.array(weights,dtype=config.floatX))
else:
th_weights = None
finetune_func = self.sdae.fine_tune(tr_x,tr_y,th_weights)
n_pretrain_batches = ceil(tr_x.get_value(borrow=True).shape[0] / self.batch_size)
validate_func = self.sdae.validate(v_x,v_y,v_ids)
n_valid_batches = ceil(v_x.get_value(borrow=True).shape[0] / self.batch_size)
min_valid_err = np.inf
for epoch in range(self.finetune_epochs):
b_indices = [i for i in range(n_pretrain_batches)]
np.random.shuffle(b_indices)
finetune_cost = []
for b in b_indices:
finetune_cost.append(finetune_func(b))
if epoch%1==0:
print('Finetune cost: ','(epoch ', epoch,'): ',np.mean(finetune_cost))
valid_cost = []
for b in range(n_valid_batches):
ids,errs,pred_y,act_y = validate_func(b)
valid_cost.append(errs)
curr_valid_err = np.mean(valid_cost)
print('Validation error: ',np.mean(valid_cost))
if curr_valid_err*0.99>min_valid_err:
break
elif curr_valid_err<min_valid_err:
min_valid_err = curr_valid_err
print('Fintune with logreg (epoch ',epoch,'): ',np.mean(finetune_cost))
def get_features(self,tr_all,v_all,ts_all,layer_idx):
tr_ids,tr_x,tr_y = tr_all
v_ids,v_x,v_y = v_all
ts_ids,ts_x = ts_all
tr_feature_func = self.sdae.get_features(tr_x,tr_y,tr_ids,layer_idx,False)
v_features_func = self.sdae.get_features(v_x,v_y,v_ids,layer_idx,False)
ts_features_func = self.sdae.get_features(ts_x,None,ts_ids,layer_idx,True)
n_train_batches = ceil(tr_x.get_value(borrow=True).shape[0] / self.batch_size)
n_valid_batches = ceil(v_x.get_value(borrow=True).shape[0] / self.batch_size)
n_test_batches = ceil(ts_x.get_value(borrow=True).shape[0] / self.batch_size)
all_tr_features = []
all_tr_outputs = []
all_ts_features = []
for b in range(n_train_batches):
features, y, ids = tr_feature_func(b)
temp = np.concatenate((np.reshape(ids,(ids.shape[0],1)),features),axis=1)
all_tr_features.extend(temp.tolist())
all_tr_outputs.extend(y)
print('Size train features: ',len(all_tr_features),' x ',len(all_tr_features[0]))
for b in range(n_valid_batches):
features, y, ids = v_features_func(b)
temp = [ids]
temp = np.concatenate((np.reshape(ids,(ids.shape[0],1)),features),axis=1)
all_tr_features.extend(temp.tolist())
all_tr_outputs.extend(y)
print('Size train+valid features: ',len(all_tr_features),' x ',len(all_tr_features[0]))
for b in range(n_test_batches):
features, y, ids = ts_features_func(b)
temp = np.concatenate((np.reshape(ids,(ids.shape[0],1)),features),axis=1)
all_ts_features.extend(temp.tolist())
print('Size test features: ',len(all_ts_features),' x ',len(all_ts_features[0]))
return (all_tr_features,all_tr_outputs),all_ts_features
def get_labels(self):
return self.theano_tr_ids,self.tr_pred,self.tr_act
def get_test_results(self,ts_data):
ts_ids, test_x = ts_data
test_x = shared(value=np.asarray(test_x,dtype=config.floatX),borrow=True)
test_func = self.sdae.test(test_x)
n_test_batches = (test_x.get_value(borrow=True).shape[0])
test_out_probs = []
for b in range(n_test_batches):
cls,probs = test_func(b)
test_out_probs.append(probs[0])
return ts_ids,test_out_probs