def _create_layer(self, name, layer, irange):
if isinstance(layer, Convolution):
return self._create_convolution_layer(name, layer, irange)
if layer.type == "Rectifier":
self._check_layer(layer, ['units'])
return mlp.RectifiedLinear(layer_name=name,
dim=layer.units,
irange=irange)
if layer.type == "Sigmoid":
self._check_layer(layer, ['units'])
return mlp.Sigmoid(layer_name=name, dim=layer.units, irange=irange)
if layer.type == "Tanh":
self._check_layer(layer, ['units'])
return mlp.Tanh(layer_name=name, dim=layer.units, irange=irange)
if layer.type == "Maxout":
self._check_layer(layer, ['units', 'pieces'])
return maxout.Maxout(layer_name=name,
num_units=layer.units,
num_pieces=layer.pieces,
irange=irange)
if layer.type == "Linear":
self._check_layer(layer, ['units'])
return mlp.Linear(layer_name=layer.name,
dim=layer.units,
irange=irange)
if layer.type == "Gaussian":
self._check_layer(layer, ['units'])
return mlp.LinearGaussian(layer_name=layer.name,
init_beta=0.1,
min_beta=0.001,
max_beta=1000,
beta_lr_scale=None,
dim=layer.units,
irange=irange)
if layer.type == "Softmax":
self._check_layer(layer, ['units'])
return mlp.Softmax(layer_name=layer.name,
n_classes=layer.units,
irange=irange)
def init_train():
# Initialize train object.
idpath = os.path.splitext(
os.path.abspath(__file__))[0] # ID for output files.
save_path = idpath + '.pkl'
# Dataset
#seed = 42
benchmark = 1
#derived_feat, nvis = False, 21
derived_feat, nvis = True, 28
#derived_feat, nvis = 'only', 7
dataset_train = pylearn2.datasets.physics.PHYSICS(
which_set='train', benchmark=benchmark, derived_feat=derived_feat)
dataset_train_monitor = pylearn2.datasets.physics.PHYSICS(
which_set='train',
benchmark=benchmark,
derived_feat=derived_feat,
start=0,
stop=100000)
dataset_valid = pylearn2.datasets.physics.PHYSICS(
which_set='valid', benchmark=benchmark, derived_feat=derived_feat)
dataset_test = pylearn2.datasets.physics.PHYSICS(which_set='test',
benchmark=benchmark,
derived_feat=derived_feat)
# Parameters
momentum_saturate = 200
# Model
model = pylearn2.models.mlp.MLP(
layers=[
mlp.Tanh(layer_name='h0', dim=300, istdev=.1),
#istdev=.05 for any intermediates
mlp.Sigmoid(layer_name='y', dim=1, istdev=.001)
],
nvis=nvis)
# Algorithm
algorithm = pylearn2.training_algorithms.sgd.SGD(
batch_size=100, # If changed, change learning rate!
learning_rate=
.05, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize.
init_momentum=.9,
monitoring_dataset={
'train': dataset_train_monitor,
'valid': dataset_valid,
'test': dataset_test
},
termination_criterion=pylearn2.termination_criteria.Or(criteria=[
pylearn2.termination_criteria.MonitorBased(
channel_name="valid_objective", prop_decrease=0.00001, N=10),
pylearn2.termination_criteria.EpochCounter(
max_epochs=momentum_saturate)
]),
cost=pylearn2.costs.cost.SumOfCosts(costs=[
pylearn2.costs.mlp.Default(),
pylearn2.costs.mlp.WeightDecay(coeffs=[.00001, .00001])
]),
update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay(
decay_factor=
1.0000002, # Decreases by this factor every batch. (1/(1.000001^8000)^100
min_lr=.000001))
# Extensions
extensions = [
#pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path)
pylearn2.training_algorithms.sgd.MomentumAdjustor(
start=0,
saturate=momentum_saturate,
final_momentum=.99 # Dropout=.5->.99 over 500 epochs.
)
]
# Train
train = pylearn2.train.Train(dataset=dataset_train,
model=model,
algorithm=algorithm,
extensions=extensions,
save_path=save_path,
save_freq=100)
return train
def __linit(self, X, y):
if (self.verbose > 0):
print "Lazy initialisation"
layers = self.layers
pylearn2mlp_layers = []
self.units_per_layer = []
#input layer units
self.units_per_layer += [X.shape[1]]
for layer in layers[:-1]:
self.units_per_layer += [layer[1]]
#Output layer units
self.units_per_layer += [y.shape[1]]
if (self.verbose > 0):
print "Units per layer", str(self.units_per_layer)
for i, layer in enumerate(layers[:-1]):
fan_in = self.units_per_layer[i] + 1
fan_out = self.units_per_layer[i + 1]
lim = np.sqrt(6) / (np.sqrt(fan_in + fan_out))
layer_name = "Hidden_%i_%s" % (i, layer[0])
activate_type = layer[0]
if activate_type == "RectifiedLinear":
hidden_layer = mlp.RectifiedLinear(dim=layer[1],
layer_name=layer_name,
irange=lim)
elif activate_type == "Sigmoid":
hidden_layer = mlp.Sigmoid(dim=layer[1],
layer_name=layer_name,
irange=lim)
elif activate_type == "Tanh":
hidden_layer = mlp.Tanh(dim=layer[1],
layer_name=layer_name,
irange=lim)
elif activate_type == "Maxout":
hidden_layer = maxout.Maxout(num_units=layer[1],
num_pieces=layer[2],
layer_name=layer_name,
irange=lim)
else:
raise NotImplementedError(
"Layer of type %s are not implemented yet" % layer[0])
pylearn2mlp_layers += [hidden_layer]
output_layer_info = layers[-1]
output_layer_name = "Output_%s" % output_layer_info[0]
fan_in = self.units_per_layer[-2] + 1
fan_out = self.units_per_layer[-1]
lim = np.sqrt(6) / (np.sqrt(fan_in + fan_out))
if (output_layer_info[0] == "Linear"):
output_layer = mlp.Linear(dim=self.units_per_layer[-1],
layer_name=output_layer_name,
irange=lim)
pylearn2mlp_layers += [output_layer]
self.mlp = mlp.MLP(pylearn2mlp_layers, nvis=self.units_per_layer[0])
self.ds = DenseDesignMatrix(X=X, y=y)
self.trainer.setup(self.mlp, self.ds)
inputs = self.mlp.get_input_space().make_theano_batch()
self.f = theano.function([inputs], self.mlp.fprop(inputs))
def main():
base_name = sys.argv[1] #文件名前缀
n_epoch = int(sys.argv[2]) # epoch次数
n_hidden = int(sys.argv[3]) # 隐含层节点数
include_rate = float(sys.argv[4]) # 包含率（1-dropout）
in_size = 943 # 输入层节点数目
out_size = 4760 #输出层节点数
b_size = 200 #batch的大小
l_rate = 5e-4 #学习速率
l_rate_min = 1e-5 #学习速率最小值
decay_factor = 0.9 #
lr_scale = 3.0 #
momentum = 0.5 #摄动因子
init_vals = np.sqrt(6.0/(np.array([in_size, n_hidden])+np.array([n_hidden, out_size])))
print 'loading data...'
#读取数据Train,Validation,Test
X_tr = np.load('bgedv2_X_tr_float64.npy')
Y_tr = np.load('bgedv2_Y_tr_0-4760_float64.npy')
Y_tr_target = np.array(Y_tr)
X_va = np.load('bgedv2_X_va_float64.npy')
Y_va = np.load('bgedv2_Y_va_0-4760_float64.npy')
Y_va_target = np.array(Y_va)
X_te = np.load('bgedv2_X_te_float64.npy')
Y_te = np.load('bgedv2_Y_te_0-4760_float64.npy')
Y_te_target = np.array(Y_te)
X_1000G = np.load('1000G_X_float64.npy')
Y_1000G = np.load('1000G_Y_0-4760_float64.npy')
Y_1000G_target = np.array(Y_1000G)
X_GTEx = np.load('GTEx_X_float64.npy')
Y_GTEx = np.load('GTEx_Y_0-4760_float64.npy')
Y_GTEx_target = np.array(Y_GTEx)
#随机化
random.seed(0)
#随机抽取5000样本进行训练
monitor_idx_tr = random.sample(range(88807), 5000)
#将数据X,Y整合成DensenMatrix类型
data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'), y=Y_tr.astype('float32'))
#取出X中对应5000样本进行训练
X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[monitor_idx_tr, :]
#设置多层感知机的隐含层计算方式
h1_layer = p2_md_mlp.Tanh(layer_name='h1', dim=n_hidden, irange=init_vals[0], W_lr_scale=1.0, b_lr_scale=1.0)
#设置多层感知机的输出层计算方式
o_layer = p2_md_mlp.Linear(layer_name='y', dim=out_size, irange=0.0001, W_lr_scale=lr_scale, b_lr_scale=1.0)
#设置好模型
model = p2_md_mlp.MLP(nvis=in_size, layers=[h1_layer, o_layer], seed=1)
#设置dropout比例
dropout_cost = p2_ct_mlp_dropout.Dropout(input_include_probs={'h1':1.0, 'y':include_rate},
input_scales={'h1':1.0,
'y':np.float32(1.0/include_rate)})
#设置训练算法（batch大小，学习速率，学习规则，终止条件，dropout比例）
algorithm = p2_alg_sgd.SGD(batch_size=b_size, learning_rate=l_rate,
learning_rule = p2_alg_lr.Momentum(momentum),
termination_criterion=p2_termcri.EpochCounter(max_epochs=1000),
cost=dropout_cost)
#设置训练类（数据集，训练模型，训练算法）
train = pylearn2.train.Train(dataset=data_tr, model=model, algorithm=algorithm)
train.setup()
x = T.matrix()
y = model.fprop(x) #训练好的模型对X的预测值
f = theano.function([x], y)
MAE_va_old = 10.0
MAE_va_best = 10.0
MAE_tr_old = 10.0
MAE_te_old = 10.0
MAE_1000G_old = 10.0
MAE_1000G_best = 10.0
MAE_GTEx_old = 10.0
outlog = open(base_name + '.log', 'w')
log_str = '\t'.join(map(str, ['epoch', 'MAE_va', 'MAE_va_change', 'MAE_te', 'MAE_te_change',
'MAE_1000G', 'MAE_1000G_change', 'MAE_GTEx', 'MAE_GTEx_change',
'MAE_tr', 'MAE_tr_change', 'learing_rate', 'time(sec)']))
print log_str
outlog.write(log_str + '\n')
sys.stdout.flush() #刷新缓冲区
for epoch in range(0, n_epoch):
t_old = time.time() #开始时间
train.algorithm.train(train.dataset)#训练
#计算不同数据集预测值
Y_va_hat = f(X_va.astype('float32')).astype('float64')
Y_te_hat = f(X_te.astype('float32')).astype('float64')
Y_tr_hat_monitor = f(X_tr_monitor.astype('float32')).astype('float64')
Y_1000G_hat = f(X_1000G.astype('float32')).astype('float64')
Y_GTEx_hat = f(X_GTEx.astype('float32')).astype('float64')
#计算预测值与真实值的MAE
MAE_va = np.abs(Y_va_target - Y_va_hat).mean()
MAE_te = np.abs(Y_te_target - Y_te_hat).mean()
MAE_tr = np.abs(Y_tr_monitor_target - Y_tr_hat_monitor).mean()
MAE_1000G = np.abs(Y_1000G_target - Y_1000G_hat).mean()
MAE_GTEx = np.abs(Y_GTEx_target - Y_GTEx_hat).mean()
#计算迭代误差
MAE_va_change = (MAE_va - MAE_va_old)/MAE_va_old
MAE_te_change = (MAE_te - MAE_te_old)/MAE_te_old
MAE_tr_change = (MAE_tr - MAE_tr_old)/MAE_tr_old
MAE_1000G_change = (MAE_1000G - MAE_1000G_old)/MAE_1000G_old
MAE_GTEx_change = (MAE_GTEx - MAE_GTEx_old)/MAE_GTEx_old
#更新MAE
MAE_va_old = MAE_va
MAE_te_old = MAE_te
MAE_tr_old = MAE_tr
MAE_1000G_old = MAE_1000G
MAE_GTEx_old = MAE_GTEx
t_new = time.time() #终止时间
l_rate = train.algorithm.learning_rate.get_value()
log_str = '\t'.join(map(str, [epoch+1, '%.6f'%MAE_va, '%.6f'%MAE_va_change, '%.6f'%MAE_te, '%.6f'%MAE_te_change,
'%.6f'%MAE_1000G, '%.6f'%MAE_1000G_change, '%.6f'%MAE_GTEx, '%.6f'%MAE_GTEx_change,
'%.6f'%MAE_tr, '%.6f'%MAE_tr_change, '%.5f'%l_rate, int(t_new-t_old)]))
print log_str
outlog.write(log_str + '\n')
sys.stdout.flush()
if MAE_tr_change > 0: #如果误差增大，减小学习速率
l_rate = l_rate*decay_factor
if l_rate < l_rate_min: #学习速率最小为l_rate_min
l_rate = l_rate_min
train.algorithm.learning_rate.set_value(np.float32(l_rate)) #更改训练类的学习速率参数
#更新Validation误差值
if MAE_va < MAE_va_best:
MAE_va_best = MAE_va
outmodel = open(base_name + '_bestva_model.pkl', 'wb')
pkl.dump(model, outmodel)
outmodel.close()
np.save(base_name + '_bestva_Y_te_hat.npy', Y_te_hat)
np.save(base_name + '_bestva_Y_va_hat.npy', Y_va_hat)
#更新1000G误差值
if MAE_1000G < MAE_1000G_best:
MAE_1000G_best = MAE_1000G
outmodel = open(base_name + '_best1000G_model.pkl', 'wb')
pkl.dump(model, outmodel)
outmodel.close()
np.save(base_name + '_best1000G_Y_1000G_hat.npy', Y_1000G_hat)
np.save(base_name + '_best1000G_Y_GTEx_hat.npy', Y_GTEx_hat)
print 'MAE_va_best : %.6f' % (MAE_va_best)
print 'MAE_1000G_best : %.6f' % (MAE_1000G_best)
outlog.write('MAE_va_best : %.6f' % (MAE_va_best) + '\n')
outlog.write('MAE_1000G_best : %.6f' % (MAE_1000G_best) + '\n')
outlog.close()
def main():
base_name = sys.argv[
1] # 获取第一个参数 sys.argv[ ]记录（获取）命令行参数 sys(system) argv(argument variable)参数变量，该变量为list列表
n_epoch = int(sys.argv[2]) #获取第二个参数
n_hidden = int(sys.argv[3]) #获取第三个参数作为隐层神经元个数
include_rate = float(sys.argv[4])
in_size = 1001 #输入层神经元个数（标记基因个数）
out_size = 1 #输出层神经元个数
b_size = 200 #偏差值
l_rate = 5e-4 #学习速率
l_rate_min = 1e-5 #学习速率最小值
decay_factor = 0.9 #衰减因数
lr_scale = 3.0
momentum = 0.5
init_vals = np.sqrt(6.0 / (np.array([in_size, n_hidden]) +
np.array([n_hidden, out_size]))) #初始值，返回平方根
print 'loading data...' #显示载入数据
X_tr = np.load(
'geno_X_tr_float64.npy') # tr(traing)以numpy专用二进制类型保存训练数据集的数据
Y_tr = np.load('pheno_Y_tr_0-4760_float64.npy')
Y_tr_pheno = np.array(Y_tr)
X_va = np.load(
'geno_X_va_float64.npy') #验证集（模型选择，在学习到不同复杂度的模型中，选择对验证集有最小预测误差的模型）
Y_va = np.load('pheno_Y_va_0-4760_float64.npy')
Y_va_target = np.array(Y_va)
X_te = np.load('geno_te_float64.npy') #测试集（对学习方法的评估）
Y_te = np.load('pheno_Y_te_0-4760_float64.npy')
Y_te_target = np.array(Y_te)
random.seed(0) #设置生成随机数用的整数起始值。调用任何其他random模块函数之前调用这个函数
monitor_idx_tr = random.sample(range(88807), 5000) #监测训练
#将训练数据集类型设为32位浮点型，The DenseDesignMatrix class and related code Functionality for representing data that can be described as a dense matrix (rather than a sparse matrix) with each row containing an example and each column corresponding to a different feature.
data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'),
y=Y_tr.astype('float32'))
X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[
monitor_idx_tr, :]
#一个隐层，用Tanh（）作激活函数; 输出层用线性函数作激活函数
h1_layer = p2_md_mlp.Tanh(layer_name='h1',
dim=n_hidden,
irange=init_vals[0],
W_lr_scale=1.0,
b_lr_scale=1.0)
o_layer = p2_md_mlp.Linear(layer_name='y',
dim=out_size,
irange=0.0001,
W_lr_scale=lr_scale,
b_lr_scale=1.0)
#Multilayer Perceptron；nvis(Number of “visible units” input units) layers(a list of layer objects，最后1层指定MLP的输出空间)
model = p2_md_mlp.MLP(nvis=in_size, layers=[h1_layer, o_layer], seed=1)
dropout_cost = p2_ct_mlp_dropout.Dropout(input_include_probs={
'h1': 1.0,
'y': include_rate
},
input_scales={
'h1':
1.0,
'y':
np.float32(1.0 / include_rate)
})
#随机梯度下降法
algorithm = p2_alg_sgd.SGD(
batch_size=b_size,
learning_rate=l_rate,
learning_rule=p2_alg_lr.Momentum(momentum),
termination_criterion=p2_termcri.EpochCounter(max_epochs=1000),
cost=dropout_cost)
#训练 根据前面的定义 ：dataset为一个密集型矩阵，model为MLP多层神经网络，algorithm为SGD
train = pylearn2.train.Train(dataset=data_tr,
model=model,
algorithm=algorithm)
train.setup()
x = T.matrix() #定义为一个二维数组
#fprop(state_below) does the forward prop transformation
y = model.fprop(x)
f = theano.function([x], y) #定义一个function函数，输入为x,输出为y
MAE_va_old = 10.0 #平均绝对误差
MAE_va_best = 10.0
MAE_tr_old = 10.0 #训练误差
MAE_te_old = 10.0
MAE_1000G_old = 10.0
MAE_1000G_best = 10.0
MAE_GTEx_old = 10.0
#base_name = sys.argv[1] # 获取第一个参数 sys.argv[ ]记录（获取）命令行参数
outlog = open(base_name + '.log', 'w')
log_str = '\t'.join(
map(str, [
'epoch', 'MAE_va', 'MAE_va_change', 'MAE_te', 'MAE_te_change',
'MAE_tr', 'MAE_tr_change', 'learing_rate', 'time(sec)'
]))
print log_str #输出运行日志
outlog.write(log_str + '\n')
#Python的标准输出缓冲（这意味着它收集“写入”标准出来之前，将其写入到终端的数据）。调用sys.stdout.flush()强制其“缓冲
sys.stdout.flush()
for epoch in range(0, n_epoch):
t_old = time.time()
train.algorithm.train(train.dataset)
Y_va_hat = f(X_va.astype('float32')).astype('float64')
Y_te_hat = f(X_te.astype('float32')).astype('float64')
Y_tr_hat_monitor = f(X_tr_monitor.astype('float32')).astype('float64')
#计算平均绝对误差
MAE_va = np.abs(Y_va_target - Y_va_hat).mean()
MAE_te = np.abs(Y_te_target - Y_te_hat).mean()
MAE_tr = np.abs(Y_tr_monitor_target - Y_tr_hat_monitor).mean()
#误差变换率
MAE_va_change = (MAE_va - MAE_va_old) / MAE_va_old
MAE_te_change = (MAE_te - MAE_te_old) / MAE_te_old
MAE_tr_change = (MAE_tr - MAE_tr_old) / MAE_tr_old
#将old误差值更新为当前误差值
MAE_va_old = MAE_va
MAE_te_old = MAE_te
MAE_tr_old = MAE_tr
#返回当前的时间戳（1970纪元后经过的浮点秒数）
t_new = time.time()
l_rate = train.algorithm.learning_rate.get_value()
log_str = '\t'.join(
map(str, [
epoch + 1,
'%.6f' % MAE_va,
'%.6f' % MAE_va_change,
'%.6f' % MAE_te,
'%.6f' % MAE_te_change,
'%.6f' % MAE_tr,
'%.6f' % MAE_tr_change,
'%.5f' % l_rate,
int(t_new - t_old)
]))
print log_str
outlog.write(log_str + '\n')
sys.stdout.flush()
if MAE_tr_change > 0: #训练误差变换率大于0时，学习速率乘上一个衰减因子
l_rate = l_rate * decay_factor
if l_rate < l_rate_min: #学习速率小于最小速率时，更新为最小速率
l_rate = l_rate_min
train.algorithm.learning_rate.set_value(np.float32(l_rate))
if MAE_va < MAE_va_best:
MAE_va_best = MAE_va
outmodel = open(base_name + '_bestva_model.pkl', 'wb')
pkl.dump(model, outmodel)
outmodel.close()
np.save(base_name + '_bestva_Y_te_hat.npy', Y_te_hat)
np.save(base_name + '_bestva_Y_va_hat.npy', Y_va_hat)
print 'MAE_va_best : %.6f' % (MAE_va_best)
outlog.write('MAE_va_best : %.6f' % (MAE_va_best) + '\n')
outlog.close()
def main():
base_name = sys.argv[1]
n_epoch = int(sys.argv[2])
n_hidden = int(sys.argv[3])
include_rate = float(sys.argv[4])
in_size = 943
out_size = 4760
b_size = 200
l_rate = 3e-4
l_rate_min = 1e-5
decay_factor = 0.9
lr_scale = 3.0
momentum = 0.5
init_vals = np.sqrt(6.0/(np.array([in_size, n_hidden, n_hidden, n_hidden])+np.array([n_hidden, n_hidden, n_hidden, out_size])))
print 'loading data...'
X_tr = np.load('bgedv2_X_tr_float64.npy')
Y_tr = np.load('bgedv2_Y_tr_4760-9520_float64.npy')
Y_tr_target = np.array(Y_tr)
X_va = np.load('bgedv2_X_va_float64.npy')
Y_va = np.load('bgedv2_Y_va_4760-9520_float64.npy')
Y_va_target = np.array(Y_va)
X_te = np.load('bgedv2_X_te_float64.npy')
Y_te = np.load('bgedv2_Y_te_4760-9520_float64.npy')
Y_te_target = np.array(Y_te)
X_1000G = np.load('1000G_X_float64.npy')
Y_1000G = np.load('1000G_Y_4760-9520_float64.npy')
Y_1000G_target = np.array(Y_1000G)
X_GTEx = np.load('GTEx_X_float64.npy')
Y_GTEx = np.load('GTEx_Y_4760-9520_float64.npy')
Y_GTEx_target = np.array(Y_GTEx)
random.seed(0)
monitor_idx_tr = random.sample(range(88807), 5000)
data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'), y=Y_tr.astype('float32'))
X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[monitor_idx_tr, :]
h1_layer = p2_md_mlp.Tanh(layer_name='h1', dim=n_hidden, irange=init_vals[0], W_lr_scale=1.0, b_lr_scale=1.0)
h2_layer = p2_md_mlp.Tanh(layer_name='h2', dim=n_hidden, irange=init_vals[1], W_lr_scale=lr_scale, b_lr_scale=1.0)
h3_layer = p2_md_mlp.Tanh(layer_name='h3', dim=n_hidden, irange=init_vals[2], W_lr_scale=lr_scale, b_lr_scale=1.0)
o_layer = p2_md_mlp.Linear(layer_name='y', dim=out_size, irange=0.0001, W_lr_scale=lr_scale, b_lr_scale=1.0)
model = p2_md_mlp.MLP(nvis=in_size, layers=[h1_layer, h2_layer, h3_layer, o_layer], seed=1)
dropout_cost = p2_ct_mlp_dropout.Dropout(input_include_probs={'h1':1.0, 'h2':include_rate, 'h3':include_rate,
'y':include_rate},
input_scales={'h1':1.0, 'h2':np.float32(1.0/include_rate),
'h3':np.float32(1.0/include_rate),
'y':np.float32(1.0/include_rate)})
algorithm = p2_alg_sgd.SGD(batch_size=b_size, learning_rate=l_rate,
learning_rule = p2_alg_lr.Momentum(momentum),
termination_criterion=p2_termcri.EpochCounter(max_epochs=1000),
cost=dropout_cost)
train = pylearn2.train.Train(dataset=data_tr, model=model, algorithm=algorithm)
train.setup()
x = T.matrix()
y = model.fprop(x)
f = theano.function([x], y)
MAE_va_old = 10.0
MAE_va_best = 10.0
MAE_tr_old = 10.0
MAE_te_old = 10.0
MAE_1000G_old = 10.0
MAE_1000G_best = 10.0
MAE_GTEx_old = 10.0
outlog = open(base_name + '.log', 'w')
log_str = '\t'.join(map(str, ['epoch', 'MAE_va', 'MAE_va_change', 'MAE_te', 'MAE_te_change',
'MAE_1000G', 'MAE_1000G_change', 'MAE_GTEx', 'MAE_GTEx_change',
'MAE_tr', 'MAE_tr_change', 'learing_rate', 'time(sec)']))
print log_str
outlog.write(log_str + '\n')
sys.stdout.flush()
for epoch in range(0, n_epoch):
t_old = time.time()
train.algorithm.train(train.dataset)
Y_va_hat = f(X_va.astype('float32')).astype('float64')
Y_te_hat = f(X_te.astype('float32')).astype('float64')
Y_tr_hat_monitor = f(X_tr_monitor.astype('float32')).astype('float64')
Y_1000G_hat = f(X_1000G.astype('float32')).astype('float64')
Y_GTEx_hat = f(X_GTEx.astype('float32')).astype('float64')
MAE_va = np.abs(Y_va_target - Y_va_hat).mean()
MAE_te = np.abs(Y_te_target - Y_te_hat).mean()
MAE_tr = np.abs(Y_tr_monitor_target - Y_tr_hat_monitor).mean()
MAE_1000G = np.abs(Y_1000G_target - Y_1000G_hat).mean()
MAE_GTEx = np.abs(Y_GTEx_target - Y_GTEx_hat).mean()
MAE_va_change = (MAE_va - MAE_va_old)/MAE_va_old
MAE_te_change = (MAE_te - MAE_te_old)/MAE_te_old
MAE_tr_change = (MAE_tr - MAE_tr_old)/MAE_tr_old
MAE_1000G_change = (MAE_1000G - MAE_1000G_old)/MAE_1000G_old
MAE_GTEx_change = (MAE_GTEx - MAE_GTEx_old)/MAE_GTEx_old
MAE_va_old = MAE_va
MAE_te_old = MAE_te
MAE_tr_old = MAE_tr
MAE_1000G_old = MAE_1000G
MAE_GTEx_old = MAE_GTEx
t_new = time.time()
l_rate = train.algorithm.learning_rate.get_value()
log_str = '\t'.join(map(str, [epoch+1, '%.6f'%MAE_va, '%.6f'%MAE_va_change, '%.6f'%MAE_te, '%.6f'%MAE_te_change,
'%.6f'%MAE_1000G, '%.6f'%MAE_1000G_change, '%.6f'%MAE_GTEx, '%.6f'%MAE_GTEx_change,
'%.6f'%MAE_tr, '%.6f'%MAE_tr_change, '%.5f'%l_rate, int(t_new-t_old)]))
print log_str
outlog.write(log_str + '\n')
sys.stdout.flush()
if MAE_tr_change > 0:
l_rate = l_rate*decay_factor
if l_rate < l_rate_min:
l_rate = l_rate_min
train.algorithm.learning_rate.set_value(np.float32(l_rate))
if MAE_va < MAE_va_best:
MAE_va_best = MAE_va
outmodel = open(base_name + '_bestva_model.pkl', 'wb')
pkl.dump(model, outmodel)
outmodel.close()
np.save(base_name + '_bestva_Y_te_hat.npy', Y_te_hat)
np.save(base_name + '_bestva_Y_va_hat.npy', Y_va_hat)
if MAE_1000G < MAE_1000G_best:
MAE_1000G_best = MAE_1000G
outmodel = open(base_name + '_best1000G_model.pkl', 'wb')
pkl.dump(model, outmodel)
outmodel.close()
np.save(base_name + '_best1000G_Y_1000G_hat.npy', Y_1000G_hat)
np.save(base_name + '_best1000G_Y_GTEx_hat.npy', Y_GTEx_hat)
print 'MAE_va_best : %.6f' % (MAE_va_best)
print 'MAE_1000G_best : %.6f' % (MAE_1000G_best)
outlog.write('MAE_va_best : %.6f' % (MAE_va_best) + '\n')
outlog.write('MAE_1000G_best : %.6f' % (MAE_1000G_best) + '\n')
outlog.close()
X_va = np.load('geno_X_va.npy') #验证集（模型选择，在学习到不同复杂度的模型中，选择对验证集有最小预测误差的模型）
Y_va = np.load('pheno_Y_va.npy')
Y_va_target = np.array(Y_va)
X_te = np.load('geno_X_te.npy') #测试集（对学习方法的评估）
Y_te = np.load('pheno_Y_te.npy')
Y_te_target = np.array(Y_te)
random.seed(0) #设置生成随机数用的整数起始值。调用任何其他random模块函数之前调用这个函数
monitor_idx_tr = random.sample(range(88807), 5000) #监测训练
#将训练数据集类型设为32位浮点型，The DenseDesignMatrix class and related code Functionality for representing data that can be described as a dense matrix (rather than a sparse matrix) with each row containing an example and each column corresponding to a different feature.
data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'), y=Y_tr.astype('float32'))
X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[monitor_idx_tr, :]
#一个隐层，用Tanh（）作激活函数; 输出层用线性函数作激活函数
h1_layer = p2_md_mlp.Tanh(layer_name='h1', dim=n_hidden, irange=init_vals[0], W_lr_scale=1.0, b_lr_scale=1.0)
o_layer = p2_md_mlp.Linear(layer_name='y', dim=out_size, irange=0.0001, W_lr_scale=lr_scale, b_lr_scale=1.0)
#Multilayer Perceptron；nvis(Number of “visible units” input units) layers(a list of layer objects，最后1层指定MLP的输出空间)
model = p2_md_mlp.MLP(nvis=in_size, layers=[h1_layer, o_layer], seed=1)
dropout_cost = p2_ct_mlp_dropout.Dropout(input_include_probs={'h1':1.0, 'y':include_rate},
input_scales={'h1':1.0,
'y':np.float32(1.0/include_rate)})
#随机梯度下降法
algorithm = p2_alg_sgd.SGD(batch_size=b_size, learning_rate=l_rate,
learning_rule = p2_alg_lr.Momentum(momentum),
termination_criterion=p2_termcri.EpochCounter(max_epochs=1000),
cost=dropout_cost)
#训练 根据前面的定义 ：dataset为一个密集型矩阵，model为MLP多层神经网络，algorithm为SGD
train = pylearn2.train.Train(dataset=data_tr, model=model, algorithm=algorithm)
train.setup()
Y_va_target = np.array(Y_va)
X_te = np.load('geno_X_te.npy') #测试集（对学习方法的评估）
Y_te = np.load('pheno_Y_te.npy')
Y_te_target = np.array(Y_te)
random.seed(0)
monitor_idx_tr = random.sample(range(88807), 5000)
data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'),
y=Y_tr.astype('float32'))
X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[
monitor_idx_tr, :]
#隐层设置
h1_layer = p2_md_mlp.Tanh(layer_name='h1',
dim=n_hidden,
irange=init_vals[0],
W_lr_scale=1.0,
b_lr_scale=1.0)
h2_layer = p2_md_mlp.Tanh(layer_name='h2',
dim=n_hidden,
irange=init_vals[1],
W_lr_scale=lr_scale,
b_lr_scale=1.0)
h3_layer = p2_md_mlp.Tanh(layer_name='h3',
dim=n_hidden,
irange=init_vals[2],
W_lr_scale=lr_scale,
b_lr_scale=1.0)
o_layer = p2_md_mlp.Linear(layer_name='y',
dim=out_size,
irange=0.0001,
def init_train(args):
# Interpret arguments.
derived_feat, seed, nhid, width, lrinit, lrdecay, momentum_init, momentum_saturate, wdecay, dropout_include, = args
derived_feat = derived_feat # string
seed = int(seed)
nhid = int(nhid)
width = int(width)
lrinit = float(lrinit)
lrdecay = float(lrdecay)
momentum_init = float(momentum_init)
momentum_saturate = int(momentum_saturate)
wdecay = float(wdecay)
dropout_include = float(
dropout_include) # dropout include probability in top layer.
# Specify output files: log and saved model pkl.
#idpath = os.path.splitext(os.path.abspath(__file__))[0] # ID for output files.
idpath = ''
idpath = idpath + '%s_%d_%d_%d_%0.5f_%0.9f_%0.2f_%d_%f_%0.1f' % (
derived_feat, seed, nhid, width, lrinit, lrdecay, momentum_init,
momentum_saturate, wdecay, dropout_include)
save_path = idpath + '.pkl'
logfile = idpath + '.log'
print 'Using=%s' % theano.config.device # Can use gpus.
print 'Writing to %s' % logfile
print 'Writing to %s' % save_path
sys.stdout = open(logfile, 'w')
# Dataset
benchmark = 1
dataset_train = pylearn2.datasets.physics.PHYSICS(
which_set='train', benchmark=benchmark, derived_feat=derived_feat)
#dataset_train = pylearn2.datasets.physics.PHYSICS(which_set='train', benchmark=benchmark, derived_feat=derived_feat, start=0, stop=2600000) # Smaller set for choosing hyperparameters.
dataset_train_monitor = pylearn2.datasets.physics.PHYSICS(
which_set='train',
benchmark=benchmark,
derived_feat=derived_feat,
start=0,
stop=100000)
dataset_valid = pylearn2.datasets.physics.PHYSICS(
which_set='valid', benchmark=benchmark, derived_feat=derived_feat)
dataset_test = pylearn2.datasets.physics.PHYSICS(which_set='test',
benchmark=benchmark,
derived_feat=derived_feat)
# Model
nvis = dataset_train.X.shape[1]
istdev = 1.0 / np.sqrt(width)
layers = []
for i in range(nhid):
# Hidden layer i
layer = mlp.Tanh(
layer_name='h%d' % i,
dim=width,
istdev=(istdev if i > 0 else
0.1), # First layer should have higher stdev.
)
layers.append(layer)
#layers.append(mlp.Sigmoid(layer_name='y', dim=1, istdev=istdev/100.0))
layers.append(mlp.Sigmoid(layer_name='y', dim=1, istdev=0.001))
model = pylearn2.models.mlp.MLP(layers, nvis=nvis, seed=seed)
# Cost
cost = pylearn2.costs.mlp.Default() # Default cost.
if dropout_include != 1.0:
# Use dropout cost if specified.
cost = pylearn2.costs.mlp.dropout.Dropout(
input_include_probs={'y': dropout_include},
input_scales={'y': 1.0 / dropout_include},
default_input_include_prob=1.0,
default_input_scale=1.0)
if wdecay != 0.0:
# Add weight decay term if specified.
cost2 = pylearn2.costs.mlp.WeightDecay(
coeffs=[wdecay] * (nhid + 1)) # wdecay if specified.
cost = pylearn2.costs.cost.SumOfCosts(costs=[cost, cost2])
# Algorithm
algorithm = pylearn2.training_algorithms.sgd.SGD(
batch_size=100, # If changed, change learning rate!
learning_rate=
lrinit, #.05, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize.
learning_rule=pylearn2.training_algorithms.learning_rule.Momentum(
init_momentum=momentum_init, ),
monitoring_dataset={
'train': dataset_train_monitor,
'valid': dataset_valid,
'test': dataset_test
},
termination_criterion=pylearn2.termination_criteria.Or(criteria=[
pylearn2.termination_criteria.MonitorBased(
channel_name="valid_y_kl", prop_decrease=0.00001, N=10),
pylearn2.termination_criteria.EpochCounter(
max_epochs=momentum_saturate)
]),
cost=cost,
update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay(
decay_factor=
lrdecay, #1.0000002 # Decreases by this factor every batch. (1/(1.000001^8000)^100
min_lr=.000001))
# Extensions
extensions = [
#pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path)
pylearn2.training_algorithms.learning_rule.MomentumAdjustor(
start=0,
saturate=momentum_saturate, # 200,500
final_momentum=0.99, # Dropout=.5->.99 over 500 epochs.
)
]
# Train
train = pylearn2.train.Train(dataset=dataset_train,
model=model,
algorithm=algorithm,
extensions=extensions,
save_path=save_path,
save_freq=100)
return train
def evaluateTeam(eventCodeList, teamNumber):
print "### saving example dataset for team #" + str(teamNumber)
roboDataset = [[[], []], [[], []]]
reattempts = 10
brokenRequest = False
for event in eventCodeList:
r = []
print "requesting matches at", event, "for team", teamNumber
rStr = 'http://www.thebluealliance.com/api/v2/team/frc' + str(
teamNumber
) + '/event/' + event + '/matches?X-TBA-App-Id=frc4534:auto-scouting:2'
try:
r = requests.get(rStr)
except:
print "first match event request for team", teamNumber, "failed, beginning reattempts", r
time.sleep(2)
while r == [] and reattempts > 0:
try:
r = requests.get(rStr)
except:
pass
time.sleep(2)
reattempts -= 1
print reattempts, "more attempts to request matches at", event, "for team", teamNumber
if r == []:
brokenRequest = True
if brokenRequest:
print "broken request, team", teamNumber, ", event:", event, ". internet disconnected/unreachable?"
if brokenRequest == False:
for i in r.json():
try:
stringMatchData = [[], []]
numMatchData = [[], []]
if "frc" + str(
teamNumber) in i['alliances']['blue']['teams']:
alliance = 'blue'
elif "frc" + str(
teamNumber) in i['alliances']['red']['teams']:
alliance = 'red'
else:
alliance = 'team is not in match alliances...'
stringMatchData[0].append('E_LowBar')
stringMatchData[0].append(
i['score_breakdown'][alliance]['position2'])
stringMatchData[0].append(
i['score_breakdown'][alliance]['position3'])
stringMatchData[0].append(
i['score_breakdown'][alliance]['position4'])
stringMatchData[0].append(
i['score_breakdown'][alliance]['position5'])
stringMatchData[1].append(
i['score_breakdown'][alliance]['position1crossings'])
stringMatchData[1].append(
i['score_breakdown'][alliance]['position2crossings'])
stringMatchData[1].append(
i['score_breakdown'][alliance]['position3crossings'])
stringMatchData[1].append(
i['score_breakdown'][alliance]['position4crossings'])
stringMatchData[1].append(
i['score_breakdown'][alliance]['position5crossings'])
incompleteMatchDataError = False
for j in stringMatchData[0]:
if j == 'E_LowBar':
numMatchData[0].append(0)
elif j == 'A_Portcullis':
numMatchData[0].append(1)
elif j == 'A_ChevalDeFrise':
numMatchData[0].append(2)
elif j == 'B_Moat':
numMatchData[0].append(3)
elif j == 'B_Ramparts':
numMatchData[0].append(4)
elif j == 'C_Drawbridge':
numMatchData[0].append(5)
elif j == 'C_SallyPort':
numMatchData[0].append(6)
elif j == 'D_RockWall':
numMatchData[0].append(7)
elif j == 'D_RoughTerrain':
numMatchData[0].append(8)
else:
incompleteMatchDataError = True
for j in stringMatchData[1]:
numMatchData[1].append(j)
if incompleteMatchDataError == False:
for j in numMatchData[0]:
roboDataset[0][0].append([j])
for j in numMatchData[1]:
roboDataset[0][1].append([j])
#roboDataset[0][0].append(numMatchData[0])
#roboDataset[0][1].append(numMatchData[1])
roboDataset[1][1].append([
i['score_breakdown'][alliance]['autoBouldersLow'],
i['score_breakdown'][alliance]['autoBouldersHigh'],
i['score_breakdown'][alliance]['teleopBouldersLow'],
i['score_breakdown'][alliance]['teleopBouldersHigh']
])
roboDataset[1][0].append([0])
except:
print "exception in event " + event + ", team " + str(
teamNumber) + ", match #" + str(i['match_number'])
pass
hidden_layer_1 = mlp.Tanh(layer_name='hidden1',
dim=16,
irange=.1,
init_bias=1.)
hidden_layer_2 = mlp.Tanh(layer_name='hidden2',
dim=8,
irange=.1,
init_bias=1.)
output_layer = mlp.Linear(layer_name='output',
dim=4,
irange=.1,
init_bias=1.)
layers = [hidden_layer_1, hidden_layer_2, output_layer]
trainer = sgd.SGD(learning_rate=.05,
batch_size=10,
termination_criterion=EpochCounter(epochsMode))
ann = mlp.MLP(layers, nvis=1)
roboDataset[1][0] = numpy.array(roboDataset[1][0])
roboDataset[1][1] = numpy.array(roboDataset[1][1])
try:
ds = datasets.DenseDesignMatrix(X=roboDataset[1][0],
y=roboDataset[1][1])
except IndexError:
print "IndexError in dataset creation for team", teamNumber, ",", "length of dataset=", len(
roboDataset[1])
ret = [[], []]
start = time.time()
if len(
roboDataset[1][1]
) > 4: ## only here to train for teams with enough matches to get _anywhere_ within reasonably reliable net results
print "Scoring team", teamNumber, "in goals"
trainer.setup(ann, ds)
print "training for <=", epochsMode, "epochs (team", teamNumber, ")"
while True:
trainer.train(dataset=ds)
ann.monitor.report_epoch()
if not trainer.continue_learning(ann):
break
print "network training time:", int(
time.time() - start), "seconds for team", teamNumber
inputs = numpy.array([[0]])
for i in ann.fprop(theano.shared(inputs, name='inputs')).eval()[0]:
ret[0].append(i)
hidden_layer_1 = mlp.Tanh(layer_name='hidden1',
dim=16,
irange=.1,
init_bias=1.)
hidden_layer_2 = mlp.Tanh(layer_name='hidden2',
dim=8,
irange=.1,
init_bias=1.)
output_layer = mlp.Linear(layer_name='output',
dim=1,
irange=.1,
init_bias=1.)
layers = [hidden_layer_1, hidden_layer_2, output_layer]
trainer = sgd.SGD(learning_rate=.05,
batch_size=10,
termination_criterion=EpochCounter(epochsMode))
ann = mlp.MLP(layers, nvis=1)
roboDataset[0][0] = numpy.array(roboDataset[0][0])
roboDataset[0][1] = numpy.array(roboDataset[0][1])
try:
ds = datasets.DenseDesignMatrix(X=roboDataset[0][0],
y=roboDataset[0][1])
except IndexError:
print "IndexError in dataset creation for team", teamNumber, ",", "length of dataset=", len(
roboDataset[0][1])
start = time.time()
if len(
roboDataset[0][1]
) > 4: ## only here to train for teams with enough matches to get _anywhere_ within reasonably reliable net results
print "Scoring team", teamNumber, "in defenses"
trainer.setup(ann, ds)
print "training for <=", epochsMode, "epochs (team", teamNumber, ")"
while True:
trainer.train(dataset=ds)
ann.monitor.report_epoch()
if not trainer.continue_learning(ann):
break
print "network training time:", int(
time.time() - start), "seconds for team", teamNumber
# inputs = numpy.array([[0]])
inputs = [[0], [1], [2], [3], [4], [5], [6], [7], [8]]
for i in inputs:
ret[1].append(
ann.fprop(theano.shared(numpy.array([i]),
name='inputs')).eval()[0][0])
# for i in ann.fprop(theano.shared(inputs, name='inputs')).eval()[0]:
# ret.append(i)
return ret
def main():
start_time = time.clock()
# optin parser
parser = OptionParser()
parser.add_option("-p",
dest="plot_prediction",
action="store_true",
default=False,
help="plot model prediction transitions")
parser.add_option("-f",
"--file",
dest="out_filename",
default=None,
help="write animation to FILE (require -a option)",
metavar="FILE")
(options, args) = parser.parse_args()
# make Detaset
ds = sinDataset(SIZE_DATA)
# make layers
hidden_layer1 = mlp.Tanh(layer_name='hidden1',
dim=20,
irange=0.5,
init_bias=1.0)
hidden_layer2 = mlp.Tanh(layer_name='hidden2',
dim=4,
irange=0.5,
init_bias=1.0)
output_layer = mlp.Linear(layer_name='out', dim=1, irange=0.5, init_bias=1)
# set layers
layers = [hidden_layer1, hidden_layer2, output_layer]
model = mlp.MLP(layers, nvis=1)
# set training rule and extensions
algorithm = sgd.SGD(learning_rate=0.01,
batch_size=1,
monitoring_batch_size=1,
monitoring_batches=1,
monitoring_dataset=ds,
termination_criterion=EpochCounter(MAX_EPOCHS))
extensions = [sgd.MonitorBasedLRAdjuster()]
if options.plot_prediction:
plotEx = PlotPredictionOnMonitor()
extensions.append(plotEx)
trainer = Train(model=model,
algorithm=algorithm,
dataset=ds,
extensions=extensions,
save_path='./funcmodel.pkl',
save_freq=500)
# training loop
trainer.main_loop()
end_time = time.clock()
print("tortal_seconds_this_learning : %f s(%f min)" %
(end_time - start_time, (end_time - start_time) / 60))
if options.plot_prediction:
plotEx.plot(out_filename=options.out_filename)
