def _create_layer(self, name, layer, irange): if isinstance(layer, Convolution): return self._create_convolution_layer(name, layer, irange) if layer.type == "Rectifier": self._check_layer(layer, ['units']) return mlp.RectifiedLinear(layer_name=name, dim=layer.units, irange=irange) if layer.type == "Sigmoid": self._check_layer(layer, ['units']) return mlp.Sigmoid(layer_name=name, dim=layer.units, irange=irange) if layer.type == "Tanh": self._check_layer(layer, ['units']) return mlp.Tanh(layer_name=name, dim=layer.units, irange=irange) if layer.type == "Maxout": self._check_layer(layer, ['units', 'pieces']) return maxout.Maxout(layer_name=name, num_units=layer.units, num_pieces=layer.pieces, irange=irange) if layer.type == "Linear": self._check_layer(layer, ['units']) return mlp.Linear(layer_name=layer.name, dim=layer.units, irange=irange) if layer.type == "Gaussian": self._check_layer(layer, ['units']) return mlp.LinearGaussian(layer_name=layer.name, init_beta=0.1, min_beta=0.001, max_beta=1000, beta_lr_scale=None, dim=layer.units, irange=irange) if layer.type == "Softmax": self._check_layer(layer, ['units']) return mlp.Softmax(layer_name=layer.name, n_classes=layer.units, irange=irange)
def init_train(): # Initialize train object. idpath = os.path.splitext( os.path.abspath(__file__))[0] # ID for output files. save_path = idpath + '.pkl' # Dataset #seed = 42 benchmark = 1 #derived_feat, nvis = False, 21 derived_feat, nvis = True, 28 #derived_feat, nvis = 'only', 7 dataset_train = pylearn2.datasets.physics.PHYSICS( which_set='train', benchmark=benchmark, derived_feat=derived_feat) dataset_train_monitor = pylearn2.datasets.physics.PHYSICS( which_set='train', benchmark=benchmark, derived_feat=derived_feat, start=0, stop=100000) dataset_valid = pylearn2.datasets.physics.PHYSICS( which_set='valid', benchmark=benchmark, derived_feat=derived_feat) dataset_test = pylearn2.datasets.physics.PHYSICS(which_set='test', benchmark=benchmark, derived_feat=derived_feat) # Parameters momentum_saturate = 200 # Model model = pylearn2.models.mlp.MLP( layers=[ mlp.Tanh(layer_name='h0', dim=300, istdev=.1), #istdev=.05 for any intermediates mlp.Sigmoid(layer_name='y', dim=1, istdev=.001) ], nvis=nvis) # Algorithm algorithm = pylearn2.training_algorithms.sgd.SGD( batch_size=100, # If changed, change learning rate! learning_rate= .05, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize. init_momentum=.9, monitoring_dataset={ 'train': dataset_train_monitor, 'valid': dataset_valid, 'test': dataset_test }, termination_criterion=pylearn2.termination_criteria.Or(criteria=[ pylearn2.termination_criteria.MonitorBased( channel_name="valid_objective", prop_decrease=0.00001, N=10), pylearn2.termination_criteria.EpochCounter( max_epochs=momentum_saturate) ]), cost=pylearn2.costs.cost.SumOfCosts(costs=[ pylearn2.costs.mlp.Default(), pylearn2.costs.mlp.WeightDecay(coeffs=[.00001, .00001]) ]), update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay( decay_factor= 1.0000002, # Decreases by this factor every batch. (1/(1.000001^8000)^100 min_lr=.000001)) # Extensions extensions = [ #pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path) pylearn2.training_algorithms.sgd.MomentumAdjustor( start=0, saturate=momentum_saturate, final_momentum=.99 # Dropout=.5->.99 over 500 epochs. ) ] # Train train = pylearn2.train.Train(dataset=dataset_train, model=model, algorithm=algorithm, extensions=extensions, save_path=save_path, save_freq=100) return train
def __linit(self, X, y): if (self.verbose > 0): print "Lazy initialisation" layers = self.layers pylearn2mlp_layers = [] self.units_per_layer = [] #input layer units self.units_per_layer += [X.shape[1]] for layer in layers[:-1]: self.units_per_layer += [layer[1]] #Output layer units self.units_per_layer += [y.shape[1]] if (self.verbose > 0): print "Units per layer", str(self.units_per_layer) for i, layer in enumerate(layers[:-1]): fan_in = self.units_per_layer[i] + 1 fan_out = self.units_per_layer[i + 1] lim = np.sqrt(6) / (np.sqrt(fan_in + fan_out)) layer_name = "Hidden_%i_%s" % (i, layer[0]) activate_type = layer[0] if activate_type == "RectifiedLinear": hidden_layer = mlp.RectifiedLinear(dim=layer[1], layer_name=layer_name, irange=lim) elif activate_type == "Sigmoid": hidden_layer = mlp.Sigmoid(dim=layer[1], layer_name=layer_name, irange=lim) elif activate_type == "Tanh": hidden_layer = mlp.Tanh(dim=layer[1], layer_name=layer_name, irange=lim) elif activate_type == "Maxout": hidden_layer = maxout.Maxout(num_units=layer[1], num_pieces=layer[2], layer_name=layer_name, irange=lim) else: raise NotImplementedError( "Layer of type %s are not implemented yet" % layer[0]) pylearn2mlp_layers += [hidden_layer] output_layer_info = layers[-1] output_layer_name = "Output_%s" % output_layer_info[0] fan_in = self.units_per_layer[-2] + 1 fan_out = self.units_per_layer[-1] lim = np.sqrt(6) / (np.sqrt(fan_in + fan_out)) if (output_layer_info[0] == "Linear"): output_layer = mlp.Linear(dim=self.units_per_layer[-1], layer_name=output_layer_name, irange=lim) pylearn2mlp_layers += [output_layer] self.mlp = mlp.MLP(pylearn2mlp_layers, nvis=self.units_per_layer[0]) self.ds = DenseDesignMatrix(X=X, y=y) self.trainer.setup(self.mlp, self.ds) inputs = self.mlp.get_input_space().make_theano_batch() self.f = theano.function([inputs], self.mlp.fprop(inputs))
def main(): base_name = sys.argv[1] #文件名前缀 n_epoch = int(sys.argv[2]) # epoch次数 n_hidden = int(sys.argv[3]) # 隐含层节点数 include_rate = float(sys.argv[4]) # 包含率(1-dropout) in_size = 943 # 输入层节点数目 out_size = 4760 #输出层节点数 b_size = 200 #batch的大小 l_rate = 5e-4 #学习速率 l_rate_min = 1e-5 #学习速率最小值 decay_factor = 0.9 # lr_scale = 3.0 # momentum = 0.5 #摄动因子 init_vals = np.sqrt(6.0/(np.array([in_size, n_hidden])+np.array([n_hidden, out_size]))) print 'loading data...' #读取数据Train,Validation,Test X_tr = np.load('bgedv2_X_tr_float64.npy') Y_tr = np.load('bgedv2_Y_tr_0-4760_float64.npy') Y_tr_target = np.array(Y_tr) X_va = np.load('bgedv2_X_va_float64.npy') Y_va = np.load('bgedv2_Y_va_0-4760_float64.npy') Y_va_target = np.array(Y_va) X_te = np.load('bgedv2_X_te_float64.npy') Y_te = np.load('bgedv2_Y_te_0-4760_float64.npy') Y_te_target = np.array(Y_te) X_1000G = np.load('1000G_X_float64.npy') Y_1000G = np.load('1000G_Y_0-4760_float64.npy') Y_1000G_target = np.array(Y_1000G) X_GTEx = np.load('GTEx_X_float64.npy') Y_GTEx = np.load('GTEx_Y_0-4760_float64.npy') Y_GTEx_target = np.array(Y_GTEx) #随机化 random.seed(0) #随机抽取5000样本进行训练 monitor_idx_tr = random.sample(range(88807), 5000) #将数据X,Y整合成DensenMatrix类型 data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'), y=Y_tr.astype('float32')) #取出X中对应5000样本进行训练 X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[monitor_idx_tr, :] #设置多层感知机的隐含层计算方式 h1_layer = p2_md_mlp.Tanh(layer_name='h1', dim=n_hidden, irange=init_vals[0], W_lr_scale=1.0, b_lr_scale=1.0) #设置多层感知机的输出层计算方式 o_layer = p2_md_mlp.Linear(layer_name='y', dim=out_size, irange=0.0001, W_lr_scale=lr_scale, b_lr_scale=1.0) #设置好模型 model = p2_md_mlp.MLP(nvis=in_size, layers=[h1_layer, o_layer], seed=1) #设置dropout比例 dropout_cost = p2_ct_mlp_dropout.Dropout(input_include_probs={'h1':1.0, 'y':include_rate}, input_scales={'h1':1.0, 'y':np.float32(1.0/include_rate)}) #设置训练算法(batch大小,学习速率,学习规则,终止条件,dropout比例) algorithm = p2_alg_sgd.SGD(batch_size=b_size, learning_rate=l_rate, learning_rule = p2_alg_lr.Momentum(momentum), termination_criterion=p2_termcri.EpochCounter(max_epochs=1000), cost=dropout_cost) #设置训练类(数据集,训练模型,训练算法) train = pylearn2.train.Train(dataset=data_tr, model=model, algorithm=algorithm) train.setup() x = T.matrix() y = model.fprop(x) #训练好的模型对X的预测值 f = theano.function([x], y) MAE_va_old = 10.0 MAE_va_best = 10.0 MAE_tr_old = 10.0 MAE_te_old = 10.0 MAE_1000G_old = 10.0 MAE_1000G_best = 10.0 MAE_GTEx_old = 10.0 outlog = open(base_name + '.log', 'w') log_str = '\t'.join(map(str, ['epoch', 'MAE_va', 'MAE_va_change', 'MAE_te', 'MAE_te_change', 'MAE_1000G', 'MAE_1000G_change', 'MAE_GTEx', 'MAE_GTEx_change', 'MAE_tr', 'MAE_tr_change', 'learing_rate', 'time(sec)'])) print log_str outlog.write(log_str + '\n') sys.stdout.flush() #刷新缓冲区 for epoch in range(0, n_epoch): t_old = time.time() #开始时间 train.algorithm.train(train.dataset)#训练 #计算不同数据集预测值 Y_va_hat = f(X_va.astype('float32')).astype('float64') Y_te_hat = f(X_te.astype('float32')).astype('float64') Y_tr_hat_monitor = f(X_tr_monitor.astype('float32')).astype('float64') Y_1000G_hat = f(X_1000G.astype('float32')).astype('float64') Y_GTEx_hat = f(X_GTEx.astype('float32')).astype('float64') #计算预测值与真实值的MAE MAE_va = np.abs(Y_va_target - Y_va_hat).mean() MAE_te = np.abs(Y_te_target - Y_te_hat).mean() MAE_tr = np.abs(Y_tr_monitor_target - Y_tr_hat_monitor).mean() MAE_1000G = np.abs(Y_1000G_target - Y_1000G_hat).mean() MAE_GTEx = np.abs(Y_GTEx_target - Y_GTEx_hat).mean() #计算迭代误差 MAE_va_change = (MAE_va - MAE_va_old)/MAE_va_old MAE_te_change = (MAE_te - MAE_te_old)/MAE_te_old MAE_tr_change = (MAE_tr - MAE_tr_old)/MAE_tr_old MAE_1000G_change = (MAE_1000G - MAE_1000G_old)/MAE_1000G_old MAE_GTEx_change = (MAE_GTEx - MAE_GTEx_old)/MAE_GTEx_old #更新MAE MAE_va_old = MAE_va MAE_te_old = MAE_te MAE_tr_old = MAE_tr MAE_1000G_old = MAE_1000G MAE_GTEx_old = MAE_GTEx t_new = time.time() #终止时间 l_rate = train.algorithm.learning_rate.get_value() log_str = '\t'.join(map(str, [epoch+1, '%.6f'%MAE_va, '%.6f'%MAE_va_change, '%.6f'%MAE_te, '%.6f'%MAE_te_change, '%.6f'%MAE_1000G, '%.6f'%MAE_1000G_change, '%.6f'%MAE_GTEx, '%.6f'%MAE_GTEx_change, '%.6f'%MAE_tr, '%.6f'%MAE_tr_change, '%.5f'%l_rate, int(t_new-t_old)])) print log_str outlog.write(log_str + '\n') sys.stdout.flush() if MAE_tr_change > 0: #如果误差增大,减小学习速率 l_rate = l_rate*decay_factor if l_rate < l_rate_min: #学习速率最小为l_rate_min l_rate = l_rate_min train.algorithm.learning_rate.set_value(np.float32(l_rate)) #更改训练类的学习速率参数 #更新Validation误差值 if MAE_va < MAE_va_best: MAE_va_best = MAE_va outmodel = open(base_name + '_bestva_model.pkl', 'wb') pkl.dump(model, outmodel) outmodel.close() np.save(base_name + '_bestva_Y_te_hat.npy', Y_te_hat) np.save(base_name + '_bestva_Y_va_hat.npy', Y_va_hat) #更新1000G误差值 if MAE_1000G < MAE_1000G_best: MAE_1000G_best = MAE_1000G outmodel = open(base_name + '_best1000G_model.pkl', 'wb') pkl.dump(model, outmodel) outmodel.close() np.save(base_name + '_best1000G_Y_1000G_hat.npy', Y_1000G_hat) np.save(base_name + '_best1000G_Y_GTEx_hat.npy', Y_GTEx_hat) print 'MAE_va_best : %.6f' % (MAE_va_best) print 'MAE_1000G_best : %.6f' % (MAE_1000G_best) outlog.write('MAE_va_best : %.6f' % (MAE_va_best) + '\n') outlog.write('MAE_1000G_best : %.6f' % (MAE_1000G_best) + '\n') outlog.close()
def main(): base_name = sys.argv[ 1] # 获取第一个参数 sys.argv[ ]记录(获取)命令行参数 sys(system) argv(argument variable)参数变量,该变量为list列表 n_epoch = int(sys.argv[2]) #获取第二个参数 n_hidden = int(sys.argv[3]) #获取第三个参数作为隐层神经元个数 include_rate = float(sys.argv[4]) in_size = 1001 #输入层神经元个数(标记基因个数) out_size = 1 #输出层神经元个数 b_size = 200 #偏差值 l_rate = 5e-4 #学习速率 l_rate_min = 1e-5 #学习速率最小值 decay_factor = 0.9 #衰减因数 lr_scale = 3.0 momentum = 0.5 init_vals = np.sqrt(6.0 / (np.array([in_size, n_hidden]) + np.array([n_hidden, out_size]))) #初始值,返回平方根 print 'loading data...' #显示载入数据 X_tr = np.load( 'geno_X_tr_float64.npy') # tr(traing)以numpy专用二进制类型保存训练数据集的数据 Y_tr = np.load('pheno_Y_tr_0-4760_float64.npy') Y_tr_pheno = np.array(Y_tr) X_va = np.load( 'geno_X_va_float64.npy') #验证集(模型选择,在学习到不同复杂度的模型中,选择对验证集有最小预测误差的模型) Y_va = np.load('pheno_Y_va_0-4760_float64.npy') Y_va_target = np.array(Y_va) X_te = np.load('geno_te_float64.npy') #测试集(对学习方法的评估) Y_te = np.load('pheno_Y_te_0-4760_float64.npy') Y_te_target = np.array(Y_te) random.seed(0) #设置生成随机数用的整数起始值。调用任何其他random模块函数之前调用这个函数 monitor_idx_tr = random.sample(range(88807), 5000) #监测训练 #将训练数据集类型设为32位浮点型,The DenseDesignMatrix class and related code Functionality for representing data that can be described as a dense matrix (rather than a sparse matrix) with each row containing an example and each column corresponding to a different feature. data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'), y=Y_tr.astype('float32')) X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[ monitor_idx_tr, :] #一个隐层,用Tanh()作激活函数; 输出层用线性函数作激活函数 h1_layer = p2_md_mlp.Tanh(layer_name='h1', dim=n_hidden, irange=init_vals[0], W_lr_scale=1.0, b_lr_scale=1.0) o_layer = p2_md_mlp.Linear(layer_name='y', dim=out_size, irange=0.0001, W_lr_scale=lr_scale, b_lr_scale=1.0) #Multilayer Perceptron;nvis(Number of “visible units” input units) layers(a list of layer objects,最后1层指定MLP的输出空间) model = p2_md_mlp.MLP(nvis=in_size, layers=[h1_layer, o_layer], seed=1) dropout_cost = p2_ct_mlp_dropout.Dropout(input_include_probs={ 'h1': 1.0, 'y': include_rate }, input_scales={ 'h1': 1.0, 'y': np.float32(1.0 / include_rate) }) #随机梯度下降法 algorithm = p2_alg_sgd.SGD( batch_size=b_size, learning_rate=l_rate, learning_rule=p2_alg_lr.Momentum(momentum), termination_criterion=p2_termcri.EpochCounter(max_epochs=1000), cost=dropout_cost) #训练 根据前面的定义 :dataset为一个密集型矩阵,model为MLP多层神经网络,algorithm为SGD train = pylearn2.train.Train(dataset=data_tr, model=model, algorithm=algorithm) train.setup() x = T.matrix() #定义为一个二维数组 #fprop(state_below) does the forward prop transformation y = model.fprop(x) f = theano.function([x], y) #定义一个function函数,输入为x,输出为y MAE_va_old = 10.0 #平均绝对误差 MAE_va_best = 10.0 MAE_tr_old = 10.0 #训练误差 MAE_te_old = 10.0 MAE_1000G_old = 10.0 MAE_1000G_best = 10.0 MAE_GTEx_old = 10.0 #base_name = sys.argv[1] # 获取第一个参数 sys.argv[ ]记录(获取)命令行参数 outlog = open(base_name + '.log', 'w') log_str = '\t'.join( map(str, [ 'epoch', 'MAE_va', 'MAE_va_change', 'MAE_te', 'MAE_te_change', 'MAE_tr', 'MAE_tr_change', 'learing_rate', 'time(sec)' ])) print log_str #输出运行日志 outlog.write(log_str + '\n') #Python的标准输出缓冲(这意味着它收集“写入”标准出来之前,将其写入到终端的数据)。调用sys.stdout.flush()强制其“缓冲 sys.stdout.flush() for epoch in range(0, n_epoch): t_old = time.time() train.algorithm.train(train.dataset) Y_va_hat = f(X_va.astype('float32')).astype('float64') Y_te_hat = f(X_te.astype('float32')).astype('float64') Y_tr_hat_monitor = f(X_tr_monitor.astype('float32')).astype('float64') #计算平均绝对误差 MAE_va = np.abs(Y_va_target - Y_va_hat).mean() MAE_te = np.abs(Y_te_target - Y_te_hat).mean() MAE_tr = np.abs(Y_tr_monitor_target - Y_tr_hat_monitor).mean() #误差变换率 MAE_va_change = (MAE_va - MAE_va_old) / MAE_va_old MAE_te_change = (MAE_te - MAE_te_old) / MAE_te_old MAE_tr_change = (MAE_tr - MAE_tr_old) / MAE_tr_old #将old误差值更新为当前误差值 MAE_va_old = MAE_va MAE_te_old = MAE_te MAE_tr_old = MAE_tr #返回当前的时间戳(1970纪元后经过的浮点秒数) t_new = time.time() l_rate = train.algorithm.learning_rate.get_value() log_str = '\t'.join( map(str, [ epoch + 1, '%.6f' % MAE_va, '%.6f' % MAE_va_change, '%.6f' % MAE_te, '%.6f' % MAE_te_change, '%.6f' % MAE_tr, '%.6f' % MAE_tr_change, '%.5f' % l_rate, int(t_new - t_old) ])) print log_str outlog.write(log_str + '\n') sys.stdout.flush() if MAE_tr_change > 0: #训练误差变换率大于0时,学习速率乘上一个衰减因子 l_rate = l_rate * decay_factor if l_rate < l_rate_min: #学习速率小于最小速率时,更新为最小速率 l_rate = l_rate_min train.algorithm.learning_rate.set_value(np.float32(l_rate)) if MAE_va < MAE_va_best: MAE_va_best = MAE_va outmodel = open(base_name + '_bestva_model.pkl', 'wb') pkl.dump(model, outmodel) outmodel.close() np.save(base_name + '_bestva_Y_te_hat.npy', Y_te_hat) np.save(base_name + '_bestva_Y_va_hat.npy', Y_va_hat) print 'MAE_va_best : %.6f' % (MAE_va_best) outlog.write('MAE_va_best : %.6f' % (MAE_va_best) + '\n') outlog.close()
def main(): base_name = sys.argv[1] n_epoch = int(sys.argv[2]) n_hidden = int(sys.argv[3]) include_rate = float(sys.argv[4]) in_size = 943 out_size = 4760 b_size = 200 l_rate = 3e-4 l_rate_min = 1e-5 decay_factor = 0.9 lr_scale = 3.0 momentum = 0.5 init_vals = np.sqrt(6.0/(np.array([in_size, n_hidden, n_hidden, n_hidden])+np.array([n_hidden, n_hidden, n_hidden, out_size]))) print 'loading data...' X_tr = np.load('bgedv2_X_tr_float64.npy') Y_tr = np.load('bgedv2_Y_tr_4760-9520_float64.npy') Y_tr_target = np.array(Y_tr) X_va = np.load('bgedv2_X_va_float64.npy') Y_va = np.load('bgedv2_Y_va_4760-9520_float64.npy') Y_va_target = np.array(Y_va) X_te = np.load('bgedv2_X_te_float64.npy') Y_te = np.load('bgedv2_Y_te_4760-9520_float64.npy') Y_te_target = np.array(Y_te) X_1000G = np.load('1000G_X_float64.npy') Y_1000G = np.load('1000G_Y_4760-9520_float64.npy') Y_1000G_target = np.array(Y_1000G) X_GTEx = np.load('GTEx_X_float64.npy') Y_GTEx = np.load('GTEx_Y_4760-9520_float64.npy') Y_GTEx_target = np.array(Y_GTEx) random.seed(0) monitor_idx_tr = random.sample(range(88807), 5000) data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'), y=Y_tr.astype('float32')) X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[monitor_idx_tr, :] h1_layer = p2_md_mlp.Tanh(layer_name='h1', dim=n_hidden, irange=init_vals[0], W_lr_scale=1.0, b_lr_scale=1.0) h2_layer = p2_md_mlp.Tanh(layer_name='h2', dim=n_hidden, irange=init_vals[1], W_lr_scale=lr_scale, b_lr_scale=1.0) h3_layer = p2_md_mlp.Tanh(layer_name='h3', dim=n_hidden, irange=init_vals[2], W_lr_scale=lr_scale, b_lr_scale=1.0) o_layer = p2_md_mlp.Linear(layer_name='y', dim=out_size, irange=0.0001, W_lr_scale=lr_scale, b_lr_scale=1.0) model = p2_md_mlp.MLP(nvis=in_size, layers=[h1_layer, h2_layer, h3_layer, o_layer], seed=1) dropout_cost = p2_ct_mlp_dropout.Dropout(input_include_probs={'h1':1.0, 'h2':include_rate, 'h3':include_rate, 'y':include_rate}, input_scales={'h1':1.0, 'h2':np.float32(1.0/include_rate), 'h3':np.float32(1.0/include_rate), 'y':np.float32(1.0/include_rate)}) algorithm = p2_alg_sgd.SGD(batch_size=b_size, learning_rate=l_rate, learning_rule = p2_alg_lr.Momentum(momentum), termination_criterion=p2_termcri.EpochCounter(max_epochs=1000), cost=dropout_cost) train = pylearn2.train.Train(dataset=data_tr, model=model, algorithm=algorithm) train.setup() x = T.matrix() y = model.fprop(x) f = theano.function([x], y) MAE_va_old = 10.0 MAE_va_best = 10.0 MAE_tr_old = 10.0 MAE_te_old = 10.0 MAE_1000G_old = 10.0 MAE_1000G_best = 10.0 MAE_GTEx_old = 10.0 outlog = open(base_name + '.log', 'w') log_str = '\t'.join(map(str, ['epoch', 'MAE_va', 'MAE_va_change', 'MAE_te', 'MAE_te_change', 'MAE_1000G', 'MAE_1000G_change', 'MAE_GTEx', 'MAE_GTEx_change', 'MAE_tr', 'MAE_tr_change', 'learing_rate', 'time(sec)'])) print log_str outlog.write(log_str + '\n') sys.stdout.flush() for epoch in range(0, n_epoch): t_old = time.time() train.algorithm.train(train.dataset) Y_va_hat = f(X_va.astype('float32')).astype('float64') Y_te_hat = f(X_te.astype('float32')).astype('float64') Y_tr_hat_monitor = f(X_tr_monitor.astype('float32')).astype('float64') Y_1000G_hat = f(X_1000G.astype('float32')).astype('float64') Y_GTEx_hat = f(X_GTEx.astype('float32')).astype('float64') MAE_va = np.abs(Y_va_target - Y_va_hat).mean() MAE_te = np.abs(Y_te_target - Y_te_hat).mean() MAE_tr = np.abs(Y_tr_monitor_target - Y_tr_hat_monitor).mean() MAE_1000G = np.abs(Y_1000G_target - Y_1000G_hat).mean() MAE_GTEx = np.abs(Y_GTEx_target - Y_GTEx_hat).mean() MAE_va_change = (MAE_va - MAE_va_old)/MAE_va_old MAE_te_change = (MAE_te - MAE_te_old)/MAE_te_old MAE_tr_change = (MAE_tr - MAE_tr_old)/MAE_tr_old MAE_1000G_change = (MAE_1000G - MAE_1000G_old)/MAE_1000G_old MAE_GTEx_change = (MAE_GTEx - MAE_GTEx_old)/MAE_GTEx_old MAE_va_old = MAE_va MAE_te_old = MAE_te MAE_tr_old = MAE_tr MAE_1000G_old = MAE_1000G MAE_GTEx_old = MAE_GTEx t_new = time.time() l_rate = train.algorithm.learning_rate.get_value() log_str = '\t'.join(map(str, [epoch+1, '%.6f'%MAE_va, '%.6f'%MAE_va_change, '%.6f'%MAE_te, '%.6f'%MAE_te_change, '%.6f'%MAE_1000G, '%.6f'%MAE_1000G_change, '%.6f'%MAE_GTEx, '%.6f'%MAE_GTEx_change, '%.6f'%MAE_tr, '%.6f'%MAE_tr_change, '%.5f'%l_rate, int(t_new-t_old)])) print log_str outlog.write(log_str + '\n') sys.stdout.flush() if MAE_tr_change > 0: l_rate = l_rate*decay_factor if l_rate < l_rate_min: l_rate = l_rate_min train.algorithm.learning_rate.set_value(np.float32(l_rate)) if MAE_va < MAE_va_best: MAE_va_best = MAE_va outmodel = open(base_name + '_bestva_model.pkl', 'wb') pkl.dump(model, outmodel) outmodel.close() np.save(base_name + '_bestva_Y_te_hat.npy', Y_te_hat) np.save(base_name + '_bestva_Y_va_hat.npy', Y_va_hat) if MAE_1000G < MAE_1000G_best: MAE_1000G_best = MAE_1000G outmodel = open(base_name + '_best1000G_model.pkl', 'wb') pkl.dump(model, outmodel) outmodel.close() np.save(base_name + '_best1000G_Y_1000G_hat.npy', Y_1000G_hat) np.save(base_name + '_best1000G_Y_GTEx_hat.npy', Y_GTEx_hat) print 'MAE_va_best : %.6f' % (MAE_va_best) print 'MAE_1000G_best : %.6f' % (MAE_1000G_best) outlog.write('MAE_va_best : %.6f' % (MAE_va_best) + '\n') outlog.write('MAE_1000G_best : %.6f' % (MAE_1000G_best) + '\n') outlog.close()
X_va = np.load('geno_X_va.npy') #验证集(模型选择,在学习到不同复杂度的模型中,选择对验证集有最小预测误差的模型) Y_va = np.load('pheno_Y_va.npy') Y_va_target = np.array(Y_va) X_te = np.load('geno_X_te.npy') #测试集(对学习方法的评估) Y_te = np.load('pheno_Y_te.npy') Y_te_target = np.array(Y_te) random.seed(0) #设置生成随机数用的整数起始值。调用任何其他random模块函数之前调用这个函数 monitor_idx_tr = random.sample(range(88807), 5000) #监测训练 #将训练数据集类型设为32位浮点型,The DenseDesignMatrix class and related code Functionality for representing data that can be described as a dense matrix (rather than a sparse matrix) with each row containing an example and each column corresponding to a different feature. data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'), y=Y_tr.astype('float32')) X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[monitor_idx_tr, :] #一个隐层,用Tanh()作激活函数; 输出层用线性函数作激活函数 h1_layer = p2_md_mlp.Tanh(layer_name='h1', dim=n_hidden, irange=init_vals[0], W_lr_scale=1.0, b_lr_scale=1.0) o_layer = p2_md_mlp.Linear(layer_name='y', dim=out_size, irange=0.0001, W_lr_scale=lr_scale, b_lr_scale=1.0) #Multilayer Perceptron;nvis(Number of “visible units” input units) layers(a list of layer objects,最后1层指定MLP的输出空间) model = p2_md_mlp.MLP(nvis=in_size, layers=[h1_layer, o_layer], seed=1) dropout_cost = p2_ct_mlp_dropout.Dropout(input_include_probs={'h1':1.0, 'y':include_rate}, input_scales={'h1':1.0, 'y':np.float32(1.0/include_rate)}) #随机梯度下降法 algorithm = p2_alg_sgd.SGD(batch_size=b_size, learning_rate=l_rate, learning_rule = p2_alg_lr.Momentum(momentum), termination_criterion=p2_termcri.EpochCounter(max_epochs=1000), cost=dropout_cost) #训练 根据前面的定义 :dataset为一个密集型矩阵,model为MLP多层神经网络,algorithm为SGD train = pylearn2.train.Train(dataset=data_tr, model=model, algorithm=algorithm) train.setup()
Y_va_target = np.array(Y_va) X_te = np.load('geno_X_te.npy') #测试集(对学习方法的评估) Y_te = np.load('pheno_Y_te.npy') Y_te_target = np.array(Y_te) random.seed(0) monitor_idx_tr = random.sample(range(88807), 5000) data_tr = p2_dt_dd.DenseDesignMatrix(X=X_tr.astype('float32'), y=Y_tr.astype('float32')) X_tr_monitor, Y_tr_monitor_target = X_tr[monitor_idx_tr, :], Y_tr_target[ monitor_idx_tr, :] #隐层设置 h1_layer = p2_md_mlp.Tanh(layer_name='h1', dim=n_hidden, irange=init_vals[0], W_lr_scale=1.0, b_lr_scale=1.0) h2_layer = p2_md_mlp.Tanh(layer_name='h2', dim=n_hidden, irange=init_vals[1], W_lr_scale=lr_scale, b_lr_scale=1.0) h3_layer = p2_md_mlp.Tanh(layer_name='h3', dim=n_hidden, irange=init_vals[2], W_lr_scale=lr_scale, b_lr_scale=1.0) o_layer = p2_md_mlp.Linear(layer_name='y', dim=out_size, irange=0.0001,
def init_train(args): # Interpret arguments. derived_feat, seed, nhid, width, lrinit, lrdecay, momentum_init, momentum_saturate, wdecay, dropout_include, = args derived_feat = derived_feat # string seed = int(seed) nhid = int(nhid) width = int(width) lrinit = float(lrinit) lrdecay = float(lrdecay) momentum_init = float(momentum_init) momentum_saturate = int(momentum_saturate) wdecay = float(wdecay) dropout_include = float( dropout_include) # dropout include probability in top layer. # Specify output files: log and saved model pkl. #idpath = os.path.splitext(os.path.abspath(__file__))[0] # ID for output files. idpath = '' idpath = idpath + '%s_%d_%d_%d_%0.5f_%0.9f_%0.2f_%d_%f_%0.1f' % ( derived_feat, seed, nhid, width, lrinit, lrdecay, momentum_init, momentum_saturate, wdecay, dropout_include) save_path = idpath + '.pkl' logfile = idpath + '.log' print 'Using=%s' % theano.config.device # Can use gpus. print 'Writing to %s' % logfile print 'Writing to %s' % save_path sys.stdout = open(logfile, 'w') # Dataset benchmark = 1 dataset_train = pylearn2.datasets.physics.PHYSICS( which_set='train', benchmark=benchmark, derived_feat=derived_feat) #dataset_train = pylearn2.datasets.physics.PHYSICS(which_set='train', benchmark=benchmark, derived_feat=derived_feat, start=0, stop=2600000) # Smaller set for choosing hyperparameters. dataset_train_monitor = pylearn2.datasets.physics.PHYSICS( which_set='train', benchmark=benchmark, derived_feat=derived_feat, start=0, stop=100000) dataset_valid = pylearn2.datasets.physics.PHYSICS( which_set='valid', benchmark=benchmark, derived_feat=derived_feat) dataset_test = pylearn2.datasets.physics.PHYSICS(which_set='test', benchmark=benchmark, derived_feat=derived_feat) # Model nvis = dataset_train.X.shape[1] istdev = 1.0 / np.sqrt(width) layers = [] for i in range(nhid): # Hidden layer i layer = mlp.Tanh( layer_name='h%d' % i, dim=width, istdev=(istdev if i > 0 else 0.1), # First layer should have higher stdev. ) layers.append(layer) #layers.append(mlp.Sigmoid(layer_name='y', dim=1, istdev=istdev/100.0)) layers.append(mlp.Sigmoid(layer_name='y', dim=1, istdev=0.001)) model = pylearn2.models.mlp.MLP(layers, nvis=nvis, seed=seed) # Cost cost = pylearn2.costs.mlp.Default() # Default cost. if dropout_include != 1.0: # Use dropout cost if specified. cost = pylearn2.costs.mlp.dropout.Dropout( input_include_probs={'y': dropout_include}, input_scales={'y': 1.0 / dropout_include}, default_input_include_prob=1.0, default_input_scale=1.0) if wdecay != 0.0: # Add weight decay term if specified. cost2 = pylearn2.costs.mlp.WeightDecay( coeffs=[wdecay] * (nhid + 1)) # wdecay if specified. cost = pylearn2.costs.cost.SumOfCosts(costs=[cost, cost2]) # Algorithm algorithm = pylearn2.training_algorithms.sgd.SGD( batch_size=100, # If changed, change learning rate! learning_rate= lrinit, #.05, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize. learning_rule=pylearn2.training_algorithms.learning_rule.Momentum( init_momentum=momentum_init, ), monitoring_dataset={ 'train': dataset_train_monitor, 'valid': dataset_valid, 'test': dataset_test }, termination_criterion=pylearn2.termination_criteria.Or(criteria=[ pylearn2.termination_criteria.MonitorBased( channel_name="valid_y_kl", prop_decrease=0.00001, N=10), pylearn2.termination_criteria.EpochCounter( max_epochs=momentum_saturate) ]), cost=cost, update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay( decay_factor= lrdecay, #1.0000002 # Decreases by this factor every batch. (1/(1.000001^8000)^100 min_lr=.000001)) # Extensions extensions = [ #pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path) pylearn2.training_algorithms.learning_rule.MomentumAdjustor( start=0, saturate=momentum_saturate, # 200,500 final_momentum=0.99, # Dropout=.5->.99 over 500 epochs. ) ] # Train train = pylearn2.train.Train(dataset=dataset_train, model=model, algorithm=algorithm, extensions=extensions, save_path=save_path, save_freq=100) return train
def evaluateTeam(eventCodeList, teamNumber): print "### saving example dataset for team #" + str(teamNumber) roboDataset = [[[], []], [[], []]] reattempts = 10 brokenRequest = False for event in eventCodeList: r = [] print "requesting matches at", event, "for team", teamNumber rStr = 'http://www.thebluealliance.com/api/v2/team/frc' + str( teamNumber ) + '/event/' + event + '/matches?X-TBA-App-Id=frc4534:auto-scouting:2' try: r = requests.get(rStr) except: print "first match event request for team", teamNumber, "failed, beginning reattempts", r time.sleep(2) while r == [] and reattempts > 0: try: r = requests.get(rStr) except: pass time.sleep(2) reattempts -= 1 print reattempts, "more attempts to request matches at", event, "for team", teamNumber if r == []: brokenRequest = True if brokenRequest: print "broken request, team", teamNumber, ", event:", event, ". internet disconnected/unreachable?" if brokenRequest == False: for i in r.json(): try: stringMatchData = [[], []] numMatchData = [[], []] if "frc" + str( teamNumber) in i['alliances']['blue']['teams']: alliance = 'blue' elif "frc" + str( teamNumber) in i['alliances']['red']['teams']: alliance = 'red' else: alliance = 'team is not in match alliances...' stringMatchData[0].append('E_LowBar') stringMatchData[0].append( i['score_breakdown'][alliance]['position2']) stringMatchData[0].append( i['score_breakdown'][alliance]['position3']) stringMatchData[0].append( i['score_breakdown'][alliance]['position4']) stringMatchData[0].append( i['score_breakdown'][alliance]['position5']) stringMatchData[1].append( i['score_breakdown'][alliance]['position1crossings']) stringMatchData[1].append( i['score_breakdown'][alliance]['position2crossings']) stringMatchData[1].append( i['score_breakdown'][alliance]['position3crossings']) stringMatchData[1].append( i['score_breakdown'][alliance]['position4crossings']) stringMatchData[1].append( i['score_breakdown'][alliance]['position5crossings']) incompleteMatchDataError = False for j in stringMatchData[0]: if j == 'E_LowBar': numMatchData[0].append(0) elif j == 'A_Portcullis': numMatchData[0].append(1) elif j == 'A_ChevalDeFrise': numMatchData[0].append(2) elif j == 'B_Moat': numMatchData[0].append(3) elif j == 'B_Ramparts': numMatchData[0].append(4) elif j == 'C_Drawbridge': numMatchData[0].append(5) elif j == 'C_SallyPort': numMatchData[0].append(6) elif j == 'D_RockWall': numMatchData[0].append(7) elif j == 'D_RoughTerrain': numMatchData[0].append(8) else: incompleteMatchDataError = True for j in stringMatchData[1]: numMatchData[1].append(j) if incompleteMatchDataError == False: for j in numMatchData[0]: roboDataset[0][0].append([j]) for j in numMatchData[1]: roboDataset[0][1].append([j]) #roboDataset[0][0].append(numMatchData[0]) #roboDataset[0][1].append(numMatchData[1]) roboDataset[1][1].append([ i['score_breakdown'][alliance]['autoBouldersLow'], i['score_breakdown'][alliance]['autoBouldersHigh'], i['score_breakdown'][alliance]['teleopBouldersLow'], i['score_breakdown'][alliance]['teleopBouldersHigh'] ]) roboDataset[1][0].append([0]) except: print "exception in event " + event + ", team " + str( teamNumber) + ", match #" + str(i['match_number']) pass hidden_layer_1 = mlp.Tanh(layer_name='hidden1', dim=16, irange=.1, init_bias=1.) hidden_layer_2 = mlp.Tanh(layer_name='hidden2', dim=8, irange=.1, init_bias=1.) output_layer = mlp.Linear(layer_name='output', dim=4, irange=.1, init_bias=1.) layers = [hidden_layer_1, hidden_layer_2, output_layer] trainer = sgd.SGD(learning_rate=.05, batch_size=10, termination_criterion=EpochCounter(epochsMode)) ann = mlp.MLP(layers, nvis=1) roboDataset[1][0] = numpy.array(roboDataset[1][0]) roboDataset[1][1] = numpy.array(roboDataset[1][1]) try: ds = datasets.DenseDesignMatrix(X=roboDataset[1][0], y=roboDataset[1][1]) except IndexError: print "IndexError in dataset creation for team", teamNumber, ",", "length of dataset=", len( roboDataset[1]) ret = [[], []] start = time.time() if len( roboDataset[1][1] ) > 4: ## only here to train for teams with enough matches to get _anywhere_ within reasonably reliable net results print "Scoring team", teamNumber, "in goals" trainer.setup(ann, ds) print "training for <=", epochsMode, "epochs (team", teamNumber, ")" while True: trainer.train(dataset=ds) ann.monitor.report_epoch() if not trainer.continue_learning(ann): break print "network training time:", int( time.time() - start), "seconds for team", teamNumber inputs = numpy.array([[0]]) for i in ann.fprop(theano.shared(inputs, name='inputs')).eval()[0]: ret[0].append(i) hidden_layer_1 = mlp.Tanh(layer_name='hidden1', dim=16, irange=.1, init_bias=1.) hidden_layer_2 = mlp.Tanh(layer_name='hidden2', dim=8, irange=.1, init_bias=1.) output_layer = mlp.Linear(layer_name='output', dim=1, irange=.1, init_bias=1.) layers = [hidden_layer_1, hidden_layer_2, output_layer] trainer = sgd.SGD(learning_rate=.05, batch_size=10, termination_criterion=EpochCounter(epochsMode)) ann = mlp.MLP(layers, nvis=1) roboDataset[0][0] = numpy.array(roboDataset[0][0]) roboDataset[0][1] = numpy.array(roboDataset[0][1]) try: ds = datasets.DenseDesignMatrix(X=roboDataset[0][0], y=roboDataset[0][1]) except IndexError: print "IndexError in dataset creation for team", teamNumber, ",", "length of dataset=", len( roboDataset[0][1]) start = time.time() if len( roboDataset[0][1] ) > 4: ## only here to train for teams with enough matches to get _anywhere_ within reasonably reliable net results print "Scoring team", teamNumber, "in defenses" trainer.setup(ann, ds) print "training for <=", epochsMode, "epochs (team", teamNumber, ")" while True: trainer.train(dataset=ds) ann.monitor.report_epoch() if not trainer.continue_learning(ann): break print "network training time:", int( time.time() - start), "seconds for team", teamNumber # inputs = numpy.array([[0]]) inputs = [[0], [1], [2], [3], [4], [5], [6], [7], [8]] for i in inputs: ret[1].append( ann.fprop(theano.shared(numpy.array([i]), name='inputs')).eval()[0][0]) # for i in ann.fprop(theano.shared(inputs, name='inputs')).eval()[0]: # ret.append(i) return ret
def main(): start_time = time.clock() # optin parser parser = OptionParser() parser.add_option("-p", dest="plot_prediction", action="store_true", default=False, help="plot model prediction transitions") parser.add_option("-f", "--file", dest="out_filename", default=None, help="write animation to FILE (require -a option)", metavar="FILE") (options, args) = parser.parse_args() # make Detaset ds = sinDataset(SIZE_DATA) # make layers hidden_layer1 = mlp.Tanh(layer_name='hidden1', dim=20, irange=0.5, init_bias=1.0) hidden_layer2 = mlp.Tanh(layer_name='hidden2', dim=4, irange=0.5, init_bias=1.0) output_layer = mlp.Linear(layer_name='out', dim=1, irange=0.5, init_bias=1) # set layers layers = [hidden_layer1, hidden_layer2, output_layer] model = mlp.MLP(layers, nvis=1) # set training rule and extensions algorithm = sgd.SGD(learning_rate=0.01, batch_size=1, monitoring_batch_size=1, monitoring_batches=1, monitoring_dataset=ds, termination_criterion=EpochCounter(MAX_EPOCHS)) extensions = [sgd.MonitorBasedLRAdjuster()] if options.plot_prediction: plotEx = PlotPredictionOnMonitor() extensions.append(plotEx) trainer = Train(model=model, algorithm=algorithm, dataset=ds, extensions=extensions, save_path='./funcmodel.pkl', save_freq=500) # training loop trainer.main_loop() end_time = time.clock() print("tortal_seconds_this_learning : %f s(%f min)" % (end_time - start_time, (end_time - start_time) / 60)) if options.plot_prediction: plotEx.plot(out_filename=options.out_filename)