def load_from_hdf5_raw(self, dname = "mimic-cancer", cohort = False, nrows=None):
data_dict = loadDataset(dname);
print("Loaded dataset {0}".format(dname))
self.x_train = data_dict['train_x']
self.x_test = data_dict['test_x']
self.x_valid = data_dict['valid_x']
if cohort:
self.y_valid = data_dict['valid_c']
self.y_train = data_dict['train_c']
self.y_test = data_dict['test_c']
else:
self.y_valid = data_dict['valid_y']
self.y_train = data_dict['train_y']
self.y_test = data_dict['test_y']
if nrows is not None:
print "Truncating rows"
self.x_train = self.x_train[0:nrows]
self.x_test = self.x_test[0:nrows]
self.x_valid = self.x_valid[0:nrows]
self.y_valid = self.y_valid[0:nrows]
self.y_train = self.y_train[0:nrows]
self.y_test = self.y_test[0:nrows]
print("Set up train/valid/test split")
def load_from_hdf5_latent(self, dname = "mimic-cancer", feat_name='mu', ssi=False, cohort=False, nrows=None):
#Load the latent representatinon instead of the raw features
representations = loadHDF5('/data/ml2/vishakh/SHARED/representations.h5')
# we need the labels anyways and we still care which class
data_dict = loadDataset(dname);
if ssi:
feat_name = 'ssi-'+feat_name
#only xs change
self.x_train = representations['train-vae-' + feat_name]
self.x_test = representations['test-vae-' + feat_name]
self.x_valid = representations['valid-vae-' + feat_name]
if cohort:
self.y_valid = data_dict['valid_c']
self.y_train = data_dict['train_c']
self.y_test = data_dict['test_c']
else:
self.y_valid = data_dict['valid_y']
self.y_train = data_dict['train_y']
self.y_test = data_dict['test_y']
if nrows is not None:
print "Truncating rows"
self.x_train = self.x_train[0:nrows]
self.x_test = self.x_test[0:nrows]
self.x_valid = self.x_valid[0:nrows]
self.y_valid = self.y_valid[0:nrows]
self.y_train = self.y_train[0:nrows]
self.y_test = self.y_test[0:nrows]
def run_model():
#load mnist dataset
X_train, y_train, X_test, y_test, num_classes = load.loadDataset("mnist")
#create model from textfile
model = cMod.createModel(filename=file_path)
#set number of epochs
epochs = 50
#print summary
print(model.summary())
#fit model and print results
model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=epochs,
batch_size=200)
score = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (score[1] * 100))
# save the model
model.save(model_name)
# 设置维度的变化范围，步长设置为1
for num_fea in range(5,31,1):
print(num_fea)
# # 设置目录
# data_Dir = "../../dataset/TCGA_LUAD/TCGA_LUAD_logFC1.5_logFC0.5/methyGeneMerge2/"
# 新建各个维度目录
nDim_DIR = data_Dir + "/merge_mrmr" + str(num_fea) + "/"
if (os.path.exists(nDim_DIR) != True):
os.mkdir(nDim_DIR)
# 导入离散数据
# get the feature data and the label
X = []
y = []
load.loadDataset(source_file_disc, X, y)
X = np.array(X)
y = np.array(y)
# 导入原始数据
# get the feature data and the label
X_raw = []
y_raw = []
load.loadDataset(source_fileT, X_raw, y_raw)
X_raw = np.array(X_raw)
y_raw = np.array(y_raw)
# ten fold cross validation
skf = StratifiedKFold(n_splits=10,random_state=14,shuffle=True)
# get the number of fold
n = skf.get_n_splits(X, y)
parser.add_argument('--num-samples',
type=int,
default=5,
help='number of predictions to make for each test item')
parser.add_argument('runme_path', help='path to relevant runme.sh script')
parser.add_argument('conf_path',
help='path to *-config.pkl file in checkpoints')
parser.add_argument('weight_path',
help='path to *-params.h5 file in checkpoints')
parser.add_argument('dest_h5', help='.h5 file to write predictions to')
if __name__ == '__main__':
args = parser.parse_args()
print('Loading dataset')
ds_dict = loadDataset()
if 'p2d' in ds_dict:
dataset = ds_dict['p2d']
else:
dataset = ds_dict['p3d']
print('Loading DKF')
dkf = load_dkf(ds_dict, args.runme_path, args.conf_path, args.weight_path)
print('Generating eval data')
is_2d = isinstance(dataset, p2d_loader.P2DDataset)
pred_usable = None
if is_2d:
result = dataset.get_ds_for_eval(train=False, discard_no_annos=True)
for_cond, for_pred = result['conditioning'], result['prediction']
pred_scales = result['prediction_scales']
import os, time, sys
""" Add the higher level directory to PYTHONPATH to be able to access the models """
sys.path.append('../')
""" Change this to modify the loadDataset function """
from load import loadDataset
"""
This will contain a hashmap where the
parameters correspond to the default ones modified
by any command line options given to this script
"""
from parse_args_dkf import parse
params = parse()
""" Some utility functions from theanomodels """
from utils.misc import removeIfExists, createIfAbsent, mapPrint, saveHDF5, displayTime
""" Load the dataset into a hashmap. See load.py for details """
dataset = loadDataset()
params['savedir'] += '-template'
createIfAbsent(params['savedir'])
""" Add dataset and NADE parameters to "params"
which will become part of the model
"""
for k in ['dim_observations', 'data_type']:
params[k] = dataset[k]
mapPrint('Options: ', params)
if params['use_nade']:
params['data_type'] = 'binary_nade'
"""
import DKF + learn/evaluate functions
"""
start_time = time.time()
from stinfmodel.dkf import DKF
import os, time, sys, addpaths
# Change this to modify the loadDataset function
from load import loadDataset
# This will contain a hashmap where the parameters correspond to the default
# ones modified by any command line options given to this script
from parse_args_dkf import parse
params = parse()
# Some utility functions from theanomodels
from utils.misc import removeIfExists, createIfAbsent, mapPrint, saveHDF5, displayTime
# Load the dataset into a hashmap. See load.py for details
dataset = loadDataset(use_cond=params['use_cond'])
if params['use_cond']:
print('Using conditioning information')
train_cond_vals = dataset['train_cond_vals']
val_cond_vals = dataset['val_cond_vals']
assert train_cond_vals.ndim == 3, train_cond_vals.shape
params['dim_cond'] = train_cond_vals.shape[2]
else:
train_cond_vals = val_cond_vals = None
params['savedir'] += '-h36m'
createIfAbsent(params['savedir'])
# Add dataset and NADE parameters to "params" which will become part of the
# model
for k in ['dim_observations', 'data_type']:
params[k] = dataset[k]
mapPrint('Options: ', params)
import load
import cMod
#load mnist dataset
XTrain, yTrain, XTest, yTest, numClasses = load.loadDataset("mnist")
#create model from textfile
model = cMod.createModel(filename='myM')
#set number of epochs
epochs = 10
#print summary
print(model.summary())
#fit model and print results
model.fit(XTrain,
yTrain,
validation_data=(XTest, yTest),
epochs=epochs,
batch_size=200)
score = model.evaluate(XTest, yTest, verbose=0)
print("Accuracy: %.2f%%" % (score[1] * 100))
def iterate_minibatches(inputs, targets, batchsize, shuffle=False):
assert len(inputs) == len(targets)
if shuffle:
indices = np.arange(len(inputs))
np.random.shuffle(indices)
for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):
if shuffle:
excerpt = indices[start_idx:start_idx + batchsize]
else:
excerpt = slice(start_idx, start_idx + batchsize)
yield inputs[excerpt], targets[excerpt]
print("Loading data...")
X_train, y_train, X_val, y_val = load.loadDataset()
input_var = T.tensor4('inputs')
target_var = T.tensor4('target')
print("Building model and compiling functions...")
batchsize = 128
# Network
#network = Model.OneLayerMLP(batchsize, input_var)
network = model.simpleConv(input_var)
# Loss Function
prediction = lasagne.layers.get_output(network)
loss = T.mean(lasagne.objectives.squared_error(prediction, target_var))
