def load_ct_dataset(self, dataset, is_train=True,is_registered=False,grayscale_corrected=True): ''' dataset: list of dataset such as ['train\GB1', 'train\B1M6'] ''' if is_train: print('Loading train ct datasets...') train_set = utils.get_training_set(self.data_dir, dataset, self.crop_size, self.scale_factor, registered=is_registered, grayscale_corrected=grayscale_corrected) return DataLoader(dataset=train_set, num_workers=self.num_threads, batch_size=self.batch_size, shuffle=True) else: print('Loading test ct datasets...') test_set = utils.get_test_set(self.data_dir, dataset, self.crop_size, self.scale_factor, registered=is_registered, grayscale_corrected=grayscale_corrected) return DataLoader(dataset=test_set, num_workers=self.num_threads, batch_size=self.test_batch_size, shuffle=False)
def train(model, X, Y, test_split=0.1, epochs=20, learning_rate=10**(-5), batch_size=8, savepath=None, save_only_best=True): if type(model) == str and os.path.isfile(model): model = load_model(model) if type(X) == str and os.path.isdir(X) and type( Y) == str and os.path.isdir(Y): X, Y = utils.get_training_set(X, Y) random_idxes = [ np.random.choice(np.array(range(len(X))), replcae=False) for _ in range(int(len(X) * test_split)) ] X_test, Y_test = X[random_idxes], Y[random_idxes] X_train = np.delete(X, random_idxes, axis=0) Y_train = np.delete(X, random_idxes, axis=0) model.fit(X_train, Y_train, validation_data=(X_test, Y_test), shuffle=True, epochs=epochs, batch_size=batch_size)
with open('positive_set.json', 'r') as f: positive_set, positive_meta = parse_dataset(json.load(f)) with open('negative_set.json', 'r') as f: negative_set, negative_meta = parse_dataset(json.load(f)) seed = 1 methods = [ ('baseline', run_baseline, 'navy'), ('naive bayes', run_nb, 'turquoise'), ('random forest', run_rf, 'darkorange'), ('svm', run_svm, 'cornflowerblue'), ('neural net', run_nnet, 'teal'), ] X, y, meta = get_training_set(positive_set, positive_meta, negative_set, negative_meta, seed) groups = meta[:, 1] # omim id plt.clf() for i, (method, run_method, color) in enumerate(methods): logging.info('Running method: {}'.format(method)) y_test, y_score = run_with_cv(run_method, X, y, groups, seed) fpr, tpr, thresholds = roc_curve(y_test, y_score, pos_label=1) auc_score = auc(fpr, tpr) plt.plot(fpr, tpr, lw=2, color=color, label='ROC curve of {} (area = {:0.2f}'.format( method, auc_score))
from inputs import inputs from models import SegNetAutoencoder from scalar_ops import accuracy, loss from tqdm import tqdm import classifier import config import tensorflow as tf import utils train_file, train_labels_file = utils.get_training_set(config.working_dataset) tf.app.flags.DEFINE_string('ckpt_dir', './ckpts', 'Train checkpoint directory') tf.app.flags.DEFINE_string('train', train_file, 'Train data') tf.app.flags.DEFINE_string('train_ckpt', './ckpts/model.ckpt', 'Train checkpoint file') tf.app.flags.DEFINE_string('train_labels', train_labels_file, 'Train labels data') tf.app.flags.DEFINE_string('train_logs', './logs/train', 'Log directory') tf.app.flags.DEFINE_boolean('strided', True, 'Use strided convolutions and deconvolutions') tf.app.flags.DEFINE_integer('summary_step', 10, 'Number of iterations before serializing log data') tf.app.flags.DEFINE_integer('batch', 12, 'Batch size') tf.app.flags.DEFINE_integer('steps', 10000, 'Number of training iterations') FLAGS = tf.app.flags.FLAGS def train(): images, labels = inputs(FLAGS.batch, FLAGS.train, FLAGS.train_labels) tf.summary.image('labels', labels) one_hot_labels = classifier.one_hot(labels)
type=int, default=4, help='Number of thread for DataLoader') opt = parser.parse_args() print(opt) if isCuda and not torch.cuda.is_available(): raise Exception("No GPU, please change isCuda False") device = torch.device("cuda" if isCuda else "cpu") print('===> Loading datasets') train_set = get_training_set(opt.upscale_factor, opt.datapath) test_set = get_test_set(opt.upscale_factor, opt.datapath) training_data_loader = DataLoader(dataset=train_set, num_workers=opt.threads, batch_size=batchSize, shuffle=True) testing_data_loader = DataLoader(dataset=test_set, num_workers=opt.threads, batch_size=testBatchSize, shuffle=False) print('===> Datasets Loading Complete') print('===> Model Initialize')
use_calcium_stats=args.use_calcium_stats) # Handles model loading / rebuilding. if args.rebuild_model: if os.path.exists(args.model_location): os.remove(args.model_location) else: if os.path.exists(args.model_location): model.load_weights(args.model_location) print('Loaded weights from "%s".' % args.model_location) else: print('No weights found at "%s".' % args.model_location) # Gets the training data. dataset, calcium, calcium_stats, spikes = utils.get_training_set( buffer_length=args.buffer_length, num_timesteps=args.num_timesteps, rebuild=args.rebuild_data) # Cuts off the beginning and end bits. spikes = spikes[:, args.buffer_length:-args.buffer_length] # Builds the inputs according to the user's specifications. inputs = [calcium] if args.use_calcium_stats: inputs.append(calcium_stats) if args.use_dataset: inputs.append(dataset) # Splits into training and validation sets. def _split(x): x = zip(*[(i[:-args.num_val], i[-args.num_val:]) for i in x])