def load_ct_dataset(self, dataset, is_train=True,is_registered=False,grayscale_corrected=True):
'''
dataset: list of dataset such as ['train\GB1', 'train\B1M6']
'''
if is_train:
print('Loading train ct datasets...')
train_set = utils.get_training_set(self.data_dir,
dataset,
self.crop_size,
self.scale_factor,
registered=is_registered,
grayscale_corrected=grayscale_corrected)
return DataLoader(dataset=train_set, num_workers=self.num_threads, batch_size=self.batch_size,
shuffle=True)
else:
print('Loading test ct datasets...')
test_set = utils.get_test_set(self.data_dir,
dataset,
self.crop_size,
self.scale_factor,
registered=is_registered,
grayscale_corrected=grayscale_corrected)
return DataLoader(dataset=test_set, num_workers=self.num_threads,
batch_size=self.test_batch_size,
shuffle=False)
def train(model,
X,
Y,
test_split=0.1,
epochs=20,
learning_rate=10**(-5),
batch_size=8,
savepath=None,
save_only_best=True):
if type(model) == str and os.path.isfile(model):
model = load_model(model)
if type(X) == str and os.path.isdir(X) and type(
Y) == str and os.path.isdir(Y):
X, Y = utils.get_training_set(X, Y)
random_idxes = [
np.random.choice(np.array(range(len(X))), replcae=False)
for _ in range(int(len(X) * test_split))
]
X_test, Y_test = X[random_idxes], Y[random_idxes]
X_train = np.delete(X, random_idxes, axis=0)
Y_train = np.delete(X, random_idxes, axis=0)
model.fit(X_train,
Y_train,
validation_data=(X_test, Y_test),
shuffle=True,
epochs=epochs,
batch_size=batch_size)
with open('positive_set.json', 'r') as f:
positive_set, positive_meta = parse_dataset(json.load(f))
with open('negative_set.json', 'r') as f:
negative_set, negative_meta = parse_dataset(json.load(f))
seed = 1
methods = [
('baseline', run_baseline, 'navy'),
('naive bayes', run_nb, 'turquoise'),
('random forest', run_rf, 'darkorange'),
('svm', run_svm, 'cornflowerblue'),
('neural net', run_nnet, 'teal'),
]
X, y, meta = get_training_set(positive_set, positive_meta, negative_set,
negative_meta, seed)
groups = meta[:, 1] # omim id
plt.clf()
for i, (method, run_method, color) in enumerate(methods):
logging.info('Running method: {}'.format(method))
y_test, y_score = run_with_cv(run_method, X, y, groups, seed)
fpr, tpr, thresholds = roc_curve(y_test, y_score, pos_label=1)
auc_score = auc(fpr, tpr)
plt.plot(fpr,
tpr,
lw=2,
color=color,
label='ROC curve of {} (area = {:0.2f}'.format(
method, auc_score))
from inputs import inputs
from models import SegNetAutoencoder
from scalar_ops import accuracy, loss
from tqdm import tqdm
import classifier
import config
import tensorflow as tf
import utils
train_file, train_labels_file = utils.get_training_set(config.working_dataset)
tf.app.flags.DEFINE_string('ckpt_dir', './ckpts', 'Train checkpoint directory')
tf.app.flags.DEFINE_string('train', train_file, 'Train data')
tf.app.flags.DEFINE_string('train_ckpt', './ckpts/model.ckpt', 'Train checkpoint file')
tf.app.flags.DEFINE_string('train_labels', train_labels_file, 'Train labels data')
tf.app.flags.DEFINE_string('train_logs', './logs/train', 'Log directory')
tf.app.flags.DEFINE_boolean('strided', True, 'Use strided convolutions and deconvolutions')
tf.app.flags.DEFINE_integer('summary_step', 10, 'Number of iterations before serializing log data')
tf.app.flags.DEFINE_integer('batch', 12, 'Batch size')
tf.app.flags.DEFINE_integer('steps', 10000, 'Number of training iterations')
FLAGS = tf.app.flags.FLAGS
def train():
images, labels = inputs(FLAGS.batch, FLAGS.train, FLAGS.train_labels)
tf.summary.image('labels', labels)
one_hot_labels = classifier.one_hot(labels)
type=int,
default=4,
help='Number of thread for DataLoader')
opt = parser.parse_args()
print(opt)
if isCuda and not torch.cuda.is_available():
raise Exception("No GPU, please change isCuda False")
device = torch.device("cuda" if isCuda else "cpu")
print('===> Loading datasets')
train_set = get_training_set(opt.upscale_factor, opt.datapath)
test_set = get_test_set(opt.upscale_factor, opt.datapath)
training_data_loader = DataLoader(dataset=train_set,
num_workers=opt.threads,
batch_size=batchSize,
shuffle=True)
testing_data_loader = DataLoader(dataset=test_set,
num_workers=opt.threads,
batch_size=testBatchSize,
shuffle=False)
print('===> Datasets Loading Complete')
print('===> Model Initialize')
use_calcium_stats=args.use_calcium_stats)
# Handles model loading / rebuilding.
if args.rebuild_model:
if os.path.exists(args.model_location):
os.remove(args.model_location)
else:
if os.path.exists(args.model_location):
model.load_weights(args.model_location)
print('Loaded weights from "%s".' % args.model_location)
else:
print('No weights found at "%s".' % args.model_location)
# Gets the training data.
dataset, calcium, calcium_stats, spikes = utils.get_training_set(
buffer_length=args.buffer_length,
num_timesteps=args.num_timesteps,
rebuild=args.rebuild_data)
# Cuts off the beginning and end bits.
spikes = spikes[:, args.buffer_length:-args.buffer_length]
# Builds the inputs according to the user's specifications.
inputs = [calcium]
if args.use_calcium_stats:
inputs.append(calcium_stats)
if args.use_dataset:
inputs.append(dataset)
# Splits into training and validation sets.
def _split(x):
x = zip(*[(i[:-args.num_val], i[-args.num_val:]) for i in x])