def main(args):
slide_fn = args.slide_fn
level = args.level
## Load model
net = models.tissue_detector_DNN()
eddl.build(net, eddl.rmsprop(0.00001), ["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU() if args.gpu else eddl.CS_CPU())
eddl.load(net, args.weights_fn, "bin")
eddl.summary(net)
## Load Slide
slide_T, s = read_slide(slide_fn, level)
## Compute tissue mask
#len_T = slide_T.getShape()[0]
#bs = args.batch_size
#nb = int(np.ceil((len_T / bs)))
#print ("n. batches: %d" % nb)
#output_l = []
#for b in range(nb):
# start = bs*b
# stop = bs*(b+1) if bs*(b+1) < len_T else len_T
# b_T = slide_T.select(["%d:%d" % (start, stop)])
output_l = eddl.predict(net, [slide_T])
print(output_l)
## Binarize the output
mask = get_mask(output_l, s, args.threshold)
np.save("mask", mask)
def main(args):
if not os.path.isfile(args.input):
raise RuntimeError("input file '%s' not found" % args.input)
eddl.download_mnist()
print("importing net from", args.input)
net = eddl.import_net_from_onnx_file(args.input)
print("input.shape:", net.layers[0].input.shape)
print("output size =", len(net.lout))
eddl.build(
net,
eddl.rmsprop(0.01),
["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU(mem=args.mem) if args.gpu else eddl.CS_CPU(mem=args.mem),
False # do not initialize weights to random values
)
net.resize(args.batch_size) # resize manually since we don't use "fit"
eddl.summary(net)
x_test = Tensor.load("mnist_tsX.bin")
y_test = Tensor.load("mnist_tsY.bin")
x_test.div_(255.0)
eddl.evaluate(net, [x_test], [y_test], bs=args.batch_size)
print("All done")
def main(args):
if not os.path.isfile(args.input):
raise RuntimeError("input file '%s' not found" % args.input)
eddl.download_mnist()
net = eddl.import_net_from_onnx_file(args.input)
eddl.build(
net,
eddl.rmsprop(0.01),
["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU(mem=args.mem) if args.gpu else eddl.CS_CPU(mem=args.mem),
False # do not initialize weights to random values
)
net.resize(args.batch_size) # resize manually since we don't use "fit"
eddl.summary(net)
x_test = Tensor.load("mnist_tsX.bin")
x_test.div_(255.0)
sys.stderr.write("forward...\n")
eddl.forward(net, [x_test])
sys.stderr.write("forward done\n")
sys.stderr.write("lout: %r\n" % (net.lout, ))
out = eddl.getOut(net)
sys.stderr.write("getOut done\n")
sys.stderr.write("out: %r\n" % (out, ))
print("All done")
def get_net(net_name='vgg16',
in_size=[256, 256],
num_classes=2,
lr=1e-5,
augs=False,
gpus=[1],
lsb=1,
init=eddl.HeNormal,
dropout=None,
l2_reg=None):
## Network definition
in_ = eddl.Input([3, in_size[0], in_size[1]])
if net_name == 'vgg16':
out = models.VGG16_promort(in_,
num_classes,
init=init,
l2_reg=l2_reg,
dropout=dropout)
else:
print('model %s not available' % net_name)
sys.exit(-1)
net = eddl.Model([in_], [out])
eddl.build(
net,
eddl.rmsprop(lr),
["soft_cross_entropy"],
["categorical_accuracy"],
#eddl.CS_GPU([1,1], mem="low_mem") if gpu else eddl.CS_CPU()
eddl.CS_GPU(gpus, mem="low_mem", lsb=lsb) if gpus else eddl.CS_CPU()
#eddl.CS_GPU(gpus, mem="low_mem") if gpus else eddl.CS_CPU()
)
eddl.summary(net)
eddl.setlogfile(net, "promort_VGG16_classification")
if augs:
## Set augmentations
training_augs = ecvl.SequentialAugmentationContainer(
[ecvl.AugMirror(.5),
ecvl.AugFlip(.5),
ecvl.AugRotate([-10, 10])])
validation_augs = ecvl.SequentialAugmentationContainer([])
dataset_augs = [training_augs, validation_augs, None]
else:
dataset_augs = [None, None, None]
return net, dataset_augs
def load_model(self, model_weights):
## Load the ANN tissue detector model implemented by using pyeddl
## Create ANN topology (Same used during training phase)
## The model is assumed being trained by using RGB triplets.
net = models.tissue_detector_DNN()
eddl.build(net, eddl.rmsprop(0.00001), ["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU() if self.gpu else eddl.CS_CPU())
# Load weights
eddl.load(net, model_weights, "bin")
self.model = net
def main(args):
eddl.download_mnist()
num_classes = 10
in_ = eddl.Input([784])
layer = in_
layer = eddl.Reshape(layer, [1, 784]) # image as a 1D signal with depth 1
layer = eddl.MaxPool1D(eddl.ReLu(eddl.Conv1D(layer, 16, [3], [1])), [4],
[4])
layer = eddl.MaxPool1D(
eddl.ReLu(eddl.Conv1D(layer, 32, [3], [1])),
[4],
[4],
)
layer = eddl.MaxPool1D(
eddl.ReLu(eddl.Conv1D(layer, 64, [3], [1])),
[4],
[4],
)
layer = eddl.MaxPool1D(
eddl.ReLu(eddl.Conv1D(layer, 64, [3], [1])),
[4],
[4],
)
layer = eddl.Reshape(layer, [-1])
out = eddl.Softmax(eddl.Dense(layer, num_classes))
net = eddl.Model([in_], [out])
eddl.build(
net, eddl.rmsprop(0.01), ["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU(mem=args.mem) if args.gpu else eddl.CS_CPU(mem=args.mem))
eddl.summary(net)
x_train = Tensor.load("mnist_trX.bin")
y_train = Tensor.load("mnist_trY.bin")
x_test = Tensor.load("mnist_tsX.bin")
y_test = Tensor.load("mnist_tsY.bin")
if args.small:
x_train = x_train.select([":6000"])
y_train = y_train.select([":6000"])
x_test = x_test.select([":1000"])
y_test = y_test.select([":1000"])
x_train.div_(255.0)
x_test.div_(255.0)
eddl.fit(net, [x_train], [y_train], args.batch_size, args.epochs)
eddl.evaluate(net, [x_test], [y_test], bs=args.batch_size)
print("All done")
def get_net(in_size=[256, 256], num_classes=2, lr=1e-5, gpu=True):
## Network definition
in_ = eddl.Input([3, in_size[0], in_size[1]])
out = models.VGG16_promort(in_, num_classes)
net = eddl.Model([in_], [out])
eddl.build(
net,
eddl.rmsprop(lr),
["soft_cross_entropy"],
["categorical_accuracy"],
#eddl.CS_GPU([1,1], mem="low_mem") if gpu else eddl.CS_CPU()
eddl.CS_GPU([1], mem="low_mem") if gpu else eddl.CS_CPU())
return net
def main(args):
eddl.download_mnist()
num_classes = 10
in_ = eddl.Input([784])
layer = in_
layer = eddl.Reshape(layer, [-1])
layer = eddl.ReLu(eddl.Dense(layer, 1024))
layer = eddl.BatchNormalization(layer, True)
layer = eddl.ReLu(eddl.Dense(layer, 1024))
layer = eddl.BatchNormalization(layer, True)
layer = eddl.ReLu(eddl.Dense(layer, 1024))
layer = eddl.BatchNormalization(layer, True)
out = eddl.Softmax(eddl.Dense(layer, num_classes))
net = eddl.Model([in_], [out])
eddl.build(
net,
eddl.rmsprop(0.01),
["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU(mem=args.mem) if args.gpu else eddl.CS_CPU(mem=args.mem),
True # initialize weights to random values
)
eddl.summary(net)
x_train = Tensor.load("mnist_trX.bin")
y_train = Tensor.load("mnist_trY.bin")
x_test = Tensor.load("mnist_tsX.bin")
y_test = Tensor.load("mnist_tsY.bin")
if args.small:
x_train = x_train.select([":6000"])
y_train = y_train.select([":6000"])
x_test = x_test.select([":1000"])
y_test = y_test.select([":1000"])
x_train.div_(255.0)
x_test.div_(255.0)
eddl.fit(net, [x_train], [y_train], args.batch_size, args.epochs)
eddl.evaluate(net, [x_test], [y_test], bs=args.batch_size)
eddl.save_net_to_onnx_file(net, args.output)
print("saved net to", args.output)
print("All done")
def main(args):
eddl.download_mnist()
num_classes = 10
in_ = eddl.Input([28])
layer = in_
layer = eddl.LeakyReLu(eddl.Dense(layer, 32))
layer = eddl.L2(eddl.LSTM(layer, 128), 0.001)
ls = layer
out = eddl.Softmax(eddl.Dense(layer, num_classes))
net = eddl.Model([in_], [out])
eddl.build(
net, eddl.rmsprop(0.001), ["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU(mem=args.mem) if args.gpu else eddl.CS_CPU(mem=args.mem))
eddl.summary(net)
eddl.plot(net, "model.pdf")
x_train = Tensor.load("mnist_trX.bin")
y_train = Tensor.load("mnist_trY.bin")
x_test = Tensor.load("mnist_tsX.bin")
y_test = Tensor.load("mnist_tsY.bin")
if args.small:
x_train = x_train.select([":6000"])
y_train = y_train.select([":6000"])
x_test = x_test.select([":1000"])
y_test = y_test.select([":1000"])
x_train.reshape_([x_train.shape[0], 28, 28])
x_test.reshape_([x_test.shape[0], 28, 28])
y_train.reshape_([y_train.shape[0], 1, 10])
y_test.reshape_([y_test.shape[0], 1, 10])
x_train.div_(255.0)
x_test.div_(255.0)
for i in range(args.epochs):
eddl.fit(net, [x_train], [y_train], args.batch_size, 1)
eddl.evaluate(net, [x_test], [y_test], bs=args.batch_size)
ls_in = eddl.getInput(ls)
ls_in.info()
ls_out = eddl.getOutput(ls)
ls_out.info()
print("All done")
def main(args):
## Read input dataset
x_train, y_train, x_test, y_test = read_input(args.in_ds)
## Net architecture
net = models.tissue_detector_DNN()
## Net compilation
eddl.build(
net,
eddl.rmsprop(0.00001),
["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU() if args.gpu else eddl.CS_CPU()
)
eddl.summary(net)
## Fit and evaluation
eddl.fit(net, [x_train], [y_train], args.batch_size, args.epochs)
eddl.evaluate(net, [x_test], [y_test])
eddl.save(net, "tissue_detector_model.bin")
def main(args):
eddl.download_mnist()
num_classes = 10
in_ = eddl.Input([784])
layer = in_
layer = eddl.LeakyReLu(eddl.Dense(layer, 1024))
layer = eddl.LeakyReLu(eddl.Dense(layer, 1024))
layer = eddl.LeakyReLu(eddl.Dense(layer, 1024))
out = eddl.Softmax(eddl.Dense(layer, num_classes), -1)
net = eddl.Model([in_], [out])
eddl.build(
net, eddl.rmsprop(0.01), ["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU(mem=args.mem) if args.gpu else eddl.CS_CPU(mem=args.mem))
eddl.summary(net)
eddl.plot(net, "model.pdf")
x_train = Tensor.load("mnist_trX.bin")
y_train = Tensor.load("mnist_trY.bin")
x_test = Tensor.load("mnist_tsX.bin")
y_test = Tensor.load("mnist_tsY.bin")
if args.small:
x_train = x_train.select([":6000"])
y_train = y_train.select([":6000"])
x_test = x_test.select([":1000"])
y_test = y_test.select([":1000"])
x_train.div_(255.0)
x_test.div_(255.0)
eddl.fit(net, [x_train], [y_train], args.batch_size, args.epochs)
eddl.evaluate(net, [x_test], [y_test], bs=args.batch_size)
print("All done")
def main(args):
eddl.download_mnist()
num_classes = 10
in_ = eddl.Input([784])
layer = in_
layer = eddl.Reshape(layer, [-1])
layer = eddl.ReLu(eddl.Dense(layer, 1024))
layer = eddl.ReLu(eddl.Dense(layer, 1024))
layer = eddl.ReLu(eddl.Dense(layer, 1024))
out = eddl.Softmax(eddl.Dense(layer, num_classes))
net = eddl.Model([in_], [out])
eddl.build(
net,
eddl.rmsprop(0.01),
["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU(mem=args.mem) if args.gpu else eddl.CS_CPU(mem=args.mem),
True # initialize weights to random values
)
serialized_net = eddl.serialize_net_to_onnx_string(net, False)
eddl.summary(net)
x_train = Tensor.load("mnist_trX.bin")
y_train = Tensor.load("mnist_trY.bin")
x_test = Tensor.load("mnist_tsX.bin")
y_test = Tensor.load("mnist_tsY.bin")
if args.small:
x_train = x_train.select([":6000"])
y_train = y_train.select([":6000"])
x_test = x_test.select([":1000"])
y_test = y_test.select([":1000"])
x_train.div_(255.0)
x_test.div_(255.0)
eddl.fit(net, [x_train], [y_train], args.batch_size, args.epochs)
print("evaluating before import")
eddl.evaluate(net, [x_test], [y_test], bs=args.batch_size)
imported_net = eddl.import_net_from_onnx_string(serialized_net)
eddl.build(
imported_net,
eddl.rmsprop(0.01),
["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU(mem=args.mem) if args.gpu else eddl.CS_CPU(mem=args.mem),
False # do not initialize weights to random values
)
eddl.summary(imported_net)
print("net layers:", len(net.layers))
print("imported_net layers:", len(imported_net.layers))
print("evaluating imported net")
eddl.evaluate(imported_net, [x_test], [y_test], bs=args.batch_size)
print("All done")
def main(args):
num_classes = 2
size = [256, 256] # size of images
in_ = eddl.Input([3, size[0], size[1]])
out = models.VGG16_promort(in_, num_classes)
net = eddl.Model([in_], [out])
eddl.build(
net,
eddl.rmsprop(1e-6),
#eddl.sgd(0.001, 0.9),
["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU([1], mem="low_mem") if args.gpu else eddl.CS_CPU()
)
eddl.summary(net)
eddl.setlogfile(net, "promort_VGG16_classification")
training_augs = ecvl.SequentialAugmentationContainer([
ecvl.AugResizeDim(size)
#ecvl.AugMirror(.5),
#ecvl.AugFlip(.5),
#ecvl.AugRotate([-180, 180]),
#ecvl.AugAdditivePoissonNoise([0, 10]),
#ecvl.AugGammaContrast([0.5, 1.5]),
#ecvl.AugGaussianBlur([0, 0.8]),
#ecvl.AugCoarseDropout([0, 0.3], [0.02, 0.05], 0.5)
])
validation_augs = ecvl.SequentialAugmentationContainer([
ecvl.AugResizeDim(size),
])
dataset_augs = ecvl.DatasetAugmentations(
[training_augs, validation_augs, None]
)
print("Reading dataset")
#d = ecvl.DLDataset(args.in_ds, args.batch_size)
d = ecvl.DLDataset(args.in_ds, args.batch_size, dataset_augs)
x = Tensor([args.batch_size, d.n_channels_, size[0], size[1]])
y = Tensor([args.batch_size, len(d.classes_)])
num_samples_train = len(d.GetSplit())
num_batches_train = num_samples_train // args.batch_size
d.SetSplit(ecvl.SplitType.validation)
num_samples_val = len(d.GetSplit())
num_batches_val = num_samples_val // args.batch_size
indices = list(range(args.batch_size))
metric = eddl.getMetric("categorical_accuracy")
print("Starting training")
### Main loop across epochs
for e in range(args.epochs):
print("Epoch {:d}/{:d} - Training".format(e + 1, args.epochs),
flush=True)
if args.out_dir:
current_path = os.path.join(args.out_dir, "Epoch_%d" % e)
for c in d.classes_:
c_dir = os.path.join(current_path, c)
os.makedirs(c_dir, exist_ok=True)
d.SetSplit(ecvl.SplitType.training)
eddl.reset_loss(net)
total_metric = []
s = d.GetSplit()
random.shuffle(s)
d.split_.training_ = s
d.ResetAllBatches()
### Looping across batches of training data
for b in range(num_batches_train):
print("Epoch {:d}/{:d} (batch {:d}/{:d}) - ".format(
e + 1, args.epochs, b + 1, num_batches_train
), end="", flush=True)
d.LoadBatch(x, y)
x.div_(255.0)
tx, ty = [x], [y]
#print (tx[0].info())
eddl.train_batch(net, tx, ty, indices)
#eddl.print_loss(net, b)
instances = (b+1) * args.batch_size
print ("loss = %.3f, acc = %.3f" % (net.fiterr[0]/instances, net.fiterr[1]/instances))
#print()
print("Saving weights")
eddl.save(net, "promort_checkpoint_%s.bin" % e, "bin")
### Evaluation on validation set
print("Epoch %d/%d - Evaluation" % (e + 1, args.epochs), flush=True)
d.SetSplit(ecvl.SplitType.validation)
for b in range(num_batches_val):
n = 0
print("Epoch {:d}/{:d} (batch {:d}/{:d}) - ".format(
e + 1, args.epochs, b + 1, num_batches_val
), end="", flush=True)
d.LoadBatch(x, y)
x.div_(255.0)
eddl.forward(net, [x])
output = eddl.getOutput(out)
sum_ = 0.0
for k in range(args.batch_size):
result = output.select([str(k)])
target = y.select([str(k)])
ca = metric.value(target, result)
total_metric.append(ca)
sum_ += ca
if args.out_dir:
result_a = np.array(result, copy=False)
target_a = np.array(target, copy=False)
classe = np.argmax(result_a).item()
gt_class = np.argmax(target_a).item()
single_image = x.select([str(k)])
img_t = ecvl.TensorToView(single_image)
img_t.colortype_ = ecvl.ColorType.BGR
single_image.mult_(255.)
filename = d.samples_[d.GetSplit()[n]].location_[0]
head, tail = os.path.splitext(os.path.basename(filename))
bname = "%s_gt_class_%s.png" % (head, gt_class)
cur_path = os.path.join(
current_path, d.classes_[classe], bname
)
ecvl.ImWrite(cur_path, img_t)
n += 1
print("categorical_accuracy:", sum_ / args.batch_size)
total_avg = sum(total_metric) / len(total_metric)
print("Total categorical accuracy:", total_avg)
def test_dataset():
num_classes = 2
size = [256, 256] # size of images
training_augs = ecvl.SequentialAugmentationContainer([
#ecvl.AugResizeDim(size),
ecvl.AugMirror(.5),
ecvl.AugFlip(.5),
ecvl.AugRotate([-180, 180]),
#ecvl.AugAdditivePoissonNoise([0, 10]),
#ecvl.AugGammaContrast([0.5, 1.5]),
#ecvl.AugGaussianBlur([0, 0.8]),
#ecvl.AugCoarseDropout([0, 0.3], [0.02, 0.05], 0.5),
])
dataset_augs = [training_augs, None, None]
in_ = eddl.Input([3, size[0], size[1]])
out = VGG16(in_, num_classes)
net = eddl.Model([in_], [out])
eddl.build(
net,
eddl.rmsprop(1e-5),
#eddl.sgd(0.001, 0.9),
["soft_cross_entropy"],
["categorical_accuracy"],
eddl.CS_GPU([1], mem="low_mem") # eddl.CS_CPU()
)
eddl.summary(net)
# Read Cassandra password
try:
from private_data import cass_pass
except ImportError:
cass_pass = getpass('Insert Cassandra password: '******'prom', password=cass_pass)
cd = CassandraDataset(ap, ['cassandra_db'])
# Flow 0: read rows from db, create splits and save everything
# Level 0
cd.init_listmanager(table='promort.ids_osk_0',
id_col='patch_id',
split_ncols=1,
num_classes=2,
metatable='promort.metadata_osk_0',
partition_cols=['sample_name', 'sample_rep', 'label'])
cd.read_rows_from_db()
cd.save_rows('/tmp/osk_0_rows.pckl')
cd.init_datatable(table='promort.data_osk_0')
cd.split_setup(batch_size=32,
split_ratios=[7, 2, 1],
max_patches=120000,
augs=[])
cd.save_splits('/tmp/osk_0_splits.pckl')
## Flow 1: read saved rows, create and save splits
#cd.load_rows('/tmp/rows.pckl')
#cd.init_datatable(table='promort.data_osk_0')
#cd.split_setup(batch_size=32, split_ratios=[7,2,1],
# max_patches=120000, augs=[])
#cd.save_splits('/tmp/osk_0_splits.pckl')
## Flow 2: read saved splits
#cd.load_splits('/tmp/osk_0_splits.pckl', batch_size=32, augs=[])
## fit generator
cassandra_fit(cd, net, epochs=1)
# change batch size
cd.set_batchsize(16)
cassandra_fit(cd, net, epochs=1)
# change partitioning and balance
cd.split_setup(max_patches=1000, split_ratios=[10, 1, 1], balance=[2, 1])
cassandra_fit(cd, net, epochs=1)