def classify(self, image, x, y, z):
import PyGreentea
if len(self.network_map) == 0:
device_id = int(os.environ.get(MICRONS_IPC_WORKER_GPU, "0"))
PyGreentea.caffe.set_device(device_id)
PyGreentea.caffe.set_mode_gpu()
key = (self.model_path, self.proto_path)
if key not in self.network_map:
net = PyGreentea.caffe.Net(self.proto_path,
self.model_path,
PyGreentea.caffe.TEST)
self.network_map[key] = net
else:
net = self.network_map[key]
if self.xpad == 0 and self.ypad == 0 and self.zpad == 0:
zero_pad_source_data = True
else:
zero_pad_source_data = False
image = normalize_image(image, self.normalize_method,
saturation_level=self.saturation_level,
offset=self.offset)
dataset = [dict(name="Volume-%d-%d-%d" % (x, y, z),
data=image[np.newaxis, :])]
output = PyGreentea.process(
net, dataset,
zero_pad_source_data=zero_pad_source_data)[0]
result = dict(
[(k, (o[self.zpad:o.shape[0] - self.zpad,
self.ypad:o.shape[1] - self.ypad,
self.xpad:o.shape[2] - self.xpad] * 255).astype(np.uint8))
for k, o in zip(self.class_names, output)])
return result
def processFile(basename,
iter,
outputName="data_tier2/tstvol-520-1-h5",
train=True):
print('processing...', iter)
if not os.path.exists(outputName):
os.makedirs(outputName)
# Load the datasets
path = '/groups/turaga/home/turagas/data/FlyEM/fibsem_medulla_7col/'
# Test set
test_dataset = []
if train:
test_dataset.append({})
test_dataset[-1]['name'] = outputName
h5im = h5py.File(join(path, "tstvol-520-1-h5", 'img_normalized.h5'),
'r')
h5im_n = pygt.normalize(
np.asarray(h5im[h5im.keys()[0]]).astype(float32), -1, 1)
test_dataset[-1]['data'] = h5im_n
if not train:
test_dataset.append({})
test_dataset[-1]['name'] = outputName
h5im = h5py.File(join(path, "tstvol-520-2-h5", 'img_normalized.h5'),
'r')
h5im_n = pygt.normalize(
np.asarray(h5im[h5im.keys()[0]]).astype(float32), -1, 1)
test_dataset[-1]['data'] = h5im_n
# Set devices
test_device = 2
print('Setting devices...')
pygt.caffe.set_mode_gpu()
pygt.caffe.set_device(test_device)
# pygt.caffe.select_device(test_device, False)
# Load model
proto = basename + 'net_test.prototxt'
model = basename + 'net_iter_' + str(iter) + '.caffemodel'
print('Loading model...')
net = pygt.caffe.Net(proto, model, pygt.caffe.TEST)
# Process
print('Processing...')
pygt.process(net, test_dataset)
print('Processing Complete')
def process_array_with_net(net, array_like, target):
import h5py
import PyGreentea as pygt
if array_like is None:
input_data = h5py.File(
"/groups/turaga/home/turagas/data/FlyEM/fibsem_medulla_7col/tstvol-520-2-h5/im_uint8.h5"
)["main"]
dataset = dict(data=input_data,
name="test",
image_scaling_factor=0.5**8)
preds = pygt.process(net, [dataset], target_arrays=[target])
else:
input_data = array_like
dataset = dict(data=input_data,
name="test",
image_scaling_factor=0.5**8,
image_is_zero_padded=True,
region_offset=array_like.offset)
preds = pygt.process(net, [dataset], target_arrays=[target])
return preds[0]
def test(args):
print('testing...')
test_dataset, _ = Data.get(args.data_path,
args.seg_path,
args.data_name,
args.seg_name,
augment=(args.augment == 1))
modelfile = '%s/net_iter_%d.caffemodel' % (args.output, args.iteration)
modelproto = '%s/net.prototxt' % (args.output)
print('modelfile:', modelfile)
print('modelproto:', modelproto)
# Set devices
test_device = 0
print('Setting devices...')
pygt.caffe.set_mode_gpu()
pygt.caffe.set_device(args.test_device)
# Load model
print('Loading model...')
net = pygt.caffe.Net(modelproto, modelfile, pygt.caffe.TEST)
start_time = time.time()
# Process
print('Processing ' + str(len(test_dataset)) + ' volume(s)...')
preds = pygt.process(net, test_dataset)
end_time = time.time()
print("elapsed seconds {0} for z ,y ,x ={1}, {2}, {3}".format(
end_time - start_time, preds[0].shape[1], preds[0].shape[2],
preds[0].shape[3]))
for i in range(0, len(test_dataset), 1):
print('Saving ' + test_dataset[i]['name'])
h5file = '%s/%s_%d-pred.h5' % (args.output,
test_dataset[i]['name'], i)
outhdf5 = h5py.File(h5file, 'w')
outdset = outhdf5.create_dataset('main',
preds[i].shape,
np.float32,
data=preds[i])
outhdf5.close()
h5file = '%s/%s_%d-mask.h5' % (args.output,
test_dataset[i]['name'], i)
outhdf5 = h5py.File(h5file, 'w')
outdset = outhdf5.create_dataset('main',
preds[i + 1].shape,
np.uint8,
data=preds[i + 1])
outhdf5.close()
def processFile(basename,iter,outputName = "data_tier2/tstvol-520-1-h5",train=True):
print('processing...',iter)
if not os.path.exists(outputName):
os.makedirs(outputName)
# Load the datasets
path = '/groups/turaga/home/turagas/data/FlyEM/fibsem_medulla_7col/'
# Test set
test_dataset = []
if train:
test_dataset.append({})
test_dataset[-1]['name'] = outputName
h5im = h5py.File(join(path,"tstvol-520-1-h5",'img_normalized.h5'),'r')
h5im_n = pygt.normalize(np.asarray(h5im[h5im.keys()[0]]).astype(float32), -1, 1)
test_dataset[-1]['data'] = h5im_n
if not train:
test_dataset.append({})
test_dataset[-1]['name'] = outputName
h5im = h5py.File(join(path,"tstvol-520-2-h5",'img_normalized.h5'),'r')
h5im_n = pygt.normalize(np.asarray(h5im[h5im.keys()[0]]).astype(float32), -1, 1)
test_dataset[-1]['data'] = h5im_n
# Set devices
test_device = 0
print('Setting devices...')
pygt.caffe.set_mode_gpu()
pygt.caffe.set_device(test_device)
# pygt.caffe.select_device(test_device, False)
# Load model
proto = basename + 'net_test.prototxt'
model = basename + 'net_iter_'+str(iter)+'.caffemodel'
print('Loading model...')
net = pygt.caffe.Net(proto, model, pygt.caffe.TEST)
# Process
print('Processing...')
pygt.process(net,test_dataset)
print('Processing Complete')
def process(modelpath,iter,outputpath,dset,normalize,divide,bigdata,test_device=2):
print('>>>>> Processing in folder: ' + modelpath + ', iteration: '+str(iter))
# Load model
protosmall = modelpath + 'net_test.prototxt'
protobig = modelpath + 'net_test_big.prototxt'
if os.path.exists(protobig) and bigdata:
proto=protobig
elif os.path.exists(protosmall):
proto=protosmall
else:
print('Error: can\'t find test proto')
return
model = modelpath + 'net_iter_'+str(iter)+'.caffemodel'
if not os.path.exists(model):
print('Iter doesn\'t exist, skipping ', model)
return
# paths
if not os.path.exists(outputpath):
os.makedirs(outputpath)
h5files = [outputpath + d['name'] + '.hdf' for d in dset]
alreadyProcessed = map(os.path.exists,h5files)
if all(alreadyProcessed):
print('Skipping ', h5files)
return
# Set devices
print('Setting device. Using device: ' + str(test_device))
pygt.caffe.set_mode_gpu()
pygt.caffe.set_device(test_device)
print('Loading proto: ' + proto)
print('Loading model: ' + model)
net = pygt.caffe.Net(proto, model, pygt.caffe.TEST)
# Process
print('Processing ' + str(len(dset)) + ' volumes...')
for i in range(len(dset)):
h5file = outputpath + dset[i]['name'] + '.hdf'
if os.path.exists(h5file):
print('Skipping ' + h5file)
continue
print('Processing to ' + h5file)
preds = pygt.process(net,[dset[i]])
print('Saving to ' + h5file)
outhdf5 = h5py.File(h5file, 'w')
outdset = outhdf5.create_dataset('main', preds[0].shape, np.float32, data=preds[0])
outhdf5.close()
# pygt.caffe.select_device(test_device, False)
# Load model
print('Loading model...')
print('laoding small model...')
small_net = pygt.caffe.Net(small_modelproto, modelfile, pygt.caffe.TEST)
print('loading big model...')
net = pygt.caffe.Net(modelproto, modelfile, pygt.caffe.TEST)
print('transfering weights....')
pygt.net_weight_transfer(net, small_net)
start_time = time.time()
# Process
print(test_dataset[-1]['data'].shape)
print('Processing ' + str(len(test_dataset)) + ' volume(s)...')
preds = pygt.process(net,test_dataset)
#for i in range(len(test_dataset)):
#print('Saving ' + test_dataset[i]['name'])
#h5file = test_dataset[i]['name'] + '.h5'
#outhdf5 = h5py.File(h5file, 'w')
#outdset = outhdf5.create_dataset('main', preds[i].shape, np.float32, data=preds[i])
#outhdf5.close()
end_time = time.time()
print("elapsed seconds {0} for z ,y ,x ={1}, {2}, {3}".format(end_time-start_time, preds[0].shape[1],preds[0].shape[2],preds[0].shape[3]))
for i in range(0,len(test_dataset),2):
print('Saving ' + test_dataset[i]['name'])
h5file = test_dataset[i]['name'] + '-pred.h5'
outhdf5 = h5py.File(h5file, 'w')
dname = 'tstvol-520-2-h5'
test_dataset[-1]['name'] = dname
h5im = h5py.File(join(path,dname,'img_normalized.h5'),'r')
h5im_n = np.asarray(h5im[h5im.keys()[0]]).astype(float32)/255
test_dataset[-1]['data'] = h5im_n
# Set devices
test_device = 2
print('Setting devices...')
pygt.caffe.set_mode_gpu()
pygt.caffe.set_device(test_device)
# pygt.caffe.select_device(test_device, False)
# Load model
print('Loading model...')
net = pygt.caffe.Net(modelproto, modelfile, pygt.caffe.TEST)
# Process
print('Processing ' + str(len(test_dataset)) + ' volume(s)...')
preds = pygt.process(net,test_dataset)
for i in range(len(test_dataset)):
print('Saving ' + test_dataset[i]['name'])
h5file = test_dataset[i]['name'] + '.h5'
outhdf5 = h5py.File(h5file, 'w')
outdset = outhdf5.create_dataset('main', preds[i].shape, np.float32, data=preds[i])
outhdf5.close()
# Load PyGreentea
import PyGreentea as pygt
test_net_file = 'net_test.prototxt'
test_device = 3
pygt.caffe.set_devices((test_device, ))
caffemodels = pygt.getCaffeModels('net')
test_net = pygt.init_testnet(test_net_file,
trained_model=caffemodels[-1][1],
test_device=test_device)
hdf5_raw_file = '../dataset_06/fibsem_medulla_7col/trvol-250-1-h5/img_normalized.h5'
hdf5_raw = h5py.File(hdf5_raw_file, 'r')
hdf5_raw_ds = pygt.normalize(
np.asarray(hdf5_raw[hdf5_raw.keys()[0]]).astype(float32), -1, 1)
test_dataset = {}
test_dataset['data'] = hdf5_raw_ds
pred_array = pygt.process(test_net, [test_dataset])
outhdf5 = h5py.File('test_out.h5', 'w')
outdset = outhdf5.create_dataset('main',
np.shape(pred_array)[1:],
np.float32,
data=pred_array)
outdset.attrs['label'] = np.string_('-1,0,0;0,-1,0;0,0,-1')
outhdf5.close()
sys.path.append(os.path.join(os.path.dirname(__file__), pygt_path))
# Other python modules
import math
# Load PyGreentea
import PyGreentea as pygt
test_net_file = 'net_test.prototxt'
test_device = 3
pygt.caffe.set_devices((test_device,))
caffemodels = pygt.getCaffeModels('net');
test_net = pygt.init_testnet(test_net_file, trained_model=caffemodels[-1][1], test_device=test_device)
hdf5_raw_file = '../dataset_06/fibsem_medulla_7col/trvol-250-1-h5/img_normalized.h5'
hdf5_raw = h5py.File(hdf5_raw_file, 'r')
hdf5_raw_ds = pygt.normalize(np.asarray(hdf5_raw[hdf5_raw.keys()[0]]).astype(float32), -1, 1)
test_dataset = {}
test_dataset['data'] = hdf5_raw_ds
pred_array = pygt.process(test_net, [test_dataset])
outhdf5 = h5py.File('test_out.h5', 'w')
outdset = outhdf5.create_dataset('main', np.shape(pred_array)[1:], np.float32, data=pred_array)
outdset.attrs['label'] = np.string_('-1,0,0;0,-1,0;0,0,-1')
outhdf5.close()
test_net = pygt.init_testnet(test_net_file, trained_model=caffemodels[-1][1], test_device=test_device)
# Load the datasets
hdf5_raw_file = '../dataset_06/fibsem_medulla_7col/tstvol-520-1-h5/img_normalized.h5'
hdf5_gt_file = '../dataset_06/fibsem_medulla_7col/tstvol-520-1-h5/groundtruth_seg.h5'
hdf5_aff_file = '../dataset_06/fibsem_medulla_7col/tstvol-520-1-h5/groundtruth_aff.h5'
hdf5_raw = h5py.File(hdf5_raw_file, 'r')
hdf5_gt = h5py.File(hdf5_gt_file, 'r')
hdf5_aff = h5py.File(hdf5_aff_file, 'r')
hdf5_raw_ds = pygt.normalize(np.asarray(hdf5_raw[hdf5_raw.keys()[0]]).astype(float32), -1, 1)
hdf5_gt_ds = np.asarray(hdf5_gt[hdf5_gt.keys()[0]]).astype(float32)
hdf5_aff_ds = np.asarray(hdf5_aff[hdf5_aff.keys()[0]]).astype(float32)
datasets = []
for i in range(0,1):
dataset = {}
dataset['data'] = hdf5_raw_ds[None, i, :]
datasets += [dataset]
pred_array = pygt.process(test_net, datasets)
pygt.dump_tikzgraph_maps(test_net, 'dump')
outhdf5 = h5py.File('test_out.h5', 'w')
outdset = outhdf5.create_dataset('main', np.shape(pred_array)[1:], np.float32, data=pred_array)
outhdf5.close()
