Exemplo n.º 1
0
def main(args):

    total_start = timeit.default_timer()
    print('Starting Preibisch fusion', args.substack_id)

    ss = SubStack(args.first_view_dir, args.substack_id)
    minz = int(ss.info['Files'][0].split("/")[-1].split('_')[-1].split('.tif')[0])
    prefix = '_'.join(ss.info['Files'][0].split("/")[-1].split('_')[0:-1])+'_'
    np_tensor_3d_first_view,_  = imtensor.load_nearby(args.tensorimage_first_view, ss, args.size_patch)
    sc_in=np_tensor_3d_first_view.shape

    if args.transformation_file is not None:
	R, t = parse_transformation_file(args.transformation_file)
        np_tensor_3d_second_view = transform_substack(args.second_view_dir, args.tensorimage_second_view, args.substack_id, R, t, args.size_patch, invert=True)
    else:
        np_tensor_3d_second_view,_  = imtensor.load_nearby(args.tensorimage_second_view, ss, args.size_patch)

    fused_image,entropy_mask__view,entropy_mask_second_view = do_content_based_fusion(np_tensor_3d_first_view,np_tensor_3d_second_view,args.size_patch, args.size_patch, speedup=1,fast_computation=True)
   
    if args.extramargin>args.size_patch:
	args.extramargin=args.size_patch
    
    offset_margin=args.size_patch - args.extramargin
    fused_image_output=fused_image[offset_margin:sc_in[0]-offset_margin,offset_margin:sc_in[1]-offset_margin,offset_margin:sc_in[2]-offset_margin]
    atom = tables.UInt8Atom()
    mkdir_p(args.outdir)
    h5f = tables.openFile(args.outdir + '/' + args.substack_id + '.h5', 'w')
    sc_out=fused_image_output.shape
    ca = h5f.createCArray(h5f.root, 'full_image', atom, sc_out)
    for z in xrange(0, sc_out[0], 1):
        ca[z, :, :] = fused_image_output[z,:,:]
    h5f.close()

    imtensor.save_tensor_as_tif(fused_image_output, args.outdir+'/'+args.substack_id, minz,prefix=prefix)
    print ("total time Preibisch fusion: %s" %(str(timeit.default_timer() - total_start)))
Exemplo n.º 2
0
def fuse_tensors(outdir,pixels_redChannel,pixels_greenChannel,pixels_blueChannel):
    """
    Method that fuses multiple tensors in a RGB tensor

    Parameters
    ----------
    
    pixels_redChannel: numpy tensor
	numpy tensor of the first view (R channel)
    pixels_greeChannel: numpy tensor
	numpy tensor of the second view (G channel)
    pixels_blueChannel: numpy tensor
	numpy tensor of the third view (B channel)
    outdir: str
	directory where the output tensor will be saved
    """
    mkdir_p(outdir)
    num_slices=pixels_redChannel.shape[0]
    for z in xrange(0, num_slices, 1):
        pixels_merged = Image.merge("RGB", (pixels_redChannel[z], pixels_greenChannel[z], pixels_blueChannel[z]))
        im = Image.fromarray(pixels_merged)
        tempname = '/tmp/'+str(uuid.uuid4())+'.tif'
        im.save(tempname)
        destname = outdir+'/slice_'+str(z).zfill(4)+'.tif'
        os.system('tiffcp -clzw:2 ' + tempname + ' ' + destname)
        os.remove(tempname)
    print('...fusion computed (%s slices) ' %z)
    print('fused stack saved in ', outdir)
Exemplo n.º 3
0
def main(args):
    st = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
    tee.log('find_cells.py running on',platform.node(),st)

    mkdir_p(args.outdir+'/'+args.substack_id)
    if args.pair_id is None:
        tee.logto('%s/%s/log.txt' % (args.outdir, args.substack_id))
	args.outdir=args.outdir+'/'+args.substack_id
    else:
        tee.logto('%s/%s/log_%s.txt' % (args.outdir, args.substack_id, args.pair_id))
	args.outdir=args.outdir+'/'+args.substack_id+'/'+args.pair_id
        mkdir_p(args.outdir)

    timers = [mscd.pca_analysis_timer, mscd.mean_shift_timer, mscd.ms_timer, mscd.patch_ms_timer]
    timers.extend([volume.save_vaa3d_timer, volume.save_markers_timer])
    timers.extend([threshold.multi_kapur_timer])
    for t in timers:
        t.reset()


    substack = volume.SubStack(args.indir, args.substack_id)
    substack.load_volume(pair_id=args.pair_id)
    if args.local:
        mscd.pms(substack, args)
    else:
        mscd.ms(substack, args)
    for t in timers:
        if t.n_calls > 0:
            tee.log(t)
    st = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
    tee.log('find_cells.py finished on',platform.node(),st)
Exemplo n.º 4
0
def main(args):

    total_start = timeit.default_timer()
    print('Starting Preibisch fusion', args.substack_id)

    ss = SubStack(args.first_view_dir, args.substack_id)
    minz = int(
        ss.info['Files'][0].split("/")[-1].split('_')[-1].split('.tif')[0])
    prefix = '_'.join(
        ss.info['Files'][0].split("/")[-1].split('_')[0:-1]) + '_'
    np_tensor_3d_first_view, _ = imtensor.load_nearby(
        args.tensorimage_first_view, ss, args.size_patch)
    sc_in = np_tensor_3d_first_view.shape

    if args.transformation_file is not None:
        R, t = parse_transformation_file(args.transformation_file)
        np_tensor_3d_second_view = transform_substack(
            args.second_view_dir,
            args.tensorimage_second_view,
            args.substack_id,
            R,
            t,
            args.size_patch,
            invert=True)
    else:
        np_tensor_3d_second_view, _ = imtensor.load_nearby(
            args.tensorimage_second_view, ss, args.size_patch)

    fused_image, entropy_mask__view, entropy_mask_second_view = do_content_based_fusion(
        np_tensor_3d_first_view,
        np_tensor_3d_second_view,
        args.size_patch,
        args.size_patch,
        speedup=1,
        fast_computation=True)

    if args.extramargin > args.size_patch:
        args.extramargin = args.size_patch

    offset_margin = args.size_patch - args.extramargin
    fused_image_output = fused_image[offset_margin:sc_in[0] - offset_margin,
                                     offset_margin:sc_in[1] - offset_margin,
                                     offset_margin:sc_in[2] - offset_margin]
    atom = tables.UInt8Atom()
    mkdir_p(args.outdir)
    h5f = tables.openFile(args.outdir + '/' + args.substack_id + '.h5', 'w')
    sc_out = fused_image_output.shape
    ca = h5f.createCArray(h5f.root, 'full_image', atom, sc_out)
    for z in xrange(0, sc_out[0], 1):
        ca[z, :, :] = fused_image_output[z, :, :]
    h5f.close()

    imtensor.save_tensor_as_tif(fused_image_output,
                                args.outdir + '/' + args.substack_id,
                                minz,
                                prefix=prefix)
    print("total time Preibisch fusion: %s" %
          (str(timeit.default_timer() - total_start)))
Exemplo n.º 5
0
def main(args):
    substack = SubStack(args.indir, args.substack_id)
    patch = substack.get_volume()
    histogram = np.histogram(patch, bins=256, range=(0, 256))[0]
    thresholds = threshold.multi_kapur(histogram, 2)
    outfile = args.outdir + '/' + args.substack_id + '/' + basename(args.indir)
    mkdir_p(args.outdir + '/' + args.substack_id)
    f = open(outfile, 'w')
    f.write(str(thresholds[0]) + ',' + str(thresholds[1]) + '\n')
    f.close()
Exemplo n.º 6
0
def main(args):
    substack = SubStack(args.indir,args.substack_id)
    patch = substack.get_volume()
    histogram = np.histogram(patch, bins=256,range=(0,256))[0]
    thresholds = threshold.multi_kapur(histogram, 2)
    outfile=args.outdir+'/'+args.substack_id+'/'+basename(args.indir)
    mkdir_p(args.outdir+'/'+args.substack_id)
    f=open(outfile,'w')
    f.write(str(thresholds[0])+','+str(thresholds[1])+'\n')
    f.close()
Exemplo n.º 7
0
def save_tensor_as_tif(np_tensor_3d,path, minz, prefix='full_'):
    """Export a 3D numpy tensor as a sequence of tiff files (one for each
z coordinate). Each file is named as prefix followed by an integer id,
which ranges in minz:minz+np_tensor_3d.shape[0].  The tensor is
expected to be stored in the order Z,Y,X

    """
    import uuid
    mkdir_p(path)
    pbar = ProgressBar(widgets=['Saving %d tiff files: ' % np_tensor_3d.shape[0], Percentage(), ' ', AdaptiveETA()])
    for z in pbar(range(np_tensor_3d.shape[0])):
        out_img = Image.fromarray(np_tensor_3d[z,:,:])
        tempname = '/tmp/'+str(uuid.uuid4())+'.tif'
        out_img.save(tempname)
        destname=path+'/'+prefix+'%06d.tif'%(minz+z)
        os.system('tiffcp -clzw:2 ' + tempname + ' ' + destname)
        os.remove(tempname)
    print('Saved substack in',path)
Exemplo n.º 8
0
def main(args):
    st = datetime.datetime.fromtimestamp(
        time.time()).strftime('%Y-%m-%d %H:%M:%S')
    tee.log('find_cells.py running on', platform.node(), st)

    mkdir_p(args.outdir + '/' + args.substack_id)
    if args.pair_id is None:
        tee.logto('%s/%s/log.txt' % (args.outdir, args.substack_id))
        args.outdir = args.outdir + '/' + args.substack_id
    else:
        tee.logto('%s/%s/log_%s.txt' %
                  (args.outdir, args.substack_id, args.pair_id))
        args.outdir = args.outdir + '/' + args.substack_id + '/' + args.pair_id
        mkdir_p(args.outdir)

    timers = [
        mscd.pca_analysis_timer, mscd.mean_shift_timer, mscd.ms_timer,
        mscd.patch_ms_timer
    ]
    timers.extend([volume.save_vaa3d_timer, volume.save_markers_timer])
    timers.extend([threshold.multi_kapur_timer])
    for t in timers:
        t.reset()

    # dovremmo passare le informazinoi su w,h,d tramite pslist
    substack = volume.SubStack(args.indir, args.substack_id)
    # ignore_info_files setted to true to avoid errors
    substack.load_volume(pair_id=args.pair_id, ignore_info_files=True)
    #todo la nostra versione è questa
    #substack.load_volume_from_3D()

    #todo lo script va lanciato per ogni volume
    if args.local:
        mscd.pms(substack, args)
    else:
        mscd.ms(substack, args)
    for t in timers:
        if t.n_calls > 0:
            tee.log(t)
    st = datetime.datetime.fromtimestamp(
        time.time()).strftime('%Y-%m-%d %H:%M:%S')
    tee.log('find_cells.py finished on', platform.node(), st)
Exemplo n.º 9
0
def main(args):
    try:
        C_firstview = m_load_markers(args.first_view,from_vaa3d=True)
    except IOError:
        print('Warning: first view marker file',args.first_view,'not found.')
        C_firstview = []
    try:
        C_secondview = m_load_markers(args.second_view,from_vaa3d=True)
    except IOError:
        print('Warning: second view marker file',args.second_view,'not found.')
        C_secondview = []

    mkdir_p(os.path.dirname(args.output_marker_file))
    substack = SubStack(args.indir,args.substack_id)
    if args.do_icp:
        C_merged, C_onlyfirstview, C_onlyfirstview, _ = do_fuse_with_icp(substack,C_firstview,C_secondview,args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
    else:
        C_merged, C_onlyfirstview, C_onlyfirstview, _ = do_fuse(substack,C_firstview,C_secondview,args.max_distance, args.verbose)
    
    save_fused_markers(substack,C_merged,C_onlyfirstview,C_onlysecondview,output_marker_file,first_view_id,second_view_id,verbose)
Exemplo n.º 10
0
    def save_above_threshold(self, Lx, Ly, Lz, thresholds=None):
        out_imgs = []
        opixels = []
        pixels = []
        for z, img in enumerate(self.substack.imgs):
            out_img = img.convert('RGB')
            out_imgs.append(out_img)
            opix = out_img.load()
            w, h = out_img.size
            for i in xrange(w):
                for j in xrange(h):
                    if thresholds:
                        opix[i, j] = (self.substack.pixels[z][i,
                                                              j], 0, 0)  # Red
                    else:
                        opix[i, j] = 0
            opixels.append(opix)
            pixels.append(img.load())
        if thresholds:
            for i in xrange(len(Lx)):
                val = pixels[Lz[i]][Lx[i], Ly[i]]
                if val < thresholds[
                        1]:  # Note: since it's on the lists Lx,Ly,Lz, it's already above thresholds[0]!
                    opixels[Lz[i]][Lx[i], Ly[i]] = (0, val, 0)  # Green
                else:
                    opixels[Lz[i]][Lx[i], Ly[i]] = (val, val, 0)  # Yellow
        else:
            for i in xrange(len(Lx)):
                opixels[Lz[i]][Lx[i], Ly[i]] = (pixels[Lz[i]][Lx[i], Ly[i]],
                                                pixels[Lz[i]][Lx[i], Ly[i]],
                                                pixels[Lz[i]][Lx[i], Ly[i]])

        save_name = 'above_threshold'
        iter_dir = '%s/%s' % (self.savedir, save_name)
        mkdir_p(iter_dir)
        for z, out_img in enumerate(out_imgs):
            out_img.save(iter_dir + '/' +
                         self.substack.info['Files'][z].split('/')[-1])
            w, h = out_img.size
Exemplo n.º 11
0
def save_tensor_as_tif(np_tensor_3d, path, minz, prefix='full_'):
    """Export a 3D numpy tensor as a sequence of tiff files (one for each
z coordinate). Each file is named as prefix followed by an integer id,
which ranges in minz:minz+np_tensor_3d.shape[0].  The tensor is
expected to be stored in the order Z,Y,X

    """
    import uuid
    mkdir_p(path)
    pbar = ProgressBar(widgets=[
        'Saving %d tiff files: ' % np_tensor_3d.shape[0],
        Percentage(), ' ',
        ETA()
    ])
    for z in pbar(range(np_tensor_3d.shape[0])):
        out_img = Image.fromarray(np_tensor_3d[z, :, :])
        tempname = '/tmp/' + str(uuid.uuid4()) + '.tif'
        out_img.save(tempname)
        destname = path + '/' + prefix + '%04d.tif' % (minz + z)
        os.system('tiffcp -clzw:2 ' + tempname + ' ' + destname)
        os.remove(tempname)

    print('Saved substack in', path)
Exemplo n.º 12
0
    def save_above_threshold(self, Lx, Ly, Lz, thresholds=None):
        out_imgs = []
        opixels = []
        pixels = []
        for z, img in enumerate(self.substack.imgs):
            out_img = img.convert('RGB')
            out_imgs.append(out_img)
            opix = out_img.load()
            w, h = out_img.size
            for i in xrange(w):
                for j in xrange(h):
                    if thresholds:
                        opix[i, j] = (self.substack.pixels[z][i,j], 0, 0)  # Red
                    else:
                        opix[i, j] = 0
            opixels.append(opix)
            pixels.append(img.load())
        if thresholds:
            for i in xrange(len(Lx)):
                val = pixels[Lz[i]][Lx[i], Ly[i]]
                if val < thresholds[1]:  # Note: since it's on the lists Lx,Ly,Lz, it's already above thresholds[0]!
                    opixels[Lz[i]][Lx[i], Ly[i]] = (0,val,0)  # Green
                else:
                    opixels[Lz[i]][Lx[i], Ly[i]] = (val,val,0)  # Yellow
        else:
            for i in xrange(len(Lx)):
                opixels[Lz[i]][Lx[i], Ly[i]] = (pixels[Lz[i]][Lx[i], Ly[i]],
                                                pixels[Lz[i]][Lx[i], Ly[i]],
                                                pixels[Lz[i]][Lx[i], Ly[i]])

        save_name = 'above_threshold'
        iter_dir = '%s/%s' % (self.savedir, save_name)
        mkdir_p(iter_dir)
        for z, out_img in enumerate(out_imgs):
            out_img.save(iter_dir+'/'+self.substack.info['Files'][z].split('/')[-1])
            w, h = out_img.size
Exemplo n.º 13
0
def main(args):
    st = datetime.datetime.fromtimestamp(
        time.time()).strftime('%Y-%m-%d %H:%M:%S')
    tee.log('find_cells.py running on', platform.node(), st)

    mkdir_p(args.outdir + '/' + args.substack_id)
    if args.pair_id is None:
        tee.logto('%s/%s/log.txt' % (args.outdir, args.substack_id))
        args.outdir = args.outdir + '/' + args.substack_id
    else:
        tee.logto('%s/%s/log_%s.txt' %
                  (args.outdir, args.substack_id, args.pair_id))
        args.outdir = args.outdir + '/' + args.substack_id + '/' + args.pair_id
        mkdir_p(args.outdir)

    timers = [
        mscd.pca_analysis_timer, mscd.mean_shift_timer, mscd.ms_timer,
        mscd.patch_ms_timer
    ]
    timers.extend([volume.save_vaa3d_timer, volume.save_markers_timer])
    timers.extend([threshold.multi_kapur_timer])
    for t in timers:
        t.reset()

    substack = volume.SubStack(args.indir, args.substack_id)
    substack.load_volume(pair_id=args.pair_id)
    if args.local:
        mscd.pms(substack, args)
    else:
        mscd.ms(substack, args)
    for t in timers:
        if t.n_calls > 0:
            tee.log(t)
    st = datetime.datetime.fromtimestamp(
        time.time()).strftime('%Y-%m-%d %H:%M:%S')
    tee.log('find_cells.py finished on', platform.node(), st)
Exemplo n.º 14
0
def make_pos_neg_dataset(tensor_first_view, tensor_second_view, ss, C, view1_id, view2_id, default_sigma=0.8,size=5, save_tiff_files=False, find_negative=True):

    tensor_first_view = tensor_first_view.astype(np.float32)
    tensor_second_view = tensor_second_view.astype(np.float32)
    H, W, D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
    patchlen = (1 + 2 * size) ** 3

    margin = ss.plist['Margin'] / 2

    c_array = np.array([[c.x, c.y, c.z] for c in C])

    trunc=1.5
    fixed_radiiassigned={c:default_sigma for c in C  if inside_margin(c,ss) > 0 }
    cccc=fixed_radiiassigned.copy()
    kdt = cKDTree(c_array)
    dist,ind = kdt.query(c_array,k=c_array.shape[0],distance_upper_bound=(2*default_sigma*trunc)+2.)
    for c,id_c in zip(C,xrange(c_array.shape[0])):
        if inside_margin(c, ss) > 0:
            for j in xrange(2,c_array.shape[0]):
                if not np.isinf(dist[id_c][j]):
                    if inside_margin(C[ind[id_c][j]], ss) > 0:
                        if trunc*(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]]) > dist[id_c][j] - 2.:
                            new_sigma=((dist[id_c][j]-1.)/trunc)*fixed_radiiassigned[c]/(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]])
                            fixed_radiiassigned[C[ind[id_c][j]]]=(new_sigma*fixed_radiiassigned[C[ind[id_c][j]]])/fixed_radiiassigned[c]
                            fixed_radiiassigned[c]=new_sigma


    target_tensor_3d = np.zeros(tensor_first_view.shape,dtype=np.uint8)
    for c in C:
        if inside_margin(c, ss) > 0:
            target_tensor_3d_tmp = np.zeros(tensor_first_view.shape)
            target_tensor_3d_tmp[c.z, c.y, c.x] = 1
            sigma=fixed_radiiassigned[c]
            target_tensor_3d_tmp = gfilter.gaussian_filter(target_tensor_3d_tmp, sigma,
                                               mode='constant', cval=0.0,
                                               truncate=trunc)  #sigma=3.5,mode='constant',cval=0.0,truncate=1.5
            target_tensor_3d_tmp = (target_tensor_3d_tmp.astype(np.float32) / np.max(target_tensor_3d_tmp))

            target_tensor_3d =  np.maximum(np.array(target_tensor_3d_tmp * 255.0, dtype=np.uint8), target_tensor_3d)

    if save_tiff_files:
        debug_path='/tmp/debug/sigma_'+str(default_sigma)
        mkdir_p(debug_path)
        minz = 0
        imtensor.save_tensor_as_tif(target_tensor_3d, debug_path+'/'+ss.substack_id+'_'+view1_id+'_'+view2_id, minz)


    print('num markers =',len(C))
    target_tensor_3d = (target_tensor_3d.astype(np.float32) / np.max(target_tensor_3d))
    tensor_first_view = tensor_first_view.astype(np.float32) / 255.0
    tensor_second_view = tensor_second_view.astype(np.float32) / 255.0

    nrej_intensity = 0
    num_negative_targets = 0
    num_positive_targets = 0

    step_x = 1
    step_y = 1
    step_z = 1
    num_iterations = (W - 2*margin +1) * (H- 2*margin) * (D - 2*margin) / (step_x * step_y * step_z)
    pbar = ProgressBar(widgets=['Making positive and negative examples for %d points: ' % num_iterations, Percentage()],
                       maxval=num_iterations).start()

    X_positive = np.zeros((num_iterations, patchlen*2), dtype=np.float32)
    y_positive = np.zeros((num_iterations, patchlen), dtype=np.float32)

    X_negative = []
    y_negative = []
    if find_negative == True:
        X_negative = np.zeros((num_iterations, patchlen*2), dtype=np.float32)
        y_negative = np.zeros((num_iterations, patchlen), dtype=np.float32)

    pbi = 0
    for x0 in range(margin, W - margin, step_x):
        for y0 in range(margin, H - margin, step_y):
            for z0 in range(margin, D - margin, step_z):

                if inside_patch(c_array, x0, y0, z0, size, offset=2.0):
                    patch_left = tensor_first_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
                    patch_right = tensor_second_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]

                    X_positive[num_positive_targets, 0:patchlen] = np.reshape(patch_left, (patchlen,))
                    X_positive[num_positive_targets, patchlen:2*patchlen] = np.reshape(patch_right, (patchlen,))
                    ypatch = target_tensor_3d[z0 - size:z0 + size + 1, y0 - size:y0 + size + 1,
                                x0 - size:x0 + size + 1]

                    y_positive[num_positive_targets, :] = np.reshape(ypatch, (patchlen,))
                    num_positive_targets += 1

                elif find_negative == True:
                    patch_left = tensor_first_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
                    patch_right = tensor_second_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
                    if np.mean(patch_left * 255) > 5 or np.mean(patch_right * 255) > 5:
                        X_negative[num_negative_targets, 0:patchlen] = np.reshape(patch_left, (patchlen,))
                        X_negative[num_negative_targets, patchlen:2*patchlen] = np.reshape(patch_right, (patchlen,))
                        ypatch = target_tensor_3d[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]


                        y_negative[num_negative_targets, :] = np.reshape(ypatch, (patchlen,))
                        num_negative_targets += 1
                    else:
                        nrej_intensity += 1
                pbar.update(pbi+ 1)
                pbi += 1
    pbar.finish()


    X_positive = X_positive[0: num_positive_targets]
    y_positive = y_positive[0: num_positive_targets]
    print('Total positive examples for substack (', ss.substack_id, '):', num_positive_targets)

    if find_negative == True:

        print('Total negative examples for substack (', ss.substack_id, '):', num_negative_targets)
        print('Rejected by intensity ', nrej_intensity)
        X_negative = X_negative[0: num_negative_targets]
        y_negative = y_negative[0: num_negative_targets]

        if (num_negative_targets > num_positive_targets):
            print('Shuffling negative examples...')
            perm = np.random.permutation(len(X_negative)).astype(int)
            X_negative = X_negative[perm]
            y_negative = y_negative[perm]
            print('Shuffling done')
            X_negative = X_negative[0: num_positive_targets]
            y_negative = y_negative[0: num_positive_targets]


    return X_positive, y_positive, X_negative, y_negative
Exemplo n.º 15
0
    def save_vaa3d(self, C, Lx, Ly, Lz, Lcluster,
                   draw_centers=True,
                   colorize_voxels=True,
                   trajectories=None,
                   floating_point=False):
        """
        save_vaa3d(C, Lx, Ly, Lz, Lcluster,
                   draw_centers=True, colorize_voxels=True, trajectories=None)

        Dump a colored substack so that Vaa3D can be used to inspect clusters

        Parameters
        ----------

        C : list
            List of centers
        Lx, Ly, Lz : array-like
            Coordinates of foreground voxels
        Lcluster : array-like
            Cluster index of every foreground voxel
        draw_centers : bool, optional
            Should centers be drawn as small circles?
        colorize_voxels : bool, optional
            Should we paint voxels in colors?
        trajectories : list or None, optional
            If not None, draw mean shift trajectories for every seed
        floating_point: bool, optional
            If true, save coordinates in floating point, else round to int
        """
        out_imgs = []
        opixels = []
        pixels = []
        self.compute_hues(C)
        for z, img in enumerate(self.substack.imgs):
            out_img = img.convert('RGB')
            out_imgs.append(out_img)
            opixels.append(out_img.load())
            pixels.append(img.load())

        if colorize_voxels:
            for i in xrange(len(Lx)):
                if Lcluster[i] is not None:
                    # draw = ImageDraw.Draw(out_imgs[Lz[i]])
                    opixels[Lz[i]][Lx[i], Ly[i]] = hi2rgb[int(255*Lcluster[i].hue)][pixels[Lz[i]][Lx[i], Ly[i]]]
        if draw_centers:
            for c in C:
                draw = ImageDraw.Draw(out_imgs[int(round(c.z))])
                rgb = hi2rgb[int(255*c.hue)][156]
                draw.ellipse((c.x-1, c.y-1, c.x+1, c.y+1), fill=rgb)
        if trajectories:
            for seed_no in trajectories:
                rgb = hi2rgb[random.randint(0, 255)][156]
                for mean in trajectories[seed_no]:
                    draw = ImageDraw.Draw(out_imgs[int(round(mean[2]))])
                    draw.point((mean[0], mean[1]), fill=rgb)
        save_name = 'last'
        iter_dir = '%s/%s' % (self.savedir, save_name)
        mkdir_p(iter_dir)
        for z, out_img in enumerate(out_imgs):
            out_img.save(iter_dir+'/'+self.substack.info['Files'][z].split('/')[-1])
            w, h = out_img.size
        # ---------- marker file used by vaa3d
        self.substack.save_markers('%s/%s.marker' % (self.savedir, save_name), C, floating_point)
Exemplo n.º 16
0
def main(args):
    """
    Files must be numbered consecutively.
    """
    prefix = None
    #convert_to_grey = True

    savewd = os.getcwd()
    os.chdir(args.indir)
    full_path = os.getcwd()
    os.chdir(savewd)
    files = os.listdir(args.indir)
    files = [f for f in files if f[0] != '.' and f.endswith(args.suffix)]
    prefix = prefix or os.path.commonprefix(files)
    if prefix == '':
        raise Exception('Files in ' + args.indir +
                        ' do not have a common prefix ' + str(files))
    # trailing zero dropped -- WHY???
    # zrange = sorted([int(str(f.split(prefix)[1].split(args.suffix)[0])[:-1]) for f in files])
    zrange = sorted([int(f.split('_')[-1].split('.')[0]) for f in files])
    # n_digits = len(files[0].split(prefix)[1].split(args.suffix)[0])
    n_digits = len(f.split('_')[-1].split('.')[0])
    if zrange != range(zrange[0], zrange[-1] + 1):
        raise Exception('Files in ' + args.indir +
                        ' are not a complete sequence: ' + str(zrange))
    print('Checking image files')
    prefix = '_'.join(prefix.split('_')[:-1])
    for z in zrange:
        image_file = ('%s_%0' + str(n_digits) + 'd' + args.suffix) % (prefix,
                                                                      z)
        img_z = Image.open(full_path + '/' + image_file)
        if z == zrange[0]:
            width, height = img_z.size
        else:
            if (width, height) != img_z.size:
                raise Exception('OOps, file', image_file, 'has size',
                                img_z.size, 'instead of', (width, height))
    depth = len(zrange)

    print(full_path)
    full_path_of_substacks = "/".join(
        full_path.split('/')[:-2]) + '/substacks/' + "/".join(
            full_path.split('/')[-2:])

    print('Volume geometry:', width, height, depth)
    print('Finding substacks')
    substacks = dict()
    # nx,ny,nz=3,8,3 # Good for V000_maggio_large_substack_blurred
    w = int(round(float((width - args.margin)) / float(args.nx)))
    h = int(round(float((height - args.margin)) / float(args.ny)))
    d = int(round(float((depth - args.margin)) / float(args.nz)))
    print(w, h, d)
    for i in range(args.nx):
        for j in range(args.ny):
            for k in range(args.nz):
                x0 = w * i
                y0 = h * j
                z0 = d * k
                x1 = min(width, x0 + w + args.margin)
                y1 = min(height, y0 + h + args.margin)
                z1 = min(depth, z0 + d + args.margin)
                identifier = "%02d%02d%02d" % (i, j, k)
                substack = dict(Width=x1 - x0,
                                Height=y1 - y0,
                                Depth=z1 - z0,
                                X0=x0,
                                Y0=y0,
                                Z0=z0,
                                Files=[])
                if args.verbose:
                    print('Substack', (i, j, k))
                    print('X:', x0, x1, w)
                    print('Y:', y0, y1, h)
                    print('Z:', z0, z1, d)
                    print('volume:',
                          ((x1 - x0) * (y1 - y0) * (z1 - z0)) / 1000000.0,
                          'MVoxels')
                substacks[identifier] = substack
                mkdir_p(full_path_of_substacks + '/' + identifier)

    num_slices = len(zrange)
    print('Saving cropped image files into', full_path_of_substacks)
    for z_from_zero, z in enumerate(zrange):
        image_file = ('%s_%0' + str(n_digits) + 'd' + args.suffix) % (prefix,
                                                                      z)
        img_z = Image.open(full_path + '/' + image_file)
        if args.verbose:
            print(full_path + '/' + image_file, end='')
        print('==========', z + 1, '/', num_slices, '=============')
        for substack_id, substack in substacks.iteritems():
            substack_dir = full_path_of_substacks + '/' + substack_id
            #mkdir_p(substack_dir)
            Z0 = substack['Z0']
            Depth = substack['Depth']
            #            print(substack)
            if z_from_zero in range(Z0, Z0 + Depth):
                X0 = substack['X0']
                Width = substack['Width']
                Y0 = substack['Y0']
                Height = substack['Height']
                region = img_z.crop((X0 + 0, Y0 + 0, X0 + Width, Y0 + Height))
                if args.convert_to_grey:
                    region = region.convert('L')
                if args.verbose:
                    print(' ', substack_id, end='')
                region.save(substack_dir + '/' + image_file)

                tmpfile = '/tmp/' + str(uuid.uuid4()) + '.tif'
                region.save(tmpfile)
                subprocess.call([
                    'tiffcp', '-clzw:2', tmpfile,
                    substack_dir + '/' + image_file
                ])
                os.unlink(tmpfile)

                # substack['Files'].append(substack_dir+'/'+image_file)
                # the path to files is now relative
                substack['Files'].append(substack_id + '/' + image_file)
        if args.verbose:
            print()

    print('Saving substack info into', full_path_of_substacks + '/info.json')
    info = dict(ImageSequenceDirectory=full_path,
                Width=width,
                Height=height,
                Depth=depth,
                Margin=args.margin,
                SubStacks=substacks,
                Full_Path_Of_Substacks=full_path_of_substacks)
    with open(full_path_of_substacks + '/info.json', 'w') as ostream:
        print(json.dumps(info), file=ostream)
Exemplo n.º 17
0
def transform_substack(indir, tensorimage, substack_id, R, t, extramargin, outdir=None, invert=False, save_tiff=False, save_hdf5=False):
    """
    Method that applies a previously estimated rigid transformation to a specific substack.

    Parameters
    ----------
    
    indir : str
	substack dir of the input view
    tensorimage : str
	the whole tensor in hdf5 format
    substack_id : str
	id of the substack that will be transformed
    R : numpy array of shape (3, 3)
	rotational component of the estimated rigid transformation
    t : numpy array of shape (3)
	translational component of the estimated rigid transformation
    extramargin : int
	extramargin used to extract the transformed substack from tensorimage
    outdir : str 
	output directory where the transformed substack will be saved (Default: None)
    invert : boolean 
	if True the tranformation is inverted (Default: False)
    save_tiff : boolean 
	save the transformed substack in a stack of tiff slices (Default: False)
    save_hdf5 : boolean 
	save the transformed substack as a hdf5 tensor (Default: False)

    Returns
    -------

    pixels_transformed_input: numpy tensor
	tensor of the transformed substack

    """
    ss = SubStack(indir, substack_id)
    input_stack_file = tensorimage
    hf5 = tables.openFile(input_stack_file, 'r')
    full_D, full_H, full_W = hf5.root.full_image.shape
    X0,Y0,Z0 = ss.info['X0'], ss.info['Y0'], ss.info['Z0']
    origin = (Z0, Y0, X0)
    H,W,D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
    or_ss_shape = (D,H,W)
    offset_W=int((3**(1/2.0)*W + W/2.0 - W)/2.0)
    offset_H=int((3**(1/2.0)*H + H/2.0 - H)/2.0)
    offset_D=int((3**(1/2.0)*D + D/2.0 - D)/2.0)


    if offset_W < extramargin:
        offset_W = extramargin
    if offset_H < extramargin:
        offset_H = extramargin
    if offset_D < extramargin:
        offset_D = extramargin

    offset_D_left = offset_D if int(origin[0] - offset_D) > 0 else origin[0]
    offset_H_left = offset_H if int(origin[1] - offset_H) > 0 else origin[1]
    offset_W_left = offset_W if int(origin[2] - offset_W) > 0 else origin[2]
    offset_D_right = offset_D if int(origin[0] + or_ss_shape[0] + offset_D) <= full_D else full_D - (origin[0] + or_ss_shape[0])
    offset_H_right = offset_H if int(origin[1] + or_ss_shape[1] + offset_H) <= full_H else full_H - (origin[1] + or_ss_shape[1])
    offset_W_right = offset_W if int(origin[2] + or_ss_shape[2] + offset_W) <= full_W else full_W - (origin[2] + or_ss_shape[2])



    pixels_input = hf5.root.full_image[origin[0] - offset_D_left:origin[0] + or_ss_shape[0] + offset_D_right,
                                       origin[1] - offset_H_left:origin[1] + or_ss_shape[1] + offset_H_right,
                                       origin[2] - offset_W_left:origin[2] + or_ss_shape[2] + offset_W_right]


    exmar_D_left = 0 if offset_D_left == origin[0] else extramargin
    exmar_H_left  = 0 if offset_H_left == origin[1] else extramargin
    exmar_W_left  = 0 if offset_W_left == origin[2] else extramargin

    depth_target, height_target, width_target = or_ss_shape[0] + 2 * extramargin, or_ss_shape[1] + 2 * extramargin, or_ss_shape[2] + 2 * extramargin #new
    depth_input, height_input, width_input = pixels_input.shape[0], pixels_input.shape[1],  pixels_input.shape[2]
    pixels_transformed_input = np.zeros((depth_target,height_target,width_target), dtype=np.uint8)


    total_start = timeit.default_timer()

    coords_2d_target = np.vstack(np.indices((width_target,height_target)).swapaxes(0,2).swapaxes(0,1))
    invR = R.T

    if invert:
        t = -np.dot(invR, t)
        invR = R

    invR_2d_transpose = np.transpose(np.dot(invR[:, 0:2], np.transpose(coords_2d_target - t[0:2])))
    offset_coords = np.array([[offset_W_left - exmar_W_left, offset_H_left - exmar_H_left, offset_D_left - exmar_D_left]]*invR_2d_transpose.shape[0])#new

    for z in xrange(0, depth_target, 1):
        R_t_3d = np.transpose(invR_2d_transpose + invR[:, 2] * (z - t[2]) + offset_coords)
        good_indices = np.array(range(R_t_3d.shape[1]))
        good_indices = good_indices[(R_t_3d[0, :] > 0) * (R_t_3d[1, :] > 0) * (R_t_3d[2, :] > 0) * (R_t_3d[0, :] < (width_input - 1)) * (R_t_3d[1, :] < (height_input - 1)) * (R_t_3d[2, :] < (depth_input - 1))]
        R_t_3d = R_t_3d.take(good_indices,axis=1)
        R_t_3d = np.round(R_t_3d).astype(int)
        coords_2d_target_tmp = coords_2d_target.take(good_indices, axis=0)
        coords_3d_target_tmp = np.hstack((coords_2d_target_tmp, np.ones((coords_2d_target_tmp.shape[0], 1)).astype(int)*z))
        pixels_transformed_input[coords_3d_target_tmp[:, 2], coords_3d_target_tmp[:, 1], coords_3d_target_tmp[:, 0]] = pixels_input[R_t_3d[2, :], R_t_3d[1, :], R_t_3d[0, :]]

    total_stop = timeit.default_timer()
    print ("total time transformation stack:%s "%(str(total_stop - total_start)))

    pixels_transformed_input = np.array(pixels_transformed_input, dtype=np.uint8)
    if save_tiff or save_hdf5:
	mkdir_p(outdir)
	if save_tiff:
	    minz = int(ss.info['Files'][0].split("/")[-1].split('_')[-1].split('.tif')[0])
	    _prefix = '_'.join(ss.info['Files'][0].split("/")[-1].split('_')[0:-1])+'_'
	    substack_outdir = outdir + '/' + substack_id
	    imtensor.save_tensor_as_tif(pixels_transformed_input, substack_outdir, minz, prefix=_prefix)
	if save_hdf5: 
	    target_shape = (depth_target, height_target, width_target)
	    atom = tables.UInt8Atom()
	    h5f = tables.openFile(outdir + '/' + ss.substack_id + '.h5', 'w')
	    ca = h5f.createCArray(h5f.root, 'full_image', atom, target_shape)
	    for z in xrange(0, depth_target, 1):
		ca[z, :, :] = pixels_transformed_input[z,:,:]
	    h5f.close()

    return pixels_transformed_input
Exemplo n.º 18
0
def merge_views(args):

    view_ids=args.view_ids
    max_len=0
    hue=0.
    C_substack=[]

    if args.thresholds_dir is None:
        print(view_ids)
        list_views = get_visible_pairs(args.base_indir,args.substack_id,view_ids)
    else:
        list_views = get_visible_pairs_from_dir(args.thresholds_dir, args.substack_id, view_ids)

    substack = SubStack(args.base_indir+'/'+view_ids[0],args.substack_id)
    valid_pairs = filter_valid_pairs(args,substack,list_views)

    if len(valid_pairs.keys())==1:
            C_substack+=valid_pairs.values()[0]
    else:
            keys_view_1=[]
            keys_view_3=[]
            for view_key in valid_pairs.keys():
                    if view_key[0] == view_ids[2]:
                            keys_view_3.append(view_key)
                    elif view_key[0] == view_ids[0]:
                            keys_view_1.append(view_key)

    if len(valid_pairs)==2:
            if len(keys_view_1) == 2 or len(keys_view_3) == 2:
                    total_list=compute_fusion(substack,valid_pairs.values()[0],valid_pairs.values()[1],args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_substack+=total_list
            elif len(keys_view_1) == 1 and len(keys_view_3) == 1:
                    C_view_3_t=transform_markers(args,substack,valid_pairs[keys_view_3[0]], view_ids)
                    total_list=compute_fusion(substack,valid_pairs[keys_view_1[0]],C_view_3_t,args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_substack+=total_list
            else:
                    raise Exception('not valid list of views %s'%(str(valid_pairs)))

    elif len(valid_pairs)==3:
            if len(keys_view_1) == 2:
                    C_view_1=compute_fusion(substack,valid_pairs[keys_view_1[0]],valid_pairs[keys_view_1[1]],args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_view_3_t=transform_markers(args,substack,valid_pairs[keys_view_3[0]], view_ids)
                    total_list=compute_fusion(substack,C_view_1,C_view_3_t,args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_substack+=total_list
            elif len(keys_view_3) == 2:
                    C_view_3=compute_fusion(substack,valid_pairs[keys_view_3[0]],valid_pairs[keys_view_3[1]],args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_view_3_t=transform_markers(args,substack,C_view_3, view_ids)
                    C_view_1=valid_pairs[keys_view_1[0]]
                    total_list=compute_fusion(substack,C_view_1,C_view_3_t,args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_substack+=total_list
            else:
                    raise Exception('not valid list of views %s'%(str(valid_pairs)))

    elif len(valid_pairs)==4:
            if len(keys_view_1) == 2 and len(keys_view_3) == 2:
                    C_view_1=compute_fusion(substack,valid_pairs[keys_view_1[0]],valid_pairs[keys_view_1[1]],args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_view_3=compute_fusion(substack,valid_pairs[keys_view_3[0]],valid_pairs[keys_view_3[1]],args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_view_3_t=transform_markers(args,substack,C_view_3, view_ids)
                    total_list=compute_fusion(substack,C_view_1,C_view_3_t,args.max_distance,match_distance=args.match_distance,verbose=args.verbose)
                    C_substack+=total_list
            else:
                    raise Exception('not valid list of views %s'%(str(valid_pairs)))

    mkdir_p(args.outdir+'/'+args.substack_id)
    if len(C_substack)>0:
            C_substack = filter_outside_markers(C_substack,substack)
            substack.save_markers(args.outdir+'/'+args.substack_id+'/ms.marker', C_substack, floating_point=True)
Exemplo n.º 19
0
    if len(sys.argv) != 4:
        usage()
        sys.exit(2)
    indir = sys.argv[1]
    infile = sys.argv[2]
    outdir = sys.argv[3]

    substack_ids = map(string.strip, os.popen('ls ' + indir).readlines())
    substack_ids = [f for f in substack_ids if len(f) == 6]
    C = load_markers(infile)

    plist = None
    substacks = {}
    for substack_id in substack_ids:
        substack = SubStack(indir, substack_id, plist)
        plist = substack.plist
        substacks[substack.substack_id] = substack
    tx, ty, tz, idx = build_trees(substacks)
    C_locals = get_locals(C, tx, ty, tz, idx, substacks)

    pbar = ProgressBar(
        widgets=['Saving %d files: ' % len(C_locals),
                 Percentage()],
        maxval=len(C_locals)).start()
    for pbi, (substack_id, C_local) in enumerate(C_locals.iteritems()):
        mkdir_p(outdir + '/' + substack_id)
        substack.save_markers(outdir + '/' + substack_id + '/ms.marker',
                              C_local)
        pbar.update(pbi + 1)
    pbar.finish()
def make_pos_neg_dataset(tensor_view, ss, C, size=5, save_tiff_files=False, default_sigma=0.8, find_negative=True):
    tensor_view = tensor_view.astype(np.float32)

    H, W, D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
    patchlen = (1 + 2 * size) ** 3
    c_array = np.array([[c.x, c.y, c.z] for c in C])

    target_tensor_3d = np.zeros(tensor_view.shape)
    margin = ss.plist['Margin'] / 2
  

    trunc=1.5
    fixed_radiiassigned={c:default_sigma for c in C  if inside_margin(c,ss) > 0 }
    kdt = cKDTree(c_array)
    dist,ind = kdt.query(c_array,k=c_array.shape[0],distance_upper_bound=(2*default_sigma*trunc)+2.)
    for c,id_c in zip(C,xrange(c_array.shape[0])):
        if inside_margin(c, ss) > 0:
            for j in xrange(2,c_array.shape[0]):
                if not np.isinf(dist[id_c][j]):
                    if inside_margin(C[ind[id_c][j]], ss) > 0:
                        if trunc*(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]]) > dist[id_c][j] - 2.:
                            new_sigma=((dist[id_c][j]-1.)/trunc)*fixed_radiiassigned[c]/(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]])
                            fixed_radiiassigned[C[ind[id_c][j]]]=(new_sigma*fixed_radiiassigned[C[ind[id_c][j]]])/fixed_radiiassigned[c]
                            fixed_radiiassigned[c]=new_sigma


    for c in C:
        if inside_margin(c, ss) > 0:
            target_tensor_3d_tmp = np.zeros(tensor_view.shape)
            target_tensor_3d_tmp[c.z, c.y, c.x] = 1
            sigma=fixed_radiiassigned[c]
            target_tensor_3d_tmp = gfilter.gaussian_filter(target_tensor_3d_tmp, sigma,
                                               mode='constant', cval=0.0,
                                               truncate=trunc)
            target_tensor_3d_tmp = (target_tensor_3d_tmp.astype(np.float32) / np.max(target_tensor_3d_tmp))
            target_tensor_3d =  np.maximum(np.array(target_tensor_3d_tmp * 255.0, dtype=np.uint8), target_tensor_3d)



    if save_tiff_files:
        debug_path='/mnt/data/marco/experiments/markers/debug_single_view_on_fused_image'
        mkdir_p(debug_path)
        minz = 0
        target_tensor_3d = np.array(target_tensor_3d, dtype=np.uint8)
        imtensor.save_tensor_as_tif(target_tensor_3d, debug_path+'/'+ss.substack_id, minz)

    target_tensor_3d = (target_tensor_3d.astype(np.float32) / np.max(target_tensor_3d))
    tensor_view = tensor_view.astype(np.float32) / 255.0

    nrej_intensity = 0
    num_negative_targets = 0
    num_positive_targets = 0

    step_x = 1
    step_y = 1
    step_z = 1
    num_iterations = (W - 2*margin) * (H- 2*margin) * (D - 2*margin) / (step_x * step_y * step_z)
    pbar = ProgressBar(widgets=['Making positive and negative examples for %d points: ' % num_iterations, Percentage()],
                       maxval=num_iterations).start()

    X_positive = np.zeros((num_iterations, patchlen), dtype=np.float32)
    y_positive = np.zeros((num_iterations, patchlen), dtype=np.float32)

    X_negative = []
    y_negative = []
    if find_negative == True:
        X_negative = np.zeros((num_iterations, patchlen), dtype=np.float32)
        y_negative = np.zeros((num_iterations, patchlen), dtype=np.float32)

    pbi = 0
    for x0 in range(margin, W - margin, step_x):
        for y0 in range(margin, H - margin, step_y):
            for z0 in range(margin, D - margin, step_z):

                if inside_patch(c_array, x0, y0, z0, size,offset=2.0):  #found a marker inside margin
                    patch = tensor_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
                    X_positive[num_positive_targets, :] = np.reshape(patch, (patchlen,))
                    ypatch = target_tensor_3d[z0 - size:z0 + size + 1, y0 - size:y0 + size + 1,
                                x0 - size:x0 + size + 1]
                    y_positive[num_positive_targets, :] = np.reshape(ypatch, (patchlen,))
                    num_positive_targets += 1  # Sample as many negatives as positives

                elif find_negative == True:  #min_distance > (size * (3**(1/2.0))) + 2.0
                    patch = tensor_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
                    if np.mean(patch * 255) > 5:
                        X_negative[num_negative_targets, :] = np.reshape(patch, (patchlen,))
                        ypatch = target_tensor_3d[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
                        y_negative[num_negative_targets, :] = np.reshape(ypatch, (patchlen,))
                        num_negative_targets += 1
                    else:
                        nrej_intensity += 1
                pbar.update(pbi + 1)
                pbi += 1

    pbar.finish()


    X_positive = X_positive[0: num_positive_targets]
    y_positive = y_positive[0: num_positive_targets]
    print('Total positive examples for substack (', ss.substack_id, '):', num_positive_targets)

    if find_negative == True:

        print('Total negative examples for substack (', ss.substack_id, '):', num_negative_targets)
        print('Rejected by intensity ', nrej_intensity)
        X_negative = X_negative[0: num_negative_targets]
        y_negative = y_negative[0: num_negative_targets]

        if (num_negative_targets > num_positive_targets):
            print('Shuffling negative examples...')
            perm = np.random.permutation(len(X_negative)).astype(int)
            X_negative = X_negative[perm]
            y_negative = y_negative[perm]
            print('Shuffling done')
            X_negative = X_negative[0: num_positive_targets]
            y_negative = y_negative[0: num_positive_targets]


    return X_positive, y_positive, X_negative, y_negative
Exemplo n.º 21
0
def main(args):
    """
    Files must be numbered consecutively.
    """
    prefix = None
    #convert_to_grey = True

    savewd = os.getcwd()
    os.chdir(args.indir)
    full_path = os.getcwd()
    os.chdir(savewd)
    files = os.listdir(args.indir)
    files = [f for f in files if f[0] != '.' and f.endswith(args.suffix)]
    prefix = prefix or os.path.commonprefix(files)
    if prefix == '':
        raise Exception('Files in '+args.indir+' do not have a common prefix ' + str(files))
    # trailing zero dropped -- WHY???
    # zrange = sorted([int(str(f.split(prefix)[1].split(args.suffix)[0])[:-1]) for f in files])
    zrange = sorted([int(f.split('_')[-1].split('.')[0]) for f in files])
    # n_digits = len(files[0].split(prefix)[1].split(args.suffix)[0])
    n_digits = len(f.split('_')[-1].split('.')[0])
    if zrange != range(zrange[0], zrange[-1]+1):
        raise Exception('Files in '+args.indir+' are not a complete sequence: '+str(zrange))
    print('Checking image files')
    prefix = '_'.join(prefix.split('_')[:-1])
    for z in zrange:
        image_file = ('%s_%0'+str(n_digits)+'d'+args.suffix) % (prefix, z)
        img_z = Image.open(full_path+'/'+image_file)
        if z == zrange[0]:
            width, height = img_z.size
        else:
            if (width, height) != img_z.size:
                raise Exception('OOps, file', image_file, 'has size', img_z.size, 'instead of', (width, height))
    depth = len(zrange)

    print(full_path)
    full_path_of_substacks = "/".join(full_path.split('/')[:-2]) + '/substacks/' + "/".join(full_path.split('/')[-2:])

    print('Volume geometry:', width, height, depth)
    print('Finding substacks')
    substacks = dict()
    # nx,ny,nz=3,8,3 # Good for V000_maggio_large_substack_blurred
    w = int(round(float((width-args.margin))/float(args.nx)))
    h = int(round(float((height-args.margin))/float(args.ny)))
    d = int(round(float((depth-args.margin))/float(args.nz)))
    print(w,h,d)
    for i in range(args.nx):
        for j in range(args.ny):
            for k in range(args.nz):
                x0 = w*i
                y0 = h*j
                z0 = d*k
                x1 = min(width, x0+w+args.margin)
                y1 = min(height, y0+h+args.margin)
                z1 = min(depth, z0+d+args.margin)
                identifier = "%02d%02d%02d" % (i, j, k)
                substack = dict(Width=x1-x0, Height=y1-y0, Depth=z1-z0, X0=x0, Y0=y0, Z0=z0, Files=[])
                if args.verbose:
                    print('Substack', (i, j, k))
                    print('X:', x0, x1, w)
                    print('Y:', y0, y1, h)
                    print('Z:', z0, z1, d)
                    print('volume:', ((x1-x0)*(y1-y0)*(z1-z0))/1000000.0, 'MVoxels')
                substacks[identifier] = substack
                mkdir_p(full_path_of_substacks + '/' + identifier)

    num_slices = len(zrange)
    print('Saving cropped image files into', full_path_of_substacks)
    for z_from_zero, z in enumerate(zrange):
        image_file = ('%s_%0'+str(n_digits)+'d'+args.suffix) % (prefix, z)
        img_z = Image.open(full_path+'/'+image_file)
        if args.verbose:
            print(full_path+'/'+image_file, end='')
        print('==========',z+1,'/',num_slices,'=============')
        for substack_id, substack in substacks.iteritems():
            substack_dir = full_path_of_substacks+'/'+substack_id
            #mkdir_p(substack_dir)
            Z0 = substack['Z0']
            Depth = substack['Depth']
#            print(substack)
            if z_from_zero in range(Z0, Z0+Depth):
                X0 = substack['X0']
                Width = substack['Width']
                Y0 = substack['Y0']
                Height = substack['Height']
                region = img_z.crop((X0+0, Y0+0, X0+Width, Y0+Height))
                if args.convert_to_grey:
                    region = region.convert('L')
                if args.verbose:
                    print(' ', substack_id, end='')
                region.save(substack_dir+'/'+image_file)

                tmpfile = '/tmp/'+str(uuid.uuid4())+'.tif'
                region.save(tmpfile)
                subprocess.call(['tiffcp', '-clzw:2', tmpfile, substack_dir+'/'+image_file])
                os.unlink(tmpfile)

                # substack['Files'].append(substack_dir+'/'+image_file)
                # the path to files is now relative
                substack['Files'].append(substack_id+'/'+image_file)
        if args.verbose:
            print()

    print('Saving substack info into', full_path_of_substacks+'/info.json')
    info = dict(ImageSequenceDirectory=full_path,
                Width=width,
                Height=height,
                Depth=depth,
                Margin=args.margin,
                SubStacks=substacks,
                Full_Path_Of_Substacks=full_path_of_substacks)
    with open(full_path_of_substacks+'/info.json', 'w') as ostream:
        print(json.dumps(info), file=ostream)
Exemplo n.º 22
0
def merge_views(args):

    view_ids = args.view_ids
    max_len = 0
    hue = 0.
    C_substack = []

    if args.thresholds_dir is None:
        print(view_ids)
        list_views = get_visible_pairs(args.base_indir, args.substack_id,
                                       view_ids)
    else:
        list_views = get_visible_pairs_from_dir(args.thresholds_dir,
                                                args.substack_id, view_ids)

    substack = SubStack(args.base_indir + '/' + view_ids[0], args.substack_id)
    valid_pairs = filter_valid_pairs(args, substack, list_views)

    if len(valid_pairs.keys()) == 1:
        C_substack += valid_pairs.values()[0]
    else:
        keys_view_1 = []
        keys_view_3 = []
        for view_key in valid_pairs.keys():
            if view_key[0] == view_ids[2]:
                keys_view_3.append(view_key)
            elif view_key[0] == view_ids[0]:
                keys_view_1.append(view_key)

    if len(valid_pairs) == 2:
        if len(keys_view_1) == 2 or len(keys_view_3) == 2:
            total_list = compute_fusion(substack,
                                        valid_pairs.values()[0],
                                        valid_pairs.values()[1],
                                        args.max_distance,
                                        match_distance=args.match_distance,
                                        verbose=args.verbose)
            C_substack += total_list
        elif len(keys_view_1) == 1 and len(keys_view_3) == 1:
            C_view_3_t = transform_markers(args, substack,
                                           valid_pairs[keys_view_3[0]],
                                           view_ids)
            total_list = compute_fusion(substack,
                                        valid_pairs[keys_view_1[0]],
                                        C_view_3_t,
                                        args.max_distance,
                                        match_distance=args.match_distance,
                                        verbose=args.verbose)
            C_substack += total_list
        else:
            raise Exception('not valid list of views %s' % (str(valid_pairs)))

    elif len(valid_pairs) == 3:
        if len(keys_view_1) == 2:
            C_view_1 = compute_fusion(substack,
                                      valid_pairs[keys_view_1[0]],
                                      valid_pairs[keys_view_1[1]],
                                      args.max_distance,
                                      match_distance=args.match_distance,
                                      verbose=args.verbose)
            C_view_3_t = transform_markers(args, substack,
                                           valid_pairs[keys_view_3[0]],
                                           view_ids)
            total_list = compute_fusion(substack,
                                        C_view_1,
                                        C_view_3_t,
                                        args.max_distance,
                                        match_distance=args.match_distance,
                                        verbose=args.verbose)
            C_substack += total_list
        elif len(keys_view_3) == 2:
            C_view_3 = compute_fusion(substack,
                                      valid_pairs[keys_view_3[0]],
                                      valid_pairs[keys_view_3[1]],
                                      args.max_distance,
                                      match_distance=args.match_distance,
                                      verbose=args.verbose)
            C_view_3_t = transform_markers(args, substack, C_view_3, view_ids)
            C_view_1 = valid_pairs[keys_view_1[0]]
            total_list = compute_fusion(substack,
                                        C_view_1,
                                        C_view_3_t,
                                        args.max_distance,
                                        match_distance=args.match_distance,
                                        verbose=args.verbose)
            C_substack += total_list
        else:
            raise Exception('not valid list of views %s' % (str(valid_pairs)))

    elif len(valid_pairs) == 4:
        if len(keys_view_1) == 2 and len(keys_view_3) == 2:
            C_view_1 = compute_fusion(substack,
                                      valid_pairs[keys_view_1[0]],
                                      valid_pairs[keys_view_1[1]],
                                      args.max_distance,
                                      match_distance=args.match_distance,
                                      verbose=args.verbose)
            C_view_3 = compute_fusion(substack,
                                      valid_pairs[keys_view_3[0]],
                                      valid_pairs[keys_view_3[1]],
                                      args.max_distance,
                                      match_distance=args.match_distance,
                                      verbose=args.verbose)
            C_view_3_t = transform_markers(args, substack, C_view_3, view_ids)
            total_list = compute_fusion(substack,
                                        C_view_1,
                                        C_view_3_t,
                                        args.max_distance,
                                        match_distance=args.match_distance,
                                        verbose=args.verbose)
            C_substack += total_list
        else:
            raise Exception('not valid list of views %s' % (str(valid_pairs)))

    mkdir_p(args.outdir + '/' + args.substack_id)
    if len(C_substack) > 0:
        C_substack = filter_outside_markers(C_substack, substack)
        substack.save_markers(args.outdir + '/' + args.substack_id +
                              '/ms.marker',
                              C_substack,
                              floating_point=True)
Exemplo n.º 23
0
if __name__ == '__main__':
    if len(sys.argv) != 4:
        usage()
        sys.exit(2)
    indir = sys.argv[1]
    infile = sys.argv[2]
    outdir = sys.argv[3]

    substack_ids = map(string.strip,os.popen('ls '+indir).readlines())
    substack_ids = [f for f in substack_ids if len(f)==6]
    C = load_markers(infile)

    plist = None
    substacks = {}
    for substack_id in substack_ids:
        substack=SubStack(indir,substack_id, plist)
        plist = substack.plist
        substacks[substack.substack_id] = substack
    tx, ty, tz, idx = build_trees(substacks)
    C_locals = get_locals(C, tx, ty, tz, idx, substacks)

    pbar = ProgressBar(widgets=['Saving %d files: ' % len(C_locals), Percentage()],
                       maxval=len(C_locals)).start()
    for pbi, (substack_id, C_local) in enumerate(C_locals.iteritems()):
        mkdir_p(outdir + '/' + substack_id)
        substack.save_markers(outdir + '/' + substack_id + '/ms.marker',C_local)
        pbar.update(pbi+1)
    pbar.finish()