Exemplo n.º 1
0
def get_roi_im(w, h):
    sz = FastImage.Size(w, h)
    roi_im = FastImage.FastImage8u(sz)
    arr = numpy.asarray(roi_im)  # numpy view
    arr[:, :] = 0
    arr[YT, XT] = 50
    return roi_im, result_validator_func
Exemplo n.º 2
0
def test_sobel_horiz_1():
    a = np.linspace(0,255,40000).astype(np.float32)
    a.shape=200,200
    imA=fi.asfastimage(a)
    result = np.asarray(imA.sobel_horiz(imA.size))

    b = np.ones_like(a)
    imB=fi.asfastimage(b)
    imA.sobel_horiz(imA.size,dest=imB)
    assert np.allclose(result,b)
Exemplo n.º 3
0
def play_func(loaded_trx, im_pts_segs_q, playing ):
    playing.set()
    try:
        n_frames = loaded_trx['n_frames']
        all_vals = loaded_trx['all_vals']
        fmf = loaded_trx['fmf']
        bg_image = loaded_trx['bg_image']
        tracker = loaded_trx['tracker']
        cam_id = loaded_trx['cam_id']
        format = loaded_trx['format']

        fibg = FastImage.asfastimage(bg_image)
        smallframe_size = None

        fmf.seek(0)
        #for fno in range(671,672):#n_frames):
        for fno in range(n_frames):
            # reconstruct original frame #################
            posx, posy, orientation, windowx, windowy, data_timestamp = all_vals[fno][:6]
            smallframe,fmf_timestamp = fmf.get_frame(fno)
            fismall = FastImage.asfastimage(smallframe)
            assert fmf_timestamp == data_timestamp
            timestamp = fmf_timestamp
            if 0:
                fullsize_image = fibg.get_8u_copy(fibg.size)
                software_roi = fullsize_image.roi( windowx, windowy, fismall.size )
                fismall.get_8u_copy_put( software_roi, fismall.size )
            else:
                fullsize_image = bg_image.copy()
                fullsize_image[ windowy:windowy+smallframe.shape[0], windowx:windowx+smallframe.shape[1]] = smallframe
                fullsize_image = FastImage.asfastimage(fullsize_image)

            # process with flytrax #################
            buf_offset=0,0
            framenumber=fno
            points,linesegs = tracker.process_frame(cam_id,
                                                    fullsize_image,
                                                    buf_offset,
                                                    timestamp,
                                                    framenumber)
            tup = fullsize_image, points, linesegs
            im_pts_segs_q.put( tup )
            #time.sleep(1e-2)
    finally:
        playing.clear()
Exemplo n.º 4
0
def test_get_8u_copy():
    a = np.linspace(0,255,40000).astype(np.float32)
    a.shape=200,200

    imA=fi.asfastimage(a)
    imB=imA.get_8u_copy(imA.size)
    b_test = np.asarray(imB)
    diff = a-b_test
    assert np.all(abs(diff)<1.0)
Exemplo n.º 5
0
    def test_mem(self):
        size = fi.Size(20,10)
        for (fif,ar_dtype) in zip(self.fastimagefactories,self.ar_dtypes):
            imA=fif(size)
            imA.set_val(0,size)
            arA = np.zeros((size.h, size.w),ar_dtype)

            imB=fi.copy(imA)
            imB.set_val(2,size)
            self.assert_( not np.allclose(np.asarray(imA),np.asarray(imB)) )
Exemplo n.º 6
0
    def test_from_nx2(self):

        A = np.array(32.2,np.float32)
        sz = fi.Size(33,323)
        arA = np.array(A,np.float32)*np.ones((sz.h, sz.w),np.float32)

        imA=fi.copy(arA)
        arA[1,3:40] = 3024.03

        self.assert_( not np.allclose(arA, np.asarray(imA)))
Exemplo n.º 7
0
def test_conversion_step():
    sz = fi.Size(660,480)
    cls_dtype_nbytes = [ (fi.FastImage8u, np.uint8, 1),
                         (fi.FastImage32f, np.float32, 4),
                         ]
    for cls, dtype, nbytes in cls_dtype_nbytes:
        imA=cls(sz)

        npB = np.empty( (sz.h,imA._step//nbytes), dtype=dtype )
        npB = npB[:,:sz.w]

        imB = fi.asfastimage( npB )

        assert imA._step == imB._step
        assert imA.size == imB.size
Exemplo n.º 8
0
    def test_mem(self):
        size = fi.Size(20,10)
        for (fif,ar_dtype) in zip(self.fastimagefactories,self.ar_dtypes):
            imA=fif(size)
            imA.set_val(0,size)
            arA = np.zeros((size.h, size.w),ar_dtype)

            imB=fi.copy(imA)
            imB.set_val(2,size)
            self.assert_( not np.allclose(np.asarray(imA),np.asarray(imB)) )

            # check that views work
            imA.set_val(0,size)
            arrA1 = np.array(imA,copy=False)
            arrA2 = np.array(imA,copy=False)
            arrA1[2,5] = 12
            assert(np.allclose(arrA1,arrA2))

            # check that copy isn't view
            imA.set_val(0,size)
            arrA1 = np.array(imA,copy=True)
            arrA2 = np.array(imA,copy=True)
            arrA1[2,5] = 12
            assert(not np.allclose(arrA1,arrA2))
Exemplo n.º 9
0
def retrack_movies(
    h5_filename,
    output_h5_filename=None,
    max_n_frames=None,
    start=None,
    stop=None,
    ufmf_dir=None,
    cfg_filename=None,
    ufmf_filenames=None,
    save_debug_images=False,
):

    # 2D data format for PyTables:
    Info2D = flydra_core.data_descriptions.Info2D

    if ufmf_filenames is None:
        ufmf_filenames = auto_discover_ufmfs.find_ufmfs(h5_filename,
                                                        ufmf_dir=ufmf_dir,
                                                        careful=True)
    print("ufmf_filenames: %r" % ufmf_filenames)
    if len(ufmf_filenames) == 0:
        raise RuntimeError(
            "nothing to do (autodetection of .ufmf files failed)")

    if ufmf_dir is not None:
        if (not ufmf_filenames[0].startswith("/")) and (not os.path.isfile(
                ufmf_filenames[0])):
            # filenames are not absolute and are not present, convert
            ufmf_filenames = [
                os.path.join(ufmf_dir, fname) for fname in ufmf_filenames
            ]
        else:
            raise RuntimeError(
                "ufmf_dir given but ufmf_filenames exist without it")

    if os.path.exists(output_h5_filename):
        raise RuntimeError("will not overwrite old file '%s'" %
                           output_h5_filename)

    # get name of data
    config = get_config_defaults()
    if cfg_filename is not None:
        loaded_cfg = cherrypy.lib.reprconf.as_dict(cfg_filename)
        for section in loaded_cfg:
            config[section].update(loaded_cfg.get(section, {}))
    default_camcfg = config["default"]
    for cam_id in config.keys():
        if cam_id == "default":
            continue
        # ensure default key/value pairs in each cam_id
        for key, value in default_camcfg.iteritems():
            if key not in config[cam_id]:
                config[cam_id][key] = value

    datetime_str = os.path.splitext(os.path.split(h5_filename)[-1])[0]
    datetime_str = datetime_str[4:19]

    retrack_cam_ids = [
        ufmf_tools.get_cam_id_from_ufmf_fname(f) for f in ufmf_filenames
    ]

    with open_file_safe(h5_filename, mode="r") as h5:

        # Find camns in original data
        camn2cam_id, cam_id2camns = result_utils.get_caminfo_dicts(h5)

        retrack_camns = []
        for cam_id in retrack_cam_ids:
            retrack_camns.extend(cam_id2camns[cam_id])
        all_camns = camn2cam_id.keys()

        # Save results to temporary file. Copy to real location on success.
        tmpdir = tempfile.mkdtemp()
        tmp_output_h5_filename = os.path.join(tmpdir, "retrack.h5")

        with open_file_safe(tmp_output_h5_filename,
                            mode="w",
                            delete_on_error=True) as output_h5:

            out_data2d = output_h5.create_table(
                output_h5.root,
                "data2d_distorted",
                Info2D,
                "2d data",
                expectedrows=h5.root.data2d_distorted.nrows,
            )

            # Are there any camns in original h5 that are not being retracked?
            if len(set(all_camns) - set(retrack_camns)):
                # Yes.

                # OK, exclude all camns to be retracked...
                orig_data2d = h5.root.data2d_distorted[:]  # read all data
                for camn in retrack_camns:
                    delete_cond = orig_data2d["camn"] == camn
                    save_cond = ~delete_cond
                    orig_data2d = orig_data2d[save_cond]

                # And save original data for untouched camns
                out_data2d.append(orig_data2d)

            for input_node in h5.root._f_iter_nodes():
                if input_node._v_name not in [
                        "data2d_distorted",
                        "kalman_estimates",
                        "ML_estimates",
                        "ML_estimates_2d_idxs",
                ]:
                    print("copying", input_node._v_name)
                    # copy everything from source to dest
                    input_node._f_copy(output_h5.root, recursive=True)

            fpc = realtime_image_analysis.FitParamsClass(
            )  # allocate FitParamsClass

            count = 0
            iterate_frames = ufmf_tools.iterate_frames  # shorten notation
            for frame_enum, (frame_dict, frame) in enumerate(
                    iterate_frames(
                        h5_filename,
                        ufmf_filenames,
                        max_n_frames=max_n_frames,
                        start=start,
                        stop=stop,
                    )):

                if (frame_enum % 100) == 0:
                    print("%s: frame %d" % (datetime_str, frame))

                for ufmf_fname in ufmf_filenames:
                    try:
                        frame_data = frame_dict[ufmf_fname]
                    except KeyError:
                        # no data saved (frame skip on Prosilica camera?)
                        continue
                    count += 1
                    camn = frame_data["camn"]
                    cam_id = frame_data["cam_id"]
                    camcfg = config.get(cam_id, default_camcfg)
                    image = frame_data["image"]
                    cam_received_timestamp = frame_data[
                        "cam_received_timestamp"]
                    timestamp = frame_data["timestamp"]

                    detected_points = True
                    obj_slices = None
                    if len(frame_data["regions"]) == 0:
                        # no data this frame -- go to next camera or frame
                        detected_points = False
                    if detected_points:
                        # print frame,cam_id,len(frame_data['regions'])
                        absdiff_im = abs(
                            frame_data["mean"].astype(np.float32) - image)
                        thresh_val = (np.max(absdiff_im) *
                                      camcfg["absdiff_max_frac_thresh"])
                        thresh_val = max(camcfg["min_absdiff"], thresh_val)
                        thresh_im = absdiff_im > thresh_val
                        labeled_im, n_labels = scipy.ndimage.label(thresh_im)
                        if not n_labels:
                            detected_points = False
                        else:
                            obj_slices = scipy.ndimage.find_objects(labeled_im)
                    detection = out_data2d.row
                    if detected_points:
                        height, width = image.shape
                        if save_debug_images:
                            xarr = []
                            yarr = []
                        frame_pt_idx = 0
                        detected_points = False  # possible not to find any below

                        for i in range(n_labels):
                            y_slice, x_slice = obj_slices[i]
                            # limit pixel operations to covering rectangle
                            this_labeled_im = labeled_im[y_slice, x_slice]
                            this_label_im = this_labeled_im == (i + 1)

                            # calculate area (number of binarized pixels)
                            xsum = np.sum(this_label_im, axis=0)
                            pixel_area = np.sum(xsum)
                            if pixel_area < camcfg["area_minimum_threshold"]:
                                continue

                            # calculate center
                            xpos = np.arange(x_slice.start, x_slice.stop,
                                             x_slice.step)
                            ypos = np.arange(y_slice.start, y_slice.stop,
                                             y_slice.step)

                            xmean = np.sum((xsum * xpos)) / np.sum(xsum)
                            ysum = np.sum(this_label_im, axis=1)
                            ymean = np.sum((ysum * ypos)) / np.sum(ysum)

                            if 1:
                                if camcfg["pixel_aspect"] == 1:
                                    this_fit_im = this_label_im
                                elif camcfg["pixel_aspect"] == 2:
                                    this_fit_im = np.repeat(this_label_im,
                                                            2,
                                                            axis=0)
                                else:
                                    raise ValueError("unknown pixel_aspect")

                                fast_foreground = FastImage.asfastimage(
                                    this_fit_im.astype(np.uint8))

                                fail_fit = False
                                try:
                                    (
                                        x0_roi,
                                        y0_roi,
                                        weighted_area,
                                        slope,
                                        eccentricity,
                                    ) = fpc.fit(fast_foreground)
                                except realtime_image_analysis.FitParamsError as err:
                                    fail_fit = True
                                    print("frame %d, ufmf %s: fit failed" %
                                          (frame, ufmf_fname))
                                    print(err)
                                else:
                                    if camcfg["pixel_aspect"] == 2:
                                        y0_roi *= 0.5
                                    xmean = x_slice.start + x0_roi
                                    ymean = y_slice.start + y0_roi
                                    del weighted_area  # don't leave room for confusion
                            else:
                                fail_fit = True

                            if fail_fit:
                                slope = np.nan
                                eccentricity = np.nan

                            detection["camn"] = camn
                            detection["frame"] = frame
                            detection["timestamp"] = timestamp
                            detection[
                                "cam_received_timestamp"] = cam_received_timestamp
                            detection["x"] = xmean
                            detection["y"] = ymean
                            detection["area"] = pixel_area
                            detection["slope"] = slope
                            detection["eccentricity"] = eccentricity
                            detection["frame_pt_idx"] = frame_pt_idx
                            # XXX These are not yet implemented:
                            detection["cur_val"] = 0
                            detection["mean_val"] = np.nan
                            detection["sumsqf_val"] = np.nan
                            frame_pt_idx += 1
                            if save_debug_images:
                                xarr.append(xmean)
                                yarr.append(ymean)
                            detection.append()
                            detected_points = True

                        if save_debug_images:
                            save_dir = "debug"
                            mkdir_p(save_dir)
                            save_fname = "debug_%s_%d.png" % (cam_id, frame)
                            save_fname_path = os.path.join(
                                save_dir, save_fname)
                            print("saving", save_fname_path)
                            from . import benu

                            canv = benu.Canvas(save_fname_path, width, height)
                            maxlabel = np.max(labeled_im)
                            fact = int(np.floor(255.0 / maxlabel))
                            canv.imshow((labeled_im * fact).astype(np.uint8),
                                        0, 0)
                            canv.scatter(
                                xarr,
                                yarr,
                                color_rgba=(0, 1, 0, 1),
                                radius=10,
                            )
                            canv.save()

                    if not detected_points:
                        # If no point was tracked for this frame,
                        # still save timestamp.
                        detection["camn"] = camn
                        detection["frame"] = frame
                        detection["timestamp"] = timestamp
                        detection[
                            "cam_received_timestamp"] = cam_received_timestamp
                        detection["x"] = np.nan
                        detection["y"] = np.nan
                        detection["area"] = np.nan
                        detection["slope"] = np.nan
                        detection["eccentricity"] = np.nan
                        detection["frame_pt_idx"] = 0
                        detection["cur_val"] = 0
                        detection["mean_val"] = np.nan
                        detection["sumsqf_val"] = np.nan
                        detection.append()
            if count == 0:
                raise RuntimeError("no frames processed")

    # move to correct location
    shutil.move(tmp_output_h5_filename, output_h5_filename)
Exemplo n.º 10
0
import sys
import os.path
import numpy
from setuptools import setup, Extension, find_packages
from Cython.Build import cythonize

import pkg_resources # make sure FastImage is importable
import motmot.FastImage as fi_mod
import motmot.FastImage.FastImage as FastImage
import motmot.FastImage.util as FastImage_util

# build with same IPP as FastImage
ipp_root = os.environ['IPPROOT']
vals = FastImage_util.get_build_info(ipp_arch=FastImage.get_IPP_arch(),
                                     ipp_static=FastImage.get_IPP_static(),
                                     ipp_root=ipp_root)

ext_modules = []

if 1:
    realtime_image_analysis_sources=['motmot/realtime_image_analysis/realtime_image_analysis.pyx',
                                     'src/c_fit_params.cpp',
                                     'src/eigen.c',
                                     'src/c_time_time.c',
                                     ]
    ext_modules.append(Extension(name='motmot.realtime_image_analysis.realtime_image_analysis',
                                 sources=realtime_image_analysis_sources,
                                 include_dirs=vals['ipp_include_dirs']+['src']+[numpy.get_include(), fi_mod.get_include()],
                                 library_dirs=vals['ipp_library_dirs'],
                                 libraries=vals['ipp_libraries'],
                                 define_macros=vals['ipp_define_macros'],
Exemplo n.º 11
0
version = flydra_camnode.version.__version__

import numpy as np

ext_modules = []

if 1:
    import motmot.FastImage.FastImage
    import motmot.FastImage as fi_mod
    FastImage = motmot.FastImage.FastImage

    import motmot.FastImage.util as FastImage_util
    IPPROOT = os.environ['IPPROOT']

    vals = FastImage_util.get_build_info(
        ipp_arch=FastImage.get_IPP_arch(),
        ipp_root=IPPROOT,
    )

    ext_modules.append(
        Extension(
            name='flydra_camnode.camnode_colors',
            sources=[
                'flydra_camnode/camnode_colors.pyx', 'flydra_camnode/colors.c'
            ],
            include_dirs=vals['ipp_include_dirs'] +
            [np.get_include(), fi_mod.get_include()],
            library_dirs=vals['ipp_library_dirs'],
            libraries=vals['ipp_libraries'],
            define_macros=vals['ipp_define_macros'],
            extra_link_args=vals['extra_link_args'],
Exemplo n.º 12
0
    def test_fast_vs_slow(self):
        h,w = 3,5
        shape = h,w # 3 rows, 5 cols

        results = []
        for func in [slow.do_bg_maint,
                     ria.do_bg_maint]:

            running_mean_im = 4*numpy.ones( shape, dtype=numpy.float32 )
            hw_roi_frame = 5*numpy.ones( shape, dtype=numpy.uint8 )
            max_frame_size = FastImage.Size( w,h )
            ALPHA = 0.25
            running_mean8u_im = numpy.empty( shape, dtype=numpy.uint8 )
            fastframef32_tmp = numpy.empty( shape, dtype=numpy.float32 )
            running_sumsqf = 16*numpy.ones( shape, dtype=numpy.float32 )
            mean2 = numpy.empty( shape, dtype=numpy.float32 )
            std2 = numpy.empty( shape, dtype=numpy.float32 )
            running_stdframe = numpy.empty( shape, dtype=numpy.float32 )
            n_sigma = 2.0
            compareframe8u = numpy.empty( shape, dtype=numpy.uint8 )
            bright_non_gaussian_cutoff = 255
            noisy_pixels_mask = numpy.ones( shape, dtype=numpy.uint8 )
            bright_non_gaussian_replacement = 5
            bench = 0

            func( FastImage.asfastimage(running_mean_im),
                  FastImage.asfastimage(hw_roi_frame),
                  max_frame_size,
                  ALPHA,
                  FastImage.asfastimage(running_mean8u_im),
                  FastImage.asfastimage(fastframef32_tmp),
                  FastImage.asfastimage(running_sumsqf),
                  FastImage.asfastimage(mean2),
                  FastImage.asfastimage(std2),
                  FastImage.asfastimage(running_stdframe),
                  n_sigma,
                  FastImage.asfastimage(compareframe8u),
                  bright_non_gaussian_cutoff,
                  FastImage.asfastimage(noisy_pixels_mask),
                  bright_non_gaussian_replacement,
                  bench=bench)

            results_order = ('running_mean8u_im',
                             'fastframef32_tmp',
                             'running_sumsqf',
                             'mean2',
                             'std2',
                             'running_stdframe',
                             )
            this_results = [ locals()[name] for name in results_order ]
            results.append( this_results )

        for i,(slow_result_arr, fast_result_arr) in enumerate(zip(*results)):
            name = results_order[i]
            if 0:
                print name
                print slow_result_arr
                print fast_result_arr
                print
            assert slow_result_arr.shape == fast_result_arr.shape
            assert numpy.allclose( slow_result_arr, fast_result_arr )
Exemplo n.º 13
0
 def test_arch(self):
     arch = fi.get_IPP_arch()
Exemplo n.º 14
0
##im.show()

##print np.__version__

##nview = np.asarray( im )
##print ai

##print nview
##nview[0,1] = 240
##print nview

##im.show()

a=np.arange(12).astype(np.uint8)
a=np.reshape(a,(3,4))
b=fi.asfastimage(a)
print b.stringview()

b.set_val(1,b.size)
br = b.roi(1,1,fi.Size(1,1))
br.set_val(10,br.size)
print a

c=fi.copy(a.astype(np.float32))
print c.stringview()

print b.stringview()
#print 'c.size == b.size',c.size == b.size

c.toself_add_weighted( b, c.size, 0.1 )
print c.stringview()
Exemplo n.º 15
0
                                                       N=stack_N_images,
                                                       )

                    # The variable fno (the first element of the results
                    # tuple) is guaranteed to be contiguous and to span
                    # the range from the first to last frames available.

                    for (fno, av_im, lowerleft,
                         orig_data2d_rownum, orig_idx,
                         orig_idxs_in_average) in results:

                        # Clip image to reduce moment arms.
                        av_im[av_im <= final_thresh] = 0

                        fail_fit = False
                        fast_av_im = FastImage.asfastimage( av_im.astype(np.uint8) )
                        try:
                            (x0_roi, y0_roi, area, slope, eccentricity) = fpc.fit(
                                fast_av_im )
                        except realtime_image_analysis.FitParamsError, err:
                            fail_fit = True

                        this_morph_failures = morph_failures[orig_idxs_in_average]
                        n_failed_images = np.sum( this_morph_failures)
                        n_good_images = stack_N_images-n_failed_images
                        if n_good_images >= stack_N_images_min:
                            n_images_is_acceptable = True
                        else:
                            n_images_is_acceptable = False

                        if fail_fit:
Exemplo n.º 16
0
    def test_fast_vs_slow(self):
        h, w = 3, 5
        shape = h, w  # 3 rows, 5 cols

        results = []
        for func in [slow.do_bg_maint, ria.do_bg_maint]:

            running_mean_im = 4 * numpy.ones(shape, dtype=numpy.float32)
            hw_roi_frame = 5 * numpy.ones(shape, dtype=numpy.uint8)
            max_frame_size = FastImage.Size(w, h)
            ALPHA = 0.25
            running_mean8u_im = numpy.empty(shape, dtype=numpy.uint8)
            fastframef32_tmp = numpy.empty(shape, dtype=numpy.float32)
            running_sumsqf = 16 * numpy.ones(shape, dtype=numpy.float32)
            mean2 = numpy.empty(shape, dtype=numpy.float32)
            std2 = numpy.empty(shape, dtype=numpy.float32)
            running_stdframe = numpy.empty(shape, dtype=numpy.float32)
            n_sigma = 2.0
            compareframe8u = numpy.empty(shape, dtype=numpy.uint8)
            bright_non_gaussian_cutoff = 255
            noisy_pixels_mask = numpy.ones(shape, dtype=numpy.uint8)
            bright_non_gaussian_replacement = 5
            bench = 0

            func(FastImage.asfastimage(running_mean_im),
                 FastImage.asfastimage(hw_roi_frame),
                 max_frame_size,
                 ALPHA,
                 FastImage.asfastimage(running_mean8u_im),
                 FastImage.asfastimage(fastframef32_tmp),
                 FastImage.asfastimage(running_sumsqf),
                 FastImage.asfastimage(mean2),
                 FastImage.asfastimage(std2),
                 FastImage.asfastimage(running_stdframe),
                 n_sigma,
                 FastImage.asfastimage(compareframe8u),
                 bright_non_gaussian_cutoff,
                 FastImage.asfastimage(noisy_pixels_mask),
                 bright_non_gaussian_replacement,
                 bench=bench)

            results_order = (
                'running_mean8u_im',
                'fastframef32_tmp',
                'running_sumsqf',
                'mean2',
                'std2',
                'running_stdframe',
            )
            this_results = [locals()[name] for name in results_order]
            results.append(this_results)

        for i, (slow_result_arr, fast_result_arr) in enumerate(zip(*results)):
            name = results_order[i]
            if 0:
                print name
                print slow_result_arr
                print fast_result_arr
                print
            assert slow_result_arr.shape == fast_result_arr.shape
            assert numpy.allclose(slow_result_arr, fast_result_arr)
Exemplo n.º 17
0
 def test_static(self):
     result = fi.get_IPP_static()
Exemplo n.º 18
0
    def process_frame(self, cam_id, buf, buf_offset, timestamp, framenumber):
        """do work on each frame

        This function gets called on every single frame capture. It is
        called within the realtime thread, NOT the wxPython
        application mainloop's thread. Therefore, be extremely careful
        (use threading locks) when sharing data with the rest of the
        class.

        """
        assert self.pixel_format == "MONO8"

        now = time.time()
        if self.pub_image is not None:
            if (now - self.pub_last_image) > 30.0:
                msg = self.pub_image_class()
                msg.header.seq = framenumber
                # XXX TODO: once camera trigger is ROS node, get accurate timestamp
                msg.header.stamp = self._rospy_time_from_sec(now)
                msg.header.frame_id = "0"

                npbuf = np.array(buf)
                (height, width) = npbuf.shape

                msg.height = height
                msg.width = width
                msg.encoding = "mono8"
                msg.step = width
                msg.data = npbuf.tostring()  # let numpy convert to string

                self.pub_image.publish(msg)
                self.pub_last_image = now

        if self.pub_mean_luminance is not None:
            if (now - self.pub_last_mean_luminance) > 0.5:
                mean_lum = np.mean(buf)
                self.pub_mean_luminance.publish(float(mean_lum))
                self.pub_last_mean_luminance = now

        buf = FastImage.asfastimage(buf)

        ros_list = []
        point_list = []
        draw_linesegs = []  # [ (x0,y0,x1,y1) ]
        if self.enabled.isSet():

            light_on_dark = self.light_on_dark.isSet()
            offset_x, offset_y = buf_offset
            if 1:

                # work on a copy of the image so the original is displayed unaltered
                copyview = self.copybuf.roi(offset_x, offset_y, buf.size)
                buf.get_8u_copy_put(copyview, buf.size)
                buf = copyview
            buf_view = np.asarray(buf)  # get numpy view of data

            # step 1. extract points
            analysis_radius = self.analysis_radius.get()
            luminance_threshold = self.luminance_threshold.get()
            if light_on_dark:
                clearval = 0
            else:
                clearval = 255

            # outside mask, set values to clearval
            mask = self.get_mask()
            buf_view[mask] = clearval

            for pt_num in range(self.num_points.get()):
                if light_on_dark:
                    # find brightest point
                    max_val, x, y = buf.max_index(buf.size)
                    if max_val < luminance_threshold:
                        break
                else:
                    # find darkest point
                    min_val, x, y = buf.min_index(buf.size)
                    if min_val > (255 - luminance_threshold):
                        break

                # calculate region around found point
                clearxmin = max(0, x - analysis_radius)
                clearxmax = min(buf.size.w - 1, x + analysis_radius)
                clearymin = max(0, y - analysis_radius)
                clearymax = min(buf.size.h - 1, y + analysis_radius)

                # extract a view of this region
                this_region = buf_view[clearymin:clearymax, clearxmin:clearxmax]
                if light_on_dark:
                    binary_region = this_region > (max_val / 1.1)
                else:
                    binary_region = this_region < (min_val * 1.1)
                num_pixels_classified = np.sum(binary_region.ravel())
                # print '%d, (%d, %d)'%(num_pixels_classified,x,y)

                if num_pixels_classified > self.max_area.get():
                    # we don't want this point
                    pass
                else:
                    # compute luminance center of mass
                    if not light_on_dark:
                        # make a light-on-dark image
                        this_region2 = 255 - this_region
                    else:
                        this_region2 = this_region
                    fibuf = FastImage.asfastimage(this_region2)
                    try:
                        results = realtime_image_analysis.py_fit_params(fibuf)
                    except Exception as err:
                        print "%s: error extracting image data. ignoring." % (err,)
                    else:
                        (x0, y0, area, slope, eccentricity) = results
                        theta = np.arctan(slope)

                        x1 = x0 + clearxmin
                        y1 = y0 + clearymin

                        # save values
                        ros_list.append((offset_x + x1, offset_y + y1, theta))
                        point_list.append((offset_x + x1, offset_y + y1))

                # clear the region near the detected point
                buf_view[clearymin:clearymax, clearxmin:clearxmax] = clearval

            # send data over ROS
            if self.num_points.get():
                if self.pub_position is not None:
                    msg = self.pub_position_class()

                    msg.header.stamp.secs = int(np.floor(timestamp))
                    msg.header.stamp.nsecs = int((timestamp % 1.0) * 1e9)
                    msg.header.frame_id = "pixels"

                    msg.framenumber = framenumber

                    for (x, y, theta) in ros_list:
                        pose = self.pub_pose_class()
                        pose.x = x
                        pose.y = y
                        pose.theta = theta
                        msg.points.append(pose)
                    self.pub_position.publish(msg)

        if self.view_mask_mode.isSet():

            w, h = self.image_size
            x = self.mask_center_x.get()
            y = self.mask_center_y.get()
            radius = self.mask_radius.get()

            draw_linesegs.extend(lineseg_circle(x, y, radius))

        return point_list, draw_linesegs
Exemplo n.º 19
0
    def process_frame(self,cam_id,buf,buf_offset,timestamp,framenumber):
        if self.pixel_format[cam_id]=='YUV422':
            buf = imops.yuv422_to_mono8( numpy.asarray(buf) ) # convert
        elif not (self.pixel_format[cam_id].startswith('MONO8') or
                  self.pixel_format[cam_id].startswith('RAW8')):
            warnings.warn("flytrax plugin incompatible with data format")
            return [], []


        bunch = self.bunches[cam_id]

        do_bg_maint = False
        clear_and_take_bg_image = self.clear_and_take_bg_image[cam_id]

        # this is called in realtime thread
        fibuf = FastImage.asfastimage(buf) # FastImage view of image data (hardware ROI)
        l,b = buf_offset
        lbrt = l, b, l+fibuf.size.w-1, b+fibuf.size.h-1

        running_mean_im = bunch.running_mean_im_full.roi(l, b, fibuf.size)  # set ROI view
        running_sumsqf = bunch.running_sumsqf_full.roi(l, b, fibuf.size)  # set ROI view

        new_clear_threshold = self.new_clear_threshold[cam_id]
        new_diff_threshold = self.new_diff_threshold[cam_id]
        new_roi2_radius = self.new_roi2_radius[cam_id]
        realtime_analyzer = self.realtime_analyzer[cam_id]
        realtime_analyzer.roi = lbrt # hardware ROI
        max_frame_size = self.max_frame_size[cam_id]
        display_active = self.display_active[cam_id]

        use_roi2 = True

        use_cmp = False # use variance-based background subtraction/analysis
        draw_points = []
        draw_linesegs = []

        running_mean8u_im_full = realtime_analyzer.get_image_view('mean')
        running_mean8u_im = running_mean8u_im_full.roi(l, b, fibuf.size)

        if (bunch.initial_take_bg_state is not None or
            clear_and_take_bg_image.isSet()):
            src_fullframe_fi = fibuf.get_8u_copy(max_frame_size)

        if bunch.initial_take_bg_state is not None:
            assert bunch.initial_take_bg_state == 'gather'

            n_initial_take = 5
            bunch.initial_take_frames.append( numpy.array(src_fullframe_fi) ) # copied above
            if len( bunch.initial_take_frames ) >= n_initial_take:

                initial_take_frames = numpy.array( bunch.initial_take_frames, dtype=numpy.float32 )
                mean_frame = numpy.mean( initial_take_frames, axis=0)
                sumsqf_frame = numpy.sum(initial_take_frames**2, axis=0)/len( initial_take_frames )

                numpy.asarray(running_mean_im)[:,:] = mean_frame
                numpy.asarray(running_sumsqf)[:,:] = sumsqf_frame

                # we're done with initial transient, set stuff
                do_bg_maint = True
                bunch.initial_take_bg_state = None
                bunch.initial_take_frames = []

        if clear_and_take_bg_image.isSet():
            bunch.initial_take_bg_state = 'gather'
            bunch.initial_take_frames = []
            with self.bg_update_lock:
                bunch.last_running_mean_im = None
            clear_and_take_bg_image.clear()

        if 1:
            self.ticks_since_last_update[cam_id] += 1

            update_interval = self.ongoing_bg_image_update_interval[cam_id].get()
            if self.ticks_since_last_update[cam_id]%update_interval == 0:
                do_bg_maint = True

        if do_bg_maint:
            hw_roi_frame = fibuf
            cur_fisize = hw_roi_frame.size
            bg_frame_alpha = 1.0/bunch.inverse_alpha.get_nowait()
            n_sigma = bunch.n_sigma.get_nowait()
            bright_non_gaussian_cutoff = 255
            bright_non_gaussian_replacement = 255

            compareframe8u_full = realtime_analyzer.get_image_view('cmp')
            compareframe8u = compareframe8u_full.roi(l, b, fibuf.size)
            fastframef32_tmp = bunch.fastframef32_tmp_full.roi(l, b, fibuf.size)

            mean2 = bunch.mean2_full.roi(l, b, fibuf.size)
            std2  =  bunch.std2_full.roi(l, b, fibuf.size)
            running_stdframe = bunch.running_stdframe_full.roi(l, b, fibuf.size)

            noisy_pixels_mask = bunch.noisy_pixels_mask_full.roi(l, b, fibuf.size)

            realtime_image_analysis.do_bg_maint(
                running_mean_im,#in
                hw_roi_frame,#in
                cur_fisize,#in
                bg_frame_alpha, #in
                running_mean8u_im,
                fastframef32_tmp,
                running_sumsqf, #in
                mean2,
                std2,
                running_stdframe,
                n_sigma,#in
                compareframe8u,
                bright_non_gaussian_cutoff,#in
                noisy_pixels_mask,#in
                bright_non_gaussian_replacement,#in
                bench=0 )
                #debug=0)
            #chainbuf.real_std_est= tmpresult
            bg_changed = True
            bg_frame_number = 0

            with self.bg_update_lock:
                bunch.last_running_mean_im = running_mean_im # XXX should copy?
                bunch.last_running_sumsqf_image = running_sumsqf # XXX should copy?
                bunch.last_bgcmp_image_timestamp = timestamp

        if new_clear_threshold.isSet():
            nv = self.clear_threshold_value[cam_id]
            realtime_analyzer.clear_threshold = nv
            #print 'set clear',nv
            new_clear_threshold.clear()

        if new_diff_threshold.isSet():
            nv = self.diff_threshold_value[cam_id]
            realtime_analyzer.diff_threshold = nv
            #print 'set diff',nv
            new_diff_threshold.clear()

        if new_roi2_radius.isSet():
            nv = self.roi2_radius_value[cam_id]
            realtime_analyzer.roi2_radius = nv
            new_roi2_radius.clear()

        n_pts = 0
        with self.ufmf_writer_lock:
            with self.tracking_enabled_lock:
                ufmf_writer = self.ufmf_writer.get(cam_id,None)
                tracking_enabled = self.tracking_enabled.get(cam_id,False)

                if (ufmf_writer is not None) or tracking_enabled:
                    try:
                        realtime_analyzer.max_num_points = bunch.max_num_points.get_nowait()
                    except AttributeError, err:
                        warnings.warn('old realtime_analyzer does not support dynamic '
                                      'setting of max_num_points')
                    points = realtime_analyzer.do_work(fibuf,
                                                       timestamp, framenumber, use_roi2,
                                                       use_cmp=use_cmp)
                    pts = []
                    w = h = realtime_analyzer.roi2_radius*2
                    for pt in points:
                        pts.append( (pt[0], pt[1], w, h ) )
                    if ufmf_writer is not None:
                        saved_points = ufmf_writer.add_frame( fibuf, timestamp, pts )
                    else:
                        saved_points = self.dummy_ufmf_writer[cam_id].add_frame( fibuf, timestamp, pts )

                    lineseg_lists = [ corners2linesegs( *corners ) for corners in saved_points]
                    for linesegs in lineseg_lists:
                        draw_linesegs.extend( linesegs )

                    # save any pending background model updates
                    with self.bg_update_lock:
                        if bunch.last_running_mean_im is not None:

                            if ufmf_writer is not None:
                                ufmf_writer.add_keyframe('mean',
                                                         bunch.last_running_mean_im,
                                                         bunch.last_bgcmp_image_timestamp)
                                ufmf_writer.add_keyframe('sumsq',
                                                         bunch.last_running_sumsqf_image,
                                                         bunch.last_bgcmp_image_timestamp)
                            # delete it
                            bunch.last_running_mean_im = None
Exemplo n.º 20
0
 def test_version(self):
     major,minor,build = fi.get_IPP_version()
Exemplo n.º 21
0
def doit(
    h5_filename=None,
    output_h5_filename=None,
    ufmf_filenames=None,
    kalman_filename=None,
    start=None,
    stop=None,
    view=None,
    erode=0,
    save_images=False,
    save_image_dir=None,
    intermediate_thresh_frac=None,
    final_thresh=None,
    stack_N_images=None,
    stack_N_images_min=None,
    old_sync_timestamp_source=False,
    do_rts_smoothing=True,
):
    """

    Copy all data in .h5 file (specified by h5_filename) to a new .h5
    file in which orientations are set based on image analysis of
    .ufmf files. Tracking data to associate 2D points from subsequent
    frames is read from the .h5 kalman file specified by
    kalman_filename.

    """
    if view is None:
        view = ["orig" for f in ufmf_filenames]
    else:
        assert len(view) == len(ufmf_filenames)

    if intermediate_thresh_frac is None or final_thresh is None:
        raise ValueError("intermediate_thresh_frac and final_thresh must be "
                         "set")

    filename2view = dict(zip(ufmf_filenames, view))

    ca = core_analysis.get_global_CachingAnalyzer()
    obj_ids, use_obj_ids, is_mat_file, data_file, extra = ca.initial_file_load(
        kalman_filename)
    try:
        ML_estimates_2d_idxs = data_file.root.ML_estimates_2d_idxs[:]
    except tables.exceptions.NoSuchNodeError as err1:
        # backwards compatibility
        try:
            ML_estimates_2d_idxs = data_file.root.kalman_observations_2d_idxs[:]
        except tables.exceptions.NoSuchNodeError as err2:
            raise err1

    if os.path.exists(output_h5_filename):
        raise RuntimeError("will not overwrite old file '%s'" %
                           output_h5_filename)
    with open_file_safe(output_h5_filename, delete_on_error=True,
                        mode="w") as output_h5:
        if save_image_dir is not None:
            if not os.path.exists(save_image_dir):
                os.mkdir(save_image_dir)

        with open_file_safe(h5_filename, mode="r") as h5:

            fps = result_utils.get_fps(h5, fail_on_error=True)

            for input_node in h5.root._f_iter_nodes():
                # copy everything from source to dest
                input_node._f_copy(output_h5.root, recursive=True)
            print("done copying")

            # Clear values in destination table that we may overwrite.
            dest_table = output_h5.root.data2d_distorted
            for colname in [
                    "x",
                    "y",
                    "area",
                    "slope",
                    "eccentricity",
                    "cur_val",
                    "mean_val",
                    "sumsqf_val",
            ]:
                if colname == "cur_val":
                    fill_value = 0
                else:
                    fill_value = np.nan
                clear_col(dest_table, colname, fill_value=fill_value)
            dest_table.flush()
            print("done clearing")

            camn2cam_id, cam_id2camns = result_utils.get_caminfo_dicts(h5)

            cam_id2fmfs = collections.defaultdict(list)
            cam_id2view = {}
            for ufmf_filename in ufmf_filenames:
                fmf = ufmf.FlyMovieEmulator(
                    ufmf_filename,
                    # darken=-50,
                    allow_no_such_frame_errors=True,
                )
                timestamps = fmf.get_all_timestamps()

                cam_id = get_cam_id_from_filename(fmf.filename,
                                                  cam_id2camns.keys())
                cam_id2fmfs[cam_id].append(
                    (fmf, result_utils.Quick1DIndexer(timestamps)))

                cam_id2view[cam_id] = filename2view[fmf.filename]

            # associate framenumbers with timestamps using 2d .h5 file
            data2d = h5.root.data2d_distorted[:]  # load to RAM
            data2d_idxs = np.arange(len(data2d))
            h5_framenumbers = data2d["frame"]
            h5_frame_qfi = result_utils.QuickFrameIndexer(h5_framenumbers)

            fpc = realtime_image_analysis.FitParamsClass(
            )  # allocate FitParamsClass

            for obj_id_enum, obj_id in enumerate(use_obj_ids):
                print("object %d of %d" % (obj_id_enum, len(use_obj_ids)))

                # get all images for this camera and this obj_id

                obj_3d_rows = ca.load_dynamics_free_MLE_position(
                    obj_id, data_file)

                this_obj_framenumbers = collections.defaultdict(list)
                if save_images:
                    this_obj_raw_images = collections.defaultdict(list)
                    this_obj_mean_images = collections.defaultdict(list)
                this_obj_absdiff_images = collections.defaultdict(list)
                this_obj_morphed_images = collections.defaultdict(list)
                this_obj_morph_failures = collections.defaultdict(list)
                this_obj_im_coords = collections.defaultdict(list)
                this_obj_com_coords = collections.defaultdict(list)
                this_obj_camn_pt_no = collections.defaultdict(list)

                for this_3d_row in obj_3d_rows:
                    # iterate over each sample in the current camera
                    framenumber = this_3d_row["frame"]
                    if start is not None:
                        if not framenumber >= start:
                            continue
                    if stop is not None:
                        if not framenumber <= stop:
                            continue
                    h5_2d_row_idxs = h5_frame_qfi.get_frame_idxs(framenumber)

                    frame2d = data2d[h5_2d_row_idxs]
                    frame2d_idxs = data2d_idxs[h5_2d_row_idxs]

                    obs_2d_idx = this_3d_row["obs_2d_idx"]
                    kobs_2d_data = ML_estimates_2d_idxs[int(obs_2d_idx)]

                    # Parse VLArray.
                    this_camns = kobs_2d_data[0::2]
                    this_camn_idxs = kobs_2d_data[1::2]

                    # Now, for each camera viewing this object at this
                    # frame, extract images.
                    for camn, camn_pt_no in zip(this_camns, this_camn_idxs):

                        # find 2D point corresponding to object
                        cam_id = camn2cam_id[camn]

                        movie_tups_for_this_camn = cam_id2fmfs[cam_id]
                        cond = (frame2d["camn"] == camn) & (
                            frame2d["frame_pt_idx"] == camn_pt_no)
                        idxs = np.nonzero(cond)[0]
                        assert len(idxs) == 1
                        idx = idxs[0]

                        orig_data2d_rownum = frame2d_idxs[idx]

                        if not old_sync_timestamp_source:
                            # Change the next line to 'timestamp' for old
                            # data (before May/June 2009 -- the switch to
                            # fview_ext_trig)
                            frame_timestamp = frame2d[idx][
                                "cam_received_timestamp"]
                        else:
                            # previous version
                            frame_timestamp = frame2d[idx]["timestamp"]
                        found = None
                        for fmf, fmf_timestamp_qi in movie_tups_for_this_camn:
                            fmf_fnos = fmf_timestamp_qi.get_idxs(
                                frame_timestamp)
                            if not len(fmf_fnos):
                                continue
                            assert len(fmf_fnos) == 1

                            # should only be one .ufmf with this frame and cam_id
                            assert found is None

                            fmf_fno = fmf_fnos[0]
                            found = (fmf, fmf_fno)
                        if found is None:
                            print(
                                "no image data for frame timestamp %s cam_id %s"
                                % (repr(frame_timestamp), cam_id))
                            continue
                        fmf, fmf_fno = found
                        image, fmf_timestamp = fmf.get_frame(fmf_fno)
                        mean_image = fmf.get_mean_for_timestamp(fmf_timestamp)
                        coding = fmf.get_format()
                        if imops.is_coding_color(coding):
                            image = imops.to_rgb8(coding, image)
                            mean_image = imops.to_rgb8(coding, mean_image)
                        else:
                            image = imops.to_mono8(coding, image)
                            mean_image = imops.to_mono8(coding, mean_image)

                        xy = (
                            int(round(frame2d[idx]["x"])),
                            int(round(frame2d[idx]["y"])),
                        )
                        maxsize = (fmf.get_width(), fmf.get_height())

                        # Accumulate cropped images. Note that the region
                        # of the full image that the cropped image
                        # occupies changes over time as the tracked object
                        # moves. Thus, averaging these cropped-and-shifted
                        # images is not the same as simply averaging the
                        # full frame.

                        roiradius = 25
                        warnings.warn(
                            "roiradius hard-coded to %d: could be set "
                            "from 3D tracking" % roiradius)
                        tmp = clip_and_math(image, mean_image, xy, roiradius,
                                            maxsize)
                        im_coords, raw_im, mean_im, absdiff_im = tmp

                        max_absdiff_im = absdiff_im.max()
                        intermediate_thresh = intermediate_thresh_frac * max_absdiff_im
                        absdiff_im[absdiff_im <= intermediate_thresh] = 0

                        if erode > 0:
                            morphed_im = scipy.ndimage.grey_erosion(absdiff_im,
                                                                    size=erode)
                            ## morphed_im = scipy.ndimage.binary_erosion(absdiff_im>1).astype(np.float32)*255.0
                        else:
                            morphed_im = absdiff_im

                        y0_roi, x0_roi = scipy.ndimage.center_of_mass(
                            morphed_im)
                        x0 = im_coords[0] + x0_roi
                        y0 = im_coords[1] + y0_roi

                        if 1:
                            morphed_im_binary = morphed_im > 0
                            labels, n_labels = scipy.ndimage.label(
                                morphed_im_binary)
                            morph_fail_because_multiple_blobs = False

                            if n_labels > 1:
                                x0, y0 = np.nan, np.nan
                                # More than one blob -- don't allow image.
                                if 1:
                                    # for min flattening
                                    morphed_im = np.empty(morphed_im.shape,
                                                          dtype=np.uint8)
                                    morphed_im.fill(255)
                                    morph_fail_because_multiple_blobs = True
                                else:
                                    # for mean flattening
                                    morphed_im = np.zeros_like(morphed_im)
                                    morph_fail_because_multiple_blobs = True

                        this_obj_framenumbers[camn].append(framenumber)
                        if save_images:
                            this_obj_raw_images[camn].append(
                                (raw_im, im_coords))
                            this_obj_mean_images[camn].append(mean_im)
                        this_obj_absdiff_images[camn].append(absdiff_im)
                        this_obj_morphed_images[camn].append(morphed_im)
                        this_obj_morph_failures[camn].append(
                            morph_fail_because_multiple_blobs)
                        this_obj_im_coords[camn].append(im_coords)
                        this_obj_com_coords[camn].append((x0, y0))
                        this_obj_camn_pt_no[camn].append(orig_data2d_rownum)
                        if 0:
                            fname = "obj%05d_%s_frame%07d_pt%02d.png" % (
                                obj_id,
                                cam_id,
                                framenumber,
                                camn_pt_no,
                            )
                            plot_image_subregion(
                                raw_im,
                                mean_im,
                                absdiff_im,
                                roiradius,
                                fname,
                                im_coords,
                                view=filename2view[fmf.filename],
                            )

                # Now, all the frames from all cameras for this obj_id
                # have been gathered. Do a camera-by-camera analysis.
                for camn in this_obj_absdiff_images:
                    cam_id = camn2cam_id[camn]
                    image_framenumbers = np.array(this_obj_framenumbers[camn])
                    if save_images:
                        raw_images = this_obj_raw_images[camn]
                        mean_images = this_obj_mean_images[camn]
                    absdiff_images = this_obj_absdiff_images[camn]
                    morphed_images = this_obj_morphed_images[camn]
                    morph_failures = np.array(this_obj_morph_failures[camn])
                    im_coords = this_obj_im_coords[camn]
                    com_coords = this_obj_com_coords[camn]
                    camn_pt_no_array = this_obj_camn_pt_no[camn]

                    all_framenumbers = np.arange(image_framenumbers[0],
                                                 image_framenumbers[-1] + 1)

                    com_coords = np.array(com_coords)
                    if do_rts_smoothing:
                        # Perform RTS smoothing on center-of-mass coordinates.

                        # Find first good datum.
                        fgnz = np.nonzero(~np.isnan(com_coords[:, 0]))
                        com_coords_smooth = np.empty(com_coords.shape,
                                                     dtype=np.float)
                        com_coords_smooth.fill(np.nan)

                        if len(fgnz[0]):
                            first_good = fgnz[0][0]

                            RTS_com_coords = com_coords[first_good:, :]

                            # Setup parameters for Kalman filter.
                            dt = 1.0 / fps
                            A = np.array(
                                [
                                    [1, 0, dt, 0],  # process update
                                    [0, 1, 0, dt],
                                    [0, 0, 1, 0],
                                    [0, 0, 0, 1],
                                ],
                                dtype=np.float,
                            )
                            C = np.array(
                                [[1, 0, 0, 0], [0, 1, 0, 0]
                                 ],  # observation matrix
                                dtype=np.float,
                            )
                            Q = 0.1 * np.eye(4)  # process noise
                            R = 1.0 * np.eye(2)  # observation noise
                            initx = np.array(
                                [
                                    RTS_com_coords[0, 0], RTS_com_coords[0, 1],
                                    0, 0
                                ],
                                dtype=np.float,
                            )
                            initV = 2 * np.eye(4)
                            initV[0, 0] = 0.1
                            initV[1, 1] = 0.1
                            y = RTS_com_coords
                            xsmooth, Vsmooth = adskalman.adskalman.kalman_smoother(
                                y, A, C, Q, R, initx, initV)
                            com_coords_smooth[first_good:] = xsmooth[:, :2]

                        # Now shift images

                        image_shift = com_coords_smooth - com_coords
                        bad_cond = np.isnan(image_shift[:, 0])
                        # broadcast zeros to places where no good tracking
                        image_shift[bad_cond, 0] = 0
                        image_shift[bad_cond, 1] = 0

                        shifted_morphed_images = [
                            shift_image(im, xy)
                            for im, xy in zip(morphed_images, image_shift)
                        ]

                        results = flatten_image_stack(
                            image_framenumbers,
                            shifted_morphed_images,
                            im_coords,
                            camn_pt_no_array,
                            N=stack_N_images,
                        )
                    else:
                        results = flatten_image_stack(
                            image_framenumbers,
                            morphed_images,
                            im_coords,
                            camn_pt_no_array,
                            N=stack_N_images,
                        )

                    # The variable fno (the first element of the results
                    # tuple) is guaranteed to be contiguous and to span
                    # the range from the first to last frames available.

                    for (
                            fno,
                            av_im,
                            lowerleft,
                            orig_data2d_rownum,
                            orig_idx,
                            orig_idxs_in_average,
                    ) in results:

                        # Clip image to reduce moment arms.
                        av_im[av_im <= final_thresh] = 0

                        fail_fit = False
                        fast_av_im = FastImage.asfastimage(
                            av_im.astype(np.uint8))
                        try:
                            (x0_roi, y0_roi, area, slope,
                             eccentricity) = fpc.fit(fast_av_im)
                        except realtime_image_analysis.FitParamsError as err:
                            fail_fit = True

                        this_morph_failures = morph_failures[
                            orig_idxs_in_average]
                        n_failed_images = np.sum(this_morph_failures)
                        n_good_images = stack_N_images - n_failed_images
                        if n_good_images >= stack_N_images_min:
                            n_images_is_acceptable = True
                        else:
                            n_images_is_acceptable = False

                        if fail_fit:
                            x0_roi = np.nan
                            y0_roi = np.nan
                            area, slope, eccentricity = np.nan, np.nan, np.nan

                        if not n_images_is_acceptable:
                            x0_roi = np.nan
                            y0_roi = np.nan
                            area, slope, eccentricity = np.nan, np.nan, np.nan

                        x0 = x0_roi + lowerleft[0]
                        y0 = y0_roi + lowerleft[1]

                        if 1:
                            for row in dest_table.iterrows(
                                    start=orig_data2d_rownum,
                                    stop=orig_data2d_rownum + 1):

                                row["x"] = x0
                                row["y"] = y0
                                row["area"] = area
                                row["slope"] = slope
                                row["eccentricity"] = eccentricity
                                row.update()  # save data

                        if save_images:
                            # Display debugging images
                            fname = "av_obj%05d_%s_frame%07d.png" % (
                                obj_id,
                                cam_id,
                                fno,
                            )
                            if save_image_dir is not None:
                                fname = os.path.join(save_image_dir, fname)

                            raw_im, raw_coords = raw_images[orig_idx]
                            mean_im = mean_images[orig_idx]
                            absdiff_im = absdiff_images[orig_idx]
                            morphed_im = morphed_images[orig_idx]
                            raw_l, raw_b = raw_coords[:2]

                            imh, imw = raw_im.shape[:2]
                            n_ims = 5

                            if 1:
                                # increase contrast
                                contrast_scale = 2.0
                                av_im_show = np.clip(av_im * contrast_scale, 0,
                                                     255)

                            margin = 10
                            scale = 3

                            # calculate the orientation line
                            yintercept = y0 - slope * x0
                            xplt = np.array([
                                lowerleft[0] - 5,
                                lowerleft[0] + av_im_show.shape[1] + 5,
                            ])
                            yplt = slope * xplt + yintercept
                            if 1:
                                # only send non-nan values to plot
                                plt_good = ~np.isnan(xplt) & ~np.isnan(yplt)
                                xplt = xplt[plt_good]
                                yplt = yplt[plt_good]

                            top_row_width = scale * imw * n_ims + (
                                1 + n_ims) * margin
                            SHOW_STACK = True
                            if SHOW_STACK:
                                n_stack_rows = 4
                                rw = scale * imw * stack_N_images + (
                                    1 + n_ims) * margin
                                row_width = max(top_row_width, rw)
                                col_height = (n_stack_rows * scale * imh +
                                              (n_stack_rows + 1) * margin)
                                stack_margin = 20
                            else:
                                row_width = top_row_width
                                col_height = scale * imh + 2 * margin
                                stack_margin = 0

                            canv = benu.Canvas(
                                fname,
                                row_width,
                                col_height + stack_margin,
                                color_rgba=(1, 1, 1, 1),
                            )

                            if SHOW_STACK:
                                for (stacki, s_orig_idx
                                     ) in enumerate(orig_idxs_in_average):

                                    (s_raw_im,
                                     s_raw_coords) = raw_images[s_orig_idx]
                                    s_raw_l, s_raw_b = s_raw_coords[:2]
                                    s_imh, s_imw = s_raw_im.shape[:2]
                                    user_rect = (s_raw_l, s_raw_b, s_imw,
                                                 s_imh)

                                    x_display = (stacki + 1) * margin + (
                                        scale * imw) * stacki
                                    for show in ["raw", "absdiff", "morphed"]:
                                        if show == "raw":
                                            y_display = scale * imh + 2 * margin
                                        elif show == "absdiff":
                                            y_display = 2 * scale * imh + 3 * margin
                                        elif show == "morphed":
                                            y_display = 3 * scale * imh + 4 * margin
                                        display_rect = (
                                            x_display,
                                            y_display + stack_margin,
                                            scale * raw_im.shape[1],
                                            scale * raw_im.shape[0],
                                        )

                                        with canv.set_user_coords(
                                                display_rect,
                                                user_rect,
                                                transform=cam_id2view[cam_id],
                                        ):

                                            if show == "raw":
                                                s_im = s_raw_im.astype(
                                                    np.uint8)
                                            elif show == "absdiff":
                                                tmp = absdiff_images[
                                                    s_orig_idx]
                                                s_im = tmp.astype(np.uint8)
                                            elif show == "morphed":
                                                tmp = morphed_images[
                                                    s_orig_idx]
                                                s_im = tmp.astype(np.uint8)

                                            canv.imshow(s_im, s_raw_l, s_raw_b)
                                            sx0, sy0 = com_coords[s_orig_idx]
                                            X = [sx0]
                                            Y = [sy0]
                                            # the raw coords in red
                                            canv.scatter(X,
                                                         Y,
                                                         color_rgba=(1, 0.5,
                                                                     0.5, 1))

                                            if do_rts_smoothing:
                                                sx0, sy0 = com_coords_smooth[
                                                    s_orig_idx]
                                                X = [sx0]
                                                Y = [sy0]
                                                # the RTS smoothed coords in green
                                                canv.scatter(
                                                    X,
                                                    Y,
                                                    color_rgba=(0.5, 1, 0.5,
                                                                1))

                                            if s_orig_idx == orig_idx:
                                                boxx = np.array([
                                                    s_raw_l,
                                                    s_raw_l,
                                                    s_raw_l + s_imw,
                                                    s_raw_l + s_imw,
                                                    s_raw_l,
                                                ])
                                                boxy = np.array([
                                                    s_raw_b,
                                                    s_raw_b + s_imh,
                                                    s_raw_b + s_imh,
                                                    s_raw_b,
                                                    s_raw_b,
                                                ])
                                                canv.plot(
                                                    boxx,
                                                    boxy,
                                                    color_rgba=(0.5, 1, 0.5,
                                                                1),
                                                )
                                        if show == "morphed":
                                            canv.text(
                                                "morphed %d" %
                                                (s_orig_idx - orig_idx, ),
                                                display_rect[0],
                                                (display_rect[1] +
                                                 display_rect[3] +
                                                 stack_margin - 20),
                                                font_size=font_size,
                                                color_rgba=(1, 0, 0, 1),
                                            )

                            # Display raw_im
                            display_rect = (
                                margin,
                                margin,
                                scale * raw_im.shape[1],
                                scale * raw_im.shape[0],
                            )
                            user_rect = (raw_l, raw_b, imw, imh)
                            with canv.set_user_coords(
                                    display_rect,
                                    user_rect,
                                    transform=cam_id2view[cam_id],
                            ):
                                canv.imshow(raw_im.astype(np.uint8), raw_l,
                                            raw_b)
                                canv.plot(
                                    xplt, yplt,
                                    color_rgba=(0, 1, 0,
                                                0.5))  # the orientation line
                            canv.text(
                                "raw",
                                display_rect[0],
                                display_rect[1] + display_rect[3],
                                font_size=font_size,
                                color_rgba=(0.5, 0.5, 0.9, 1),
                                shadow_offset=1,
                            )

                            # Display mean_im
                            display_rect = (
                                2 * margin + (scale * imw),
                                margin,
                                scale * mean_im.shape[1],
                                scale * mean_im.shape[0],
                            )
                            user_rect = (raw_l, raw_b, imw, imh)
                            with canv.set_user_coords(
                                    display_rect,
                                    user_rect,
                                    transform=cam_id2view[cam_id],
                            ):
                                canv.imshow(mean_im.astype(np.uint8), raw_l,
                                            raw_b)
                            canv.text(
                                "mean",
                                display_rect[0],
                                display_rect[1] + display_rect[3],
                                font_size=font_size,
                                color_rgba=(0.5, 0.5, 0.9, 1),
                                shadow_offset=1,
                            )

                            # Display absdiff_im
                            display_rect = (
                                3 * margin + (scale * imw) * 2,
                                margin,
                                scale * absdiff_im.shape[1],
                                scale * absdiff_im.shape[0],
                            )
                            user_rect = (raw_l, raw_b, imw, imh)
                            absdiff_clip = np.clip(absdiff_im * contrast_scale,
                                                   0, 255)
                            with canv.set_user_coords(
                                    display_rect,
                                    user_rect,
                                    transform=cam_id2view[cam_id],
                            ):
                                canv.imshow(absdiff_clip.astype(np.uint8),
                                            raw_l, raw_b)
                            canv.text(
                                "absdiff",
                                display_rect[0],
                                display_rect[1] + display_rect[3],
                                font_size=font_size,
                                color_rgba=(0.5, 0.5, 0.9, 1),
                                shadow_offset=1,
                            )

                            # Display morphed_im
                            display_rect = (
                                4 * margin + (scale * imw) * 3,
                                margin,
                                scale * morphed_im.shape[1],
                                scale * morphed_im.shape[0],
                            )
                            user_rect = (raw_l, raw_b, imw, imh)
                            morphed_clip = np.clip(morphed_im * contrast_scale,
                                                   0, 255)
                            with canv.set_user_coords(
                                    display_rect,
                                    user_rect,
                                    transform=cam_id2view[cam_id],
                            ):
                                canv.imshow(morphed_clip.astype(np.uint8),
                                            raw_l, raw_b)
                            if 0:
                                canv.text(
                                    "morphed",
                                    display_rect[0],
                                    display_rect[1] + display_rect[3],
                                    font_size=font_size,
                                    color_rgba=(0.5, 0.5, 0.9, 1),
                                    shadow_offset=1,
                                )

                            # Display time-averaged absdiff_im
                            display_rect = (
                                5 * margin + (scale * imw) * 4,
                                margin,
                                scale * av_im_show.shape[1],
                                scale * av_im_show.shape[0],
                            )
                            user_rect = (
                                lowerleft[0],
                                lowerleft[1],
                                av_im_show.shape[1],
                                av_im_show.shape[0],
                            )
                            with canv.set_user_coords(
                                    display_rect,
                                    user_rect,
                                    transform=cam_id2view[cam_id],
                            ):
                                canv.imshow(
                                    av_im_show.astype(np.uint8),
                                    lowerleft[0],
                                    lowerleft[1],
                                )
                                canv.plot(
                                    xplt, yplt,
                                    color_rgba=(0, 1, 0,
                                                0.5))  # the orientation line
                            canv.text(
                                "stacked/flattened",
                                display_rect[0],
                                display_rect[1] + display_rect[3],
                                font_size=font_size,
                                color_rgba=(0.5, 0.5, 0.9, 1),
                                shadow_offset=1,
                            )

                            canv.text(
                                "%s frame % 7d: eccentricity % 5.1f, min N images %d, actual N images %d"
                                % (
                                    cam_id,
                                    fno,
                                    eccentricity,
                                    stack_N_images_min,
                                    n_good_images,
                                ),
                                0,
                                15,
                                font_size=font_size,
                                color_rgba=(0.6, 0.7, 0.9, 1),
                                shadow_offset=1,
                            )
                            canv.save()

                # Save results to new table
                if 0:
                    recarray = np.rec.array(list_of_rows_of_data2d,
                                            dtype=Info2DCol_description)
                    dest_table.append(recarray)
                    dest_table.flush()
            dest_table.attrs.has_ibo_data = True
        data_file.close()