Пример #1
0
def _get_models(ibs, species, modeldir="default", cfg_override=True, verbose=VERBOSE_RF):
    r"""
    Args:
        ibs (IBEISController):  ibeis controller object
        species (?):
        modeldir (str): (default = 'default')
        cfg_override (bool): (default = True)
        verbose (bool):  verbosity flag(default = False)

    Returns:
        ?: fpath_list

    CommandLine:
        python -m ibeis.algo.detect.randomforest --test-_get_models

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.algo.detect.randomforest import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(defaultdb='testdb1')
        >>> species = ibeis.const.TEST_SPECIES.ZEB_PLAIN
        >>> modeldir = 'default'
        >>> cfg_override = True
        >>> verbose = False
        >>> fpath_list = _get_models(ibs, species, modeldir, cfg_override, verbose)
        >>> result = ('fpath_list = %s' % (str(fpath_list),))
        >>> print(result)
    """
    # with ut.embed_on_exception_context:
    if cfg_override and len(ibs.cfg.detect_cfg.trees_path) > 0:
        trees_path = ibs.cfg.detect_cfg.trees_path
    else:
        # Ensure all models downloaded and accounted for
        assert species is not None, "[_get_models] Cannot detect without specifying a species"
        grabmodels.ensure_models(modeldir=modeldir, verbose=verbose)
        trees_path = grabmodels.get_species_trees_paths(species, modeldir=modeldir)
    # Load tree paths
    if ut.checkpath(trees_path, verbose=verbose):
        fpath_list = ut.ls(trees_path, "*.txt")
        # direct = Directory(trees_path, include_extensions=['txt'])
        # files = direct.files()
    else:
        # If the models do not exist, return None
        fpath_list = None
    if fpath_list is None or len(fpath_list) == 0:
        msg = (
            ut.codeblock(
                """
            [_get_models] Error loading trees, either directory or fpath_list not found
              * trees_path = %r
              * fpath_list = %r
              * species = %r
              * model_dir = %r
              * cfg_override = %r
            """
            )
            % (trees_path, fpath_list, species, modeldir, cfg_override)
        )
        raise AssertionError(msg)
    return fpath_list
Пример #2
0
def _get_models(ibs,
                species,
                modeldir='default',
                cfg_override=True,
                verbose=VERBOSE_RF):
    r"""
    Args:
        ibs (IBEISController):  ibeis controller object
        species (?):
        modeldir (str): (default = 'default')
        cfg_override (bool): (default = True)
        verbose (bool):  verbosity flag(default = False)

    Returns:
        ?: fpath_list

    CommandLine:
        python -m ibeis.algo.detect.randomforest --test-_get_models

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.algo.detect.randomforest import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(defaultdb='testdb1')
        >>> species = ibeis.const.TEST_SPECIES.ZEB_PLAIN
        >>> modeldir = 'default'
        >>> cfg_override = True
        >>> verbose = False
        >>> fpath_list = _get_models(ibs, species, modeldir, cfg_override, verbose)
        >>> result = ('fpath_list = %s' % (str(fpath_list),))
        >>> print(result)
    """
    #with ut.embed_on_exception_context:
    if cfg_override and len(ibs.cfg.detect_cfg.trees_path) > 0:
        trees_path = ibs.cfg.detect_cfg.trees_path
    else:
        # Ensure all models downloaded and accounted for
        assert species is not None, '[_get_models] Cannot detect without specifying a species'
        grabmodels.ensure_models(modeldir=modeldir, verbose=verbose)
        trees_path = grabmodels.get_species_trees_paths(species,
                                                        modeldir=modeldir)
    # Load tree paths
    if ut.checkpath(trees_path, verbose=verbose):
        fpath_list = ut.ls(trees_path, '*.txt')
        #direct = Directory(trees_path, include_extensions=['txt'])
        #files = direct.files()
    else:
        # If the models do not exist, return None
        fpath_list = None
    if fpath_list is None or len(fpath_list) == 0:
        msg = ut.codeblock('''
            [_get_models] Error loading trees, either directory or fpath_list not found
              * trees_path = %r
              * fpath_list = %r
              * species = %r
              * model_dir = %r
              * cfg_override = %r
            ''') % (trees_path, fpath_list, species, modeldir, cfg_override)
        raise AssertionError(msg)
    return fpath_list
Пример #3
0
def init_console2():
    assert ut.WIN32, 'win32 only script'
    url = 'http://downloads.sourceforge.net/project/console/console-devel/2.00/Console-2.00b148-Beta_32bit.zip'
    unzipped_fpath = ut.grab_zipped_url(url)
    # FIXME: bugged
    unzipped_fpath2 = join(dirname(unzipped_fpath), 'Console2')
    win32_bin = ut.truepath('~/local/PATH')
    ut.copy(ut.ls(unzipped_fpath2), win32_bin)
Пример #4
0
def list_published_distinctivness():
    r"""
    CommandLine:
        python -m ibeis.algo.hots.distinctiveness_normalizer --test-list_published_distinctivness

    Example:
        >>> # SLOW_DOCTEST
        >>> from ibeis.algo.hots.distinctiveness_normalizer import *  # NOQA
        >>> published_fpaths = list_published_distinctivness()
        >>> print(ut.list_str(published_fpaths))
    """
    published_fpaths = ut.ls(PUBLISH_DIR)
    return published_fpaths
def list_published_distinctivness():
    r"""
    CommandLine:
        python -m wbia.algo.hots.distinctiveness_normalizer --test-list_published_distinctivness

    Example:
        >>> # SLOW_DOCTEST
        >>> from wbia.algo.hots.distinctiveness_normalizer import *  # NOQA
        >>> published_fpaths = list_published_distinctivness()
        >>> print(ut.repr2(published_fpaths))
    """
    published_fpaths = ut.ls(PUBLISH_DIR)
    return published_fpaths
Пример #6
0
 def load_splitsets(dataset):
     import parse
     fpath_dict = {}
     fmtstr = dataset.get_split_fmtstr(forward=False)
     for fpath in ut.ls(dataset.split_dpath):
         parsed = parse.parse(fmtstr, basename(fpath))
         if parsed is None:
             print('WARNING: invalid filename %r' % (fpath,))
             continue
         key = parsed['key']
         type_ = parsed['type_']
         splitset = fpath_dict.get(key, {})
         splitset[type_] = fpath
         fpath_dict[key] = splitset
     # check validity of loaded data
     for key, val in fpath_dict.items():
         assert 'data' in val, 'subset missing data'
     dataset.fpath_dict.update(**fpath_dict)
Пример #7
0
def list_distinctivness_cache():
    global_distinctdir = sysres.get_global_distinctiveness_modeldir()
    print(ut.list_str(ut.ls(global_distinctdir)))
Пример #8
0
def test_pyrf():
    r"""
    CommandLine:
        python run_tests.py --test-test_pyrf

    Example:
        >>> # ENABLE_DOCTEST
        >>> from run_tests import *  # NOQA
        >>> result = test_pyrf()
        >>> print(result)
    """

    #=================================
    # Initialization
    #=================================

    category = 'zebra_plains'

    #detect_config = {
    #    'save_detection_images':        True,
    #    'percentage_top':               0.40,
    #}

    testdata_dir = ut.unixpath('~/code/pyrf/results')
    # assert ut.checkpath(testdata_dir)
    if ut.get_argflag('--vd'):
        print(ut.ls(testdata_dir))

    # Create detector
    detector = Random_Forest_Detector()

    test_path = ut.grab_zipped_url(TEST_DATA_DETECT_URL, appname='utool')
    models_path = ut.grab_zipped_url(TEST_DATA_MODEL_URL, appname='utool')
    trees_path = join(models_path, category)
    detect_path = join(test_path, category, 'detect')
    ut.ensuredir(detect_path)
    ut.ensuredir(test_path)
    ut.ensuredir(trees_path)

    #=================================
    # Load Input Images
    #=================================

    # Get input images
    big_gpath_list = ut.list_images(test_path, fullpath=True, recursive=False)
    print(big_gpath_list)
    # Resize images to standard size
    if ut.get_argflag('--small'):
        big_gpath_list = big_gpath_list[0:8]
    #big_gpath_list = big_gpath_list[0:8]
    output_dir = join(test_path, 'resized')
    std_gpath_list = resize_imagelist_to_sqrtarea(big_gpath_list,
                                                  sqrt_area=800,
                                                  output_dir=output_dir,
                                                  checkexists=True)
    dst_gpath_list = [join(detect_path, split(gpath)[1]) for gpath in std_gpath_list]
    #ut.view_directory(test_path)
    #ut.view_directory('.')
    print(std_gpath_list)
    num_images = len(std_gpath_list)
    #assert num_images == 16, 'the test has diverged!'
    print('Testing on %r images' % num_images)

    #=================================
    # Load Pretrained Forests
    #=================================

    # Load forest, so we don't have to reload every time
    trees_fpath_list = ut.ls(trees_path, '*.txt')
    #forest = detector.load(trees_path, category + '-')
    forest = detector.forest(trees_fpath_list)
    #detector.set_detect_params(**detect_config)
    results_list1 = []

    #=================================
    # Detect using Random Forest
    #=================================

    with ut.Timer('[test_pyrf] for loop detector.detect') as t1:
        if not ut.get_argflag('--skip1'):
            results_list1 = detector.detect(forest, std_gpath_list, output_gpath_list=dst_gpath_list)
            #for ix, (img_fpath, dst_fpath) in enumerate(zip(std_gpath_list, dst_gpath_list)):
            #    #img_fname = split(img_fpath)[1]
            #    #dst_fpath = join(detect_path, img_fname)
            #    #print('  * img_fpath = %r' % img_fpath)
            #    #print('  * dst_fpath = %r' % dst_fpath)
            #    with ut.Timer('[test_pyrf] detector.detect ix=%r' % (ix,)):
            #        results = detector.detect(forest, img_fpath, dst_fpath)
            #    results_list1.append(results)
            #    print('num results = %r' % len(results))
            #else:
            #    print('...skipped')

    #with ut.Timer('[test_pyrf] detector.detect_many') as t2:
    #    results_list2 = detector.detect_many(forest, std_gpath_list,
    #                                         dst_gpath_list, use_openmp=True)
    detector.free_forest(forest)

    print('')
    print('+ --------------')
    print('| total time1: %r' % t1.ellapsed)
    #print('| total time2: %r' % t2.ellapsed)
    print('|')
    print('| num results1 = %r' % (list(map(len, results_list1))))
    #print('| num results2 = %r' % (list(map(len, results_list2))))
    #assert results_list2 == results_list1
    return locals()
Пример #9
0
def load_oxford_2007():
    """
    Loads data from
    http://www.robots.ox.ac.uk:5000/~vgg/publications/2007/Philbin07/philbin07.pdf

    >>> from wbia.algo.smk.script_smk import *  # NOQA
    """
    from os.path import join, basename, splitext
    import pandas as pd
    import vtool as vt

    dbdir = ut.truepath('/raid/work/Oxford/')
    data_fpath0 = join(dbdir, 'data_2007.pkl')

    if ut.checkpath(data_fpath0):
        data = ut.load_data(data_fpath0)
        return data
    else:
        word_dpath = join(dbdir, 'word_oxc1_hesaff_sift_16M_1M')
        _word_fpath_list = ut.ls(word_dpath)
        imgid_to_word_fpath = {
            splitext(basename(word_fpath))[0]: word_fpath
            for word_fpath in _word_fpath_list
        }
        readme_fpath = join(dbdir, 'README2.txt')
        imgid_order = ut.readfrom(readme_fpath).split('\n')[20:-1]

        imgid_order = imgid_order
        data_uri_order = [x.replace('oxc1_', '') for x in imgid_order]

        imgid_to_df = {}
        for imgid in ut.ProgIter(imgid_order, label='reading kpts'):
            word_fpath = imgid_to_word_fpath[imgid]
            row_gen = (map(float,
                           line.strip('\n').split(' '))
                       for line in ut.read_lines_from(word_fpath)[2:])
            rows = [(int(word_id), x, y, e11, e12, e22)
                    for (word_id, x, y, e11, e12, e22) in row_gen]
            df = pd.DataFrame(
                rows, columns=['word_id', 'x', 'y', 'e11', 'e12', 'e22'])
            imgid_to_df[imgid] = df

        df_list = ut.take(imgid_to_df, imgid_order)

        nfeat_list = [len(df_) for df_ in df_list]
        offset_list = [0] + ut.cumsum(nfeat_list)
        shape = (offset_list[-1], 128)
        # shape = (16334970, 128)
        sift_fpath = join(dbdir, 'OxfordSIFTDescriptors',
                          'feat_oxc1_hesaff_sift.bin')
        try:
            file_ = open(sift_fpath, 'rb')
            with ut.Timer('Reading SIFT binary file'):
                nbytes = np.prod(shape)
                all_vecs = np.fromstring(file_.read(nbytes), dtype=np.uint8)
            all_vecs = all_vecs.reshape(shape)
        finally:
            file_.close()

        kpts_list = [
            df_.loc[:, ('x', 'y', 'e11', 'e12', 'e22')].values
            for df_ in df_list
        ]
        wordid_list = [df_.loc[:, 'word_id'].values for df_ in df_list]
        kpts_Z = np.vstack(kpts_list)
        idx_to_wx = np.hstack(wordid_list)

        # assert len(np.unique(idx_to_wx)) == 1E6

        # Reqd standard query order
        query_files = sorted(
            ut.glob(dbdir + '/oxford_groundtruth', '*_query.txt'))
        query_uri_order = []
        for qpath in query_files:
            text = ut.readfrom(qpath, verbose=0)
            query_uri = text.split(' ')[0].replace('oxc1_', '')
            query_uri_order.append(query_uri)

        logger.info('converting to invV')
        all_kpts = vt.convert_kptsZ_to_kpts(kpts_Z)

        data = {
            'offset_list': offset_list,
            'all_kpts': all_kpts,
            'all_vecs': all_vecs,
            'idx_to_wx': idx_to_wx,
            'data_uri_order': data_uri_order,
            'query_uri_order': query_uri_order,
        }
        ut.save_data(data_fpath0, data)
    return data
Пример #10
0
def build_linux_zip_binaries():
    fpath_list = ut.ls('dist/ibeis')
    archive_fpath = 'dist/ibeis-linux-binary.zip'
    ut.archive_files(archive_fpath, fpath_list)
    return archive_fpath
Пример #11
0
 def ls(self):
     return ut.ls(self.abspath)
Пример #12
0
def train_gid_list(ibs,
                   gid_list,
                   trees_path=None,
                   species=None,
                   setup=True,
                   teardown=False,
                   **kwargs):
    """
    Args:
        gid_list (list of int): the list of IBEIS image_rowids that need detection
        trees_path (str): the path that the trees will be saved into (along
            with temporary training inventory folders that are deleted once
            training is finished)
        species (str): the species that should be used to assign to the newly
            trained trees

    Kwargs (optional): refer to the PyRF documentation for configuration settings

    Returns:
        None
    """
    print("[randomforest.train()] training with %d gids and species=%r" % (
        len(gid_list),
        species,
    ))
    if trees_path is None and species is not None:
        trees_path = join(ibs.get_cachedir(), 'trees', species)

    # Get positive chip paths
    if species is None:
        aids_list = ibs.get_image_aids(gid_list)
    else:
        aids_list = ibs.get_image_aids_of_species(gid_list, species)

    # ##### TEMP #####
    # gid_list_ = []
    # aids_list_ = []
    # for gid, aid_list in zip(gid_list, aids_list):
    #     if len(aid_list) > 1:
    #         gid_list_.append(gid)
    #         aids_list_.append(aid_list)
    #     elif len(aid_list) == 1:
    #         (xtl, ytl, width, height) = ibs.get_annot_bboxes(aid_list)[0]
    #         if xtl > 5 and ytl > 5:
    #             gid_list_.append(gid)
    #             aids_list_.append(aid_list)
    # gid_list = gid_list_
    # aids_list = aids_list_
    # kwargs['trees_max_patches'] = 100000
    # ##### TEMP #####

    aid_list = ut.flatten(aids_list)
    train_pos_cpath_list = ibs.get_annot_chip_fpath(aid_list)

    # Ensure directories for negatives
    negatives_cache = join(ibs.get_cachedir(), 'pyrf_train_negatives')
    if (setup and not exists(negatives_cache)) or setup == 'force':
        # Force Check
        if exists(negatives_cache):
            ut.remove_dirs(negatives_cache)
        ut.ensuredir(negatives_cache)
        # Get negative chip paths
        print("[randomforest.train()] Mining %d negative patches" %
              (len(train_pos_cpath_list), ))
        train_neg_cpath_list = []
        while len(train_neg_cpath_list) < len(train_pos_cpath_list):
            sample = random.randint(0, len(gid_list) - 1)
            gid = gid_list[sample]
            img_width, img_height = ibs.get_image_sizes(gid)
            size = min(img_width, img_height)
            if species is None:
                aid_list = ibs.get_image_aids(gid)
            else:
                aid_list = ibs.get_image_aids_of_species(gid, species)
            annot_bbox_list = ibs.get_annot_bboxes(aid_list)
            # Find square patches
            square = random.randint(int(size / 4), int(size / 2))
            xmin = random.randint(0, img_width - square)
            xmax = xmin + square
            ymin = random.randint(0, img_height - square)
            ymax = ymin + square
            if _valid_candidate((xmin, xmax, ymin, ymax), annot_bbox_list):
                if VERBOSE_RF:
                    print(
                        "[%d / %d] MINING NEGATIVE PATCH (%04d, %04d, %04d, %04d) FROM GID %d"
                        % (
                            len(train_neg_cpath_list),
                            len(train_pos_cpath_list),
                            xmin,
                            xmax,
                            ymin,
                            ymax,
                            gid,
                        ))
                img = ibs.get_images(gid)
                img_path = join(
                    negatives_cache,
                    "neg_%07d.JPEG" % (len(train_neg_cpath_list), ))
                img = img[ymin:ymax, xmin:xmax]
                cv2.imwrite(img_path, img)
                train_neg_cpath_list.append(img_path)
    else:
        train_neg_cpath_list = ut.ls(negatives_cache, '*.JPEG')
        #direct = Directory(negatives_cache, include_extensions=['JPEG'])
        #train_neg_cpath_list = direct.files()

    # Train trees
    train_gpath_list(ibs,
                     train_pos_cpath_list,
                     train_neg_cpath_list,
                     trees_path=trees_path,
                     species=species,
                     **kwargs)

    # Remove cached negatives directory
    if teardown:
        ut.remove_dirs(negatives_cache)
Пример #13
0
 def ls(self):
     return ut.ls(self.abspath)
Пример #14
0
def batch_move(r, search, repl, force=False):
    r"""
    This function has not yet been successfully implemented.
    Its a start though.

    rob batch_move train_patchmetric\(.*\) patchmatch\1 False
    ut.named_field('rest', '.*' )
    ut.backref_field('rest')

    search = 'train_patchmetric(?P<rest>.*)'
    repl = 'patchmatch\\g<rest>'
    """
    force = rutil.cast(force, bool)
    # rob batch_move '\(*\)util.py' 'util_\1.py'
    print('Batch Move')
    print('force = %r' % force)
    print('search = %r' % search)
    print('repl = %r' % repl)
    dpath_list = [os.getcwd()]
    spec_open = ['\\(', '\(']
    spec_close = ['\\)', '\)']
    special_repl_strs = ['\1', '\\1']
    print('special_repl_strs = %r' % special_repl_strs)
    print('special_search_strs = %r' % ((spec_open, spec_close,),))

    search_pat = ut.extend_regex(search)
    #for spec in spec_open + spec_close:
    #    search_pat = search_pat.replace(spec, '')
    print('search_pat=%r' % search_pat)

    include_patterns = [search_pat]

    import utool as ut
    import re
    fpath_list = ut.ls('.')
    matching_fpaths = [fpath for fpath in fpath_list if re.search(search_pat, basename(fpath))]
    repl_fpaths = [re.sub(search_pat, repl, fpath) for fpath in matching_fpaths]

    ut.rrrr()
    for fpath1, fpath2 in zip(matching_fpaths, repl_fpaths):
        ut.util_path.copy(fpath1, fpath2, deeplink=False, dryrun=False)
    #for fpath in rob_nav._matching_fnames(dpath_list, include_patterns, recursive=False):
    #    print(fpath)
    return

    parse_str = search
    for spec in spec_open:
        parse_str = parse_str.replace(spec, '{')
    for spec in spec_close:
        parse_str = parse_str.replace(spec, '}')
    parse_str = parse_str.replace('{*}', '{}')
    print('parse_str = %r' % parse_str)

    for fpath in rob_nav._matching_fnames(dpath_list, include_patterns, recursive=False):
        dpath, fname = split(fpath)
        name, ext = splitext(fname)
        # Hard coded parsing
        parsed = parse.parse(parse_str, fname)
        repl1 = parsed[0]
        #print(fname)
        newfname = 'util_' + repl1 + ext
        newfpath = join(dpath, newfname)
        print('move')
        print(fpath)
        print(newfpath)
        if force is True:
            shutil.move(fpath, newfpath)
            print('real run')
        else:
            print('dry run')
        pass
def list_distinctivness_cache():
    global_distinctdir = sysres.get_global_distinctiveness_modeldir()
    logger.info(ut.repr2(ut.ls(global_distinctdir)))
Пример #16
0
def test_pyrf():
    category = 'zebra_plains'

    detect_config = {
        'save_detection_images':        True,
        'save_scales':                  True,
        'percentage_top':               0.40,
    }

    #=================================
    # Train / Detect Initialization
    #=================================

    testdata_dir = utool.unixpath('~/code/pyrf/results')
    # assert utool.checkpath(testdata_dir)
    if utool.get_argflag('--vd'):
        print(utool.ls(testdata_dir))

    # Create detector
    detector = Random_Forest_Detector()

    test_path = utool.grab_zipped_url(TEST_DATA_DETECT_URL, appname='utool')
    models_path = utool.grab_zipped_url(TEST_DATA_MODEL_URL, appname='utool')
    trees_path = join(models_path, category)

    results_path  = join(utool.unixpath('~/code/pyrf/results'), category)
    # detect_path   = join(results_path, 'detect')
    trees_path    = join(results_path, 'trees')

    detect_path = join(test_path, category, 'detect')
    utool.ensuredir(detect_path)
    utool.ensuredir(test_path)
    utool.ensuredir(trees_path)

    #=================================
    # Detect using Random Forest
    #=================================

    # Get input images
    from vtool import image
    big_gpath_list = utool.list_images(test_path, fullpath=True, recursive=False)
    print(big_gpath_list)
    # Resize images to standard size
    if utool.get_argflag('--small'):
        big_gpath_list = big_gpath_list[0:8]
    #big_gpath_list = big_gpath_list[0:8]
    output_dir = join(test_path, 'resized')
    std_gpath_list = image.resize_imagelist_to_sqrtarea(big_gpath_list,
                                                        sqrt_area=800,
                                                        output_dir=output_dir,
                                                        checkexists=True)
    dst_gpath_list = [join(detect_path, split(gpath)[1]) for gpath in std_gpath_list]
    #utool.view_directory(test_path)
    #utool.view_directory('.')
    print(std_gpath_list)
    num_images = len(std_gpath_list)
    #assert num_images == 16, 'the test has diverged!'
    print('Testing on %r images' % num_images)

    # Load forest, so we don't have to reload every time
    forest = detector.load(trees_path, category + '-', num_trees=25)
    detector.set_detect_params(**detect_config)
    results_list1 = []
    with utool.Timer('[test_pyrf] for loop detector.detect') as t1:
        if not utool.get_argflag('--skip1'):
            for ix, (img_fpath, dst_fpath) in enumerate(zip(std_gpath_list, dst_gpath_list)):
                #img_fname = split(img_fpath)[1]
                #dst_fpath = join(detect_path, img_fname)
                #print('  * img_fpath = %r' % img_fpath)
                #print('  * dst_fpath = %r' % dst_fpath)
                with utool.Timer('[test_pyrf] detector.detect ix=%r' % (ix,)):
                    results = detector.detect(forest, img_fpath, dst_fpath)
                results_list1.append(results)
                print('num results = %r' % len(results))
            else:
                print('...skipped')

    # with utool.Timer('[test_pyrf] detector.detect_many') as t2:
    #     results_list2 = detector.detect_many(forest, std_gpath_list,
    #                                          dst_gpath_list, use_openmp=True)

    print('')
    print('+ --------------')
    print('| total time1: %r' % t1.ellapsed)
    # print('| total time2: %r' % t2.ellapsed)
    print('|')
    print('| num results1 = %r' % (list(map(len, results_list1))))
    # print('| num results2 = %r' % (list(map(len, results_list2))))
    #assert results_list2 == results_list1
    return locals()
Пример #17
0
def build_linux_zip_binaries():
    fpath_list = ut.ls('dist/ibeis')
    archive_fpath = 'dist/ibeis-linux-binary.zip'
    ut.archive_files(archive_fpath, fpath_list)
    return archive_fpath
Пример #18
0
def make_wordfigures(ibs, metrics, invindex, figdir, wx_sample, wx2_dpath):
    """
    Builds mosaics of patches assigned to words in sample
    ouptuts them to disk
    """
    from plottool import draw_func2 as df2
    import vtool as vt
    import parse

    vocabdir = join(figdir, 'vocab_patches2')
    ut.ensuredir(vocabdir)
    dump_word_patches(ibs, vocabdir, invindex, wx_sample, metrics)

    # COLLECTING PART --- collects patches in word folders
    #vocabdir

    seldpath = vocabdir + '_selected'
    ut.ensurepath(seldpath)
    # stack for show
    for wx, dpath in ut.progiter(six.iteritems(wx2_dpath), lbl='Dumping Word Images:', num=len(wx2_dpath), freq=1, backspace=False):
        #df2.rrr()
        fpath_list = ut.ls(dpath)
        fname_list = [basename(fpath_) for fpath_ in fpath_list]
        patch_list = [gtool.imread(fpath_) for fpath_ in fpath_list]
        # color each patch by nid
        nid_list = [int(parse.parse('{}_nid={nid}_{}', fname)['nid']) for fname in fname_list]
        nid_set = set(nid_list)
        nid_list = np.array(nid_list)
        if len(nid_list) == len(nid_set):
            # no duplicate names
            newpatch_list = patch_list
        else:
            # duplicate names. do coloring
            sortx = nid_list.argsort()
            patch_list = np.array(patch_list, dtype=object)[sortx]
            fname_list = np.array(fname_list, dtype=object)[sortx]
            nid_list = nid_list[sortx]
            colors = (255 * np.array(df2.distinct_colors(len(nid_set)))).astype(np.int32)
            color_dict = dict(zip(nid_set, colors))
            wpad, hpad = 3, 3
            newshape_list = [tuple((np.array(patch.shape) + (wpad * 2, hpad * 2, 0)).tolist()) for patch in patch_list]
            color_list = [color_dict[nid_] for nid_ in nid_list]
            newpatch_list = [np.zeros(shape) + color[None, None] for shape, color in zip(newshape_list, color_list)]
            for patch, newpatch in zip(patch_list, newpatch_list):
                newpatch[wpad:-wpad, hpad:-hpad, :] = patch
            #img_list = patch_list
            #bigpatch = vt.stack_image_recurse(patch_list)
        #bigpatch = vt.stack_image_list(patch_list, vert=False)
        bigpatch = vt.stack_square_images(newpatch_list)
        bigpatch_fpath = join(seldpath, basename(dpath) + '_patches.png')

        #
        def _dictstr(dict_):
            str_ = ut.dict_str(dict_, newlines=False)
            str_ = str_.replace('\'', '').replace(': ', '=').strip('{},')
            return str_

        figtitle = '\n'.join([
            'wx=%r' % wx,
            'stat(pdist): %s' % _dictstr(metrics.wx2_pdist_stats[wx]),
            'stat(wdist): %s' % _dictstr(metrics.wx2_wdist_stats[wx]),
        ])
        metrics.wx2_nMembers[wx]

        df2.figure(fnum=1, doclf=True, docla=True)
        fig, ax = df2.imshow(bigpatch, figtitle=figtitle)
        #fig.show()
        df2.set_figtitle(figtitle)
        df2.adjust_subplots(top=.878, bottom=0)
        df2.save_figure(1, bigpatch_fpath)
Пример #19
0
def train_gid_list(ibs, gid_list, trees_path=None, species=None, setup=True, teardown=False, **kwargs):
    """
    Args:
        gid_list (list of int): the list of IBEIS image_rowids that need detection
        trees_path (str): the path that the trees will be saved into (along
            with temporary training inventory folders that are deleted once
            training is finished)
        species (str): the species that should be used to assign to the newly
            trained trees

    Kwargs (optional): refer to the PyRF documentation for configuration settings

    Returns:
        None
    """
    print("[randomforest.train()] training with %d gids and species=%r" % (len(gid_list), species))
    if trees_path is None and species is not None:
        trees_path = join(ibs.get_cachedir(), "trees", species)

    # Get positive chip paths
    if species is None:
        aids_list = ibs.get_image_aids(gid_list)
    else:
        aids_list = ibs.get_image_aids_of_species(gid_list, species)

    # ##### TEMP #####
    # gid_list_ = []
    # aids_list_ = []
    # for gid, aid_list in zip(gid_list, aids_list):
    #     if len(aid_list) > 1:
    #         gid_list_.append(gid)
    #         aids_list_.append(aid_list)
    #     elif len(aid_list) == 1:
    #         (xtl, ytl, width, height) = ibs.get_annot_bboxes(aid_list)[0]
    #         if xtl > 5 and ytl > 5:
    #             gid_list_.append(gid)
    #             aids_list_.append(aid_list)
    # gid_list = gid_list_
    # aids_list = aids_list_
    # kwargs['trees_max_patches'] = 100000
    # ##### TEMP #####

    aid_list = ut.flatten(aids_list)
    train_pos_cpath_list = ibs.get_annot_chip_fpath(aid_list)

    # Ensure directories for negatives
    negatives_cache = join(ibs.get_cachedir(), "pyrf_train_negatives")
    if (setup and not exists(negatives_cache)) or setup == "force":
        # Force Check
        if exists(negatives_cache):
            ut.remove_dirs(negatives_cache)
        ut.ensuredir(negatives_cache)
        # Get negative chip paths
        print("[randomforest.train()] Mining %d negative patches" % (len(train_pos_cpath_list),))
        train_neg_cpath_list = []
        while len(train_neg_cpath_list) < len(train_pos_cpath_list):
            sample = random.randint(0, len(gid_list) - 1)
            gid = gid_list[sample]
            img_width, img_height = ibs.get_image_sizes(gid)
            size = min(img_width, img_height)
            if species is None:
                aid_list = ibs.get_image_aids(gid)
            else:
                aid_list = ibs.get_image_aids_of_species(gid, species)
            annot_bbox_list = ibs.get_annot_bboxes(aid_list)
            # Find square patches
            square = random.randint(int(size / 4), int(size / 2))
            xmin = random.randint(0, img_width - square)
            xmax = xmin + square
            ymin = random.randint(0, img_height - square)
            ymax = ymin + square
            if _valid_candidate((xmin, xmax, ymin, ymax), annot_bbox_list):
                if VERBOSE_RF:
                    print(
                        "[%d / %d] MINING NEGATIVE PATCH (%04d, %04d, %04d, %04d) FROM GID %d"
                        % (len(train_neg_cpath_list), len(train_pos_cpath_list), xmin, xmax, ymin, ymax, gid)
                    )
                img = ibs.get_images(gid)
                img_path = join(negatives_cache, "neg_%07d.JPEG" % (len(train_neg_cpath_list),))
                img = img[ymin:ymax, xmin:xmax]
                cv2.imwrite(img_path, img)
                train_neg_cpath_list.append(img_path)
    else:
        train_neg_cpath_list = ut.ls(negatives_cache, "*.JPEG")
        # direct = Directory(negatives_cache, include_extensions=['JPEG'])
        # train_neg_cpath_list = direct.files()

    # Train trees
    train_gpath_list(ibs, train_pos_cpath_list, train_neg_cpath_list, trees_path=trees_path, species=species, **kwargs)

    # Remove cached negatives directory
    if teardown:
        ut.remove_dirs(negatives_cache)
Пример #20
0
def make_wordfigures(ibs, metrics, invindex, figdir, wx_sample, wx2_dpath):
    """
    Builds mosaics of patches assigned to words in sample
    ouptuts them to disk
    """
    from plottool import draw_func2 as df2
    import vtool as vt
    import parse

    vocabdir = join(figdir, 'vocab_patches2')
    ut.ensuredir(vocabdir)
    dump_word_patches(ibs, vocabdir, invindex, wx_sample, metrics)

    # COLLECTING PART --- collects patches in word folders
    #vocabdir

    seldpath = vocabdir + '_selected'
    ut.ensurepath(seldpath)
    # stack for show
    for wx, dpath in ut.progiter(six.iteritems(wx2_dpath),
                                 lbl='Dumping Word Images:',
                                 num=len(wx2_dpath),
                                 freq=1,
                                 backspace=False):
        #df2.rrr()
        fpath_list = ut.ls(dpath)
        fname_list = [basename(fpath_) for fpath_ in fpath_list]
        patch_list = [vt.imread(fpath_) for fpath_ in fpath_list]
        # color each patch by nid
        nid_list = [
            int(parse.parse('{}_nid={nid}_{}', fname)['nid'])
            for fname in fname_list
        ]
        nid_set = set(nid_list)
        nid_list = np.array(nid_list)
        if len(nid_list) == len(nid_set):
            # no duplicate names
            newpatch_list = patch_list
        else:
            # duplicate names. do coloring
            sortx = nid_list.argsort()
            patch_list = np.array(patch_list, dtype=object)[sortx]
            fname_list = np.array(fname_list, dtype=object)[sortx]
            nid_list = nid_list[sortx]
            colors = (255 *
                      np.array(df2.distinct_colors(len(nid_set)))).astype(
                          np.int32)
            color_dict = dict(zip(nid_set, colors))
            wpad, hpad = 3, 3
            newshape_list = [
                tuple(
                    (np.array(patch.shape) + (wpad * 2, hpad * 2, 0)).tolist())
                for patch in patch_list
            ]
            color_list = [color_dict[nid_] for nid_ in nid_list]
            newpatch_list = [
                np.zeros(shape) + color[None, None]
                for shape, color in zip(newshape_list, color_list)
            ]
            for patch, newpatch in zip(patch_list, newpatch_list):
                newpatch[wpad:-wpad, hpad:-hpad, :] = patch
            #img_list = patch_list
            #bigpatch = vt.stack_image_recurse(patch_list)
        #bigpatch = vt.stack_image_list(patch_list, vert=False)
        bigpatch = vt.stack_square_images(newpatch_list)
        bigpatch_fpath = join(seldpath, basename(dpath) + '_patches.png')

        #
        def _dictstr(dict_):
            str_ = ut.dict_str(dict_, newlines=False)
            str_ = str_.replace('\'', '').replace(': ', '=').strip('{},')
            return str_

        figtitle = '\n'.join([
            'wx=%r' % wx,
            'stat(pdist): %s' % _dictstr(metrics.wx2_pdist_stats[wx]),
            'stat(wdist): %s' % _dictstr(metrics.wx2_wdist_stats[wx]),
        ])
        metrics.wx2_nMembers[wx]

        df2.figure(fnum=1, doclf=True, docla=True)
        fig, ax = df2.imshow(bigpatch, figtitle=figtitle)
        #fig.show()
        df2.set_figtitle(figtitle)
        df2.adjust_subplots(top=.878, bottom=0)
        df2.save_figure(1, bigpatch_fpath)
Пример #21
0
        if fpaths:
            cmd_to_fpaths[cmd].extend(fpaths)


for key in cmd_to_fpaths.keys():
    cmd = key.lstrip('\\')
    if not root.find_descendant_type(cmd):
        print(key)

from os.path import abspath, dirname
used_fpaths = ut.flatten(cmd_to_fpaths.values())
used_fpaths = set(ut.emap(abspath, used_fpaths))

all_fpaths = set(ut.emap(abspath, ut.glob('.', ['*.png', '*.jpg'], recursive=True)))

unused = list(all_fpaths - used_fpaths)

unuse_dirs = ut.group_items(unused, ut.emap(dirname, unused))


semi_used = {}
for dpath, fpaths in unuse_dirs.items():
    used_in_dpath = set(ut.ls(dpath)) - set(fpaths)
    if len(used_in_dpath) == 0:
        # completely unused directories
        print(dpath)
    else:
        semi_used[dpath] = fpaths

print(ut.repr4(list(semi_used.keys())))