Example #1
0
def load_gztest(ibs):
    r"""
    CommandLine:
        python -m ibeis.algo.hots.special_query --test-load_gztest

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.hots.devcases import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb('GZ_ALL')
    """
    from os.path import join
    from ibeis.algo.hots import match_chips4 as mc4
    dir_ = ut.get_module_dir(mc4)
    eval_text = ut.read_from(join(dir_,  'GZ_TESTTUP.txt'))
    testcases = eval(eval_text)
    count_dict = ut.count_dict_vals(testcases)
    print(ut.dict_str(count_dict))

    testtup_list = ut.flatten(ut.dict_take_list(testcases, ['vsone_wins',
                                                            'vsmany_outperformed',
                                                            'vsmany_dominates',
                                                            'vsmany_wins']))
    qaid_list = [testtup.qaid_t for testtup in testtup_list]
    visual_uuids = ibs.get_annot_visual_uuids(qaid_list)
    visual_uuids
def write_script_lines(line_list, fname):
    exename = 'python'
    regen_cmd = (exename + ' ' + ' '.join(sys.argv)).replace(
        expanduser('~'), '~')
    script_lines = []
    script_lines.append('#!/bin/sh')
    script_lines.append("echo << 'EOF' > /dev/null")
    script_lines.append('RegenCommand:')
    script_lines.append('   ' + regen_cmd)
    script_lines.append('CommandLine:')
    script_lines.append('   sh ' + fname)
    script_lines.append('dont forget to tmuxnew')
    script_lines.append('EOF')
    script_lines.extend(line_list)
    script = '\n'.join(script_lines)
    logger.info(script)
    import wbia
    from os.path import dirname, join

    dpath = dirname(ut.get_module_dir(wbia))
    fpath = join(dpath, fname)
    if not ut.get_argflag('--dryrun'):
        ut.writeto(fpath, script)
        ut.chmod_add_executable(fpath)
    return fname, script, line_list
Example #3
0
def dev_autogen_explicit_injects():
    r"""
    CommandLine:
        python -m ibeis --tf dev_autogen_explicit_injects

    Example:
        >>> # SCRIPT
        >>> from ibeis.control.controller_inject import *  # NOQA
        >>> dev_autogen_explicit_injects()
    """
    import ibeis  # NOQA
    classname = CONTROLLER_CLASSNAME
    regen_command = (
        'python -m ibeis.control.controller_inject '
        '--exec-dev_autogen_explicit_injects')
    import ibeis.control.IBEISControl
    conditional_imports = [
        modname for modname in ibeis.control.IBEISControl.AUTOLOAD_PLUGIN_MODNAMES
        if isinstance(modname, tuple)
    ]
    source_block = ut.autogen_explicit_injectable_metaclass(
        classname, regen_command, conditional_imports)
    dpath = ut.get_module_dir(ibeis.control.IBEISControl)
    fpath = ut.unixjoin(dpath, '_autogen_explicit_controller.py')
    ut.writeto(fpath, source_block)
Example #4
0
def load_gztest(ibs):
    r"""
    CommandLine:
        python -m ibeis.algo.hots.special_query --test-load_gztest

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.hots.devcases import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb('GZ_ALL')
    """
    from os.path import join
    from ibeis.algo.hots import match_chips4 as mc4
    dir_ = ut.get_module_dir(mc4)
    eval_text = ut.read_from(join(dir_, 'GZ_TESTTUP.txt'))
    testcases = eval(eval_text)
    count_dict = ut.count_dict_vals(testcases)
    print(ut.dict_str(count_dict))

    testtup_list = ut.flatten(
        ut.dict_take_list(testcases, [
            'vsone_wins', 'vsmany_outperformed', 'vsmany_dominates',
            'vsmany_wins'
        ]))
    qaid_list = [testtup.qaid_t for testtup in testtup_list]
    visual_uuids = ibs.get_annot_visual_uuids(qaid_list)
    visual_uuids
Example #5
0
def testdata_expts(defaultdb='testdb1',
                   default_acfgstr_name_list=['default:qindex=0:10:4,dindex=0:20'],
                   default_test_cfg_name_list=['default'],
                   a=None,
                   t=None,
                   qaid_override=None,
                   daid_override=None,
                   initial_aids=None,
                   ):
    """
    Use this if you want data from an experiment.
    Command line interface to quickly get testdata for test_results.

    Command line flags can be used to specify db, aidcfg, pipecfg, qaid
    override, daid override (and maybe initial aids).

    """
    print('[main_helpers] testdata_expts')
    import ibeis
    from ibeis.expt import experiment_harness
    from ibeis.expt import test_result
    if a is not None:
        default_acfgstr_name_list = a
    if t is not None:
        default_test_cfg_name_list = t

    if isinstance(default_acfgstr_name_list, six.string_types):
        default_acfgstr_name_list = [default_acfgstr_name_list]
    if isinstance(default_test_cfg_name_list, six.string_types):
        default_test_cfg_name_list = [default_test_cfg_name_list]

    #from ibeis.expt import experiment_helpers
    ibs = ibeis.opendb(defaultdb=defaultdb)
    acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=list,
                                   default=default_acfgstr_name_list)
    test_cfg_name_list = ut.get_argval(('-t', '-p'), type_=list, default=default_test_cfg_name_list)
    daid_override = ut.get_argval(('--daid-override', '--daids-override'), type_=list, default=daid_override)
    qaid_override = ut.get_argval(('--qaid', '--qaids-override', '--qaid-override'), type_=list, default=qaid_override)

    # Hack a cache here
    use_bigtest_cache3 = not ut.get_argflag(('--nocache', '--nocache-hs'))
    use_bigtest_cache3 &= ut.is_developer()
    use_bigtest_cache3 &= False
    if use_bigtest_cache3:
        from os.path import dirname, join
        cache_dir = ut.ensuredir(join(dirname(ut.get_module_dir(ibeis)), 'BIG_TESTLIST_CACHE3'))
        load_testres = ut.cached_func('testreslist', cache_dir=cache_dir)(experiment_harness.run_test_configurations2)
    else:
        load_testres = experiment_harness.run_test_configurations2
    testres_list = load_testres(
        ibs, acfg_name_list, test_cfg_name_list, qaid_override=qaid_override,
        daid_override=daid_override, initial_aids=initial_aids)
    testres = test_result.combine_testres_list(ibs, testres_list)

    print(testres)

    return ibs, testres
Example #6
0
def convert_tests_from_ibeis_to_nose(module_list):
    # PARSE OUT TESTABLE DOCTESTTUPS
    #import utool as ut
    testtup_list = []
    seen_ = set()

    topimport_list = []

    for module in module_list:
        mod_doctest_tup = ut.get_module_doctest_tup(module=module,
                                                    verbose=False,
                                                    allexamples=True)
        enabled_testtup_list, frame_fpath, all_testflags, module = mod_doctest_tup
        flags = [tup.src not in seen_ for tup in enabled_testtup_list]
        enabled_testtup_list = ut.compress(enabled_testtup_list, flags)
        testtup_list.extend(enabled_testtup_list)
        if len(enabled_testtup_list) > 0:
            topimport_list.append('from %s import *  # NOQA' % (module.__name__,))

    print('Found %d test tups' % (len(testtup_list)))

    autogen_test_src_funcs = []
    #import redbaron
    for testtup in testtup_list:
        name = testtup.name
        num  = testtup.num
        src  = testtup.src
        want = testtup.want
        import re
        src = re.sub('# ENABLE_DOCTEST\n', '', src)
        src = re.sub('from [^*]* import \* *# NOQA\n', '', src)
        src = re.sub(r'from [^*]* import \*\n', '', src)

        src = ut.str_between(src, None, 'ut.quit_if_noshow').rstrip('\n')
        src = ut.str_between(src, None, 'ut.show_if_requested').rstrip('\n')
        # import utool
        # utool.embed()
        """
        """
        #flag = testtup.flag
        if want.endswith('\n'):
            want = want[:-1]
        if want:
            #src_node = redbaron.RedBaron(src)
            #if len(src_node.find_all('name', 'result')) > 0:
            #    src_node.append('assert result == %r' % (want,))
            if '\nresult = ' in src:
                src += '\nassert str(result) == %r' % (want,)
        func_src = 'def test_%s_%d():\n' % (name.replace('.', '_'), num,) + ut.indent(src)
        autogen_test_src_funcs.append(func_src)

    autogen_test_src = '\n'.join(topimport_list) + '\n\n\n' + '\n\n\n'.join(autogen_test_src_funcs) + '\n'
    from ibeis import tests
    from os.path import join
    moddir = ut.get_module_dir(tests)
    ut.writeto(join(moddir, 'test_autogen_nose_tests.py'), autogen_test_src)
Example #7
0
def testdata_depc3():
    """
    Example of local registration
    sudo pip install freetype-py

    CommandLine:
        python -m dtool.example_depcache2 testdata_depc3 --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from dtool.example_depcache2 import *  # NOQA
        >>> depc = testdata_depc3()
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> depc.show_graph()
        >>> depc['smk_match'].show_input_graph()
        >>> depc['vsone'].show_input_graph()
        >>> depc['vocab'].show_input_graph()
        >>> depc['neighbs'].show_input_graph()
        >>> depc['viewpoint_classification'].show_input_graph()
        >>> print(depc['smk_match'].compute_order)
        >>> ut.show_if_requested()
    """
    import dtool

    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE2')

    root = 'annot'

    depc = dtool.DependencyCache(
        root_tablename=root, cache_dpath=cache_dpath, use_globals=False)

    # ----------
    dummy_cols = dict(colnames=['data'], coltypes=[np.ndarray])
    def dummy_func(depc, *args, **kwargs):
        return None

    depc.register_preproc(tablename='indexer', parents=['annot*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='neighbs', parents=['annot', 'indexer'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='vocab', parents=['annot*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='smk_vec', parents=['annot', 'vocab'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='inv_index', parents=['smk_vec*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='smk_match', parents=['smk_vec', 'inv_index'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='vsone', parents=['annot', 'annot'], **dummy_cols)(dummy_func)

    depc.register_preproc(tablename='viewpoint_classifier', parents=['annot*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='viewpoint_classification', parents=['annot', 'viewpoint_classifier'], **dummy_cols)(dummy_func)

    depc.initialize()
    return depc
Example #8
0
def convert_tests_from_utool_to_nose(module_list):
    # PARSE OUT TESTABLE DOCTESTTUPS
    #import utool as ut
    testtup_list = []
    seen_ = set()

    topimport_list = []

    for module in module_list:
        mod_doctest_tup = ut.get_module_doctest_tup(module=module,
                                                    verbose=False,
                                                    allexamples=True)
        enabled_testtup_list, frame_fpath, all_testflags, module = mod_doctest_tup
        flags = [tup.src not in seen_ for tup in enabled_testtup_list]
        enabled_testtup_list = ut.compress(enabled_testtup_list, flags)
        testtup_list.extend(enabled_testtup_list)
        if len(enabled_testtup_list) > 0:
            topimport_list.append('from %s import *  # NOQA' % (module.__name__,))

    print('Found %d test tups' % (len(testtup_list)))

    autogen_test_src_funcs = []
    #import redbaron
    for testtup in testtup_list:
        name = testtup.name
        num  = testtup.num
        src  = testtup.src
        want = testtup.want
        import re
        src = re.sub('# ENABLE_DOCTEST\n', '', src)
        src = re.sub('from [^*]* import \* *# NOQA\n', '', src)
        src = re.sub('from [^*]* import \*\n', '', src)
        #flag = testtup.flag
        if want.endswith('\n'):
            want = want[:-1]
        if want:
            #src_node = redbaron.RedBaron(src)
            #if len(src_node.find_all('name', 'result')) > 0:
            #    src_node.append('assert result == %r' % (want,))
            if '\nresult = ' in src:
                src += '\nassert str(result) == %r' % (want,)
        func_src = 'def test_%s_%d():\n' % (name.replace('.', '_'), num,) + ut.indent(src)
        autogen_test_src_funcs.append(func_src)

    autogen_test_src = '\n'.join(topimport_list) + '\n\n\n' + '\n\n\n'.join(autogen_test_src_funcs) + '\n'
    from utool import tests
    from os.path import join
    moddir = ut.get_module_dir(tests)
    ut.writeto(join(moddir, 'test_autogen_nose_tests.py'), autogen_test_src)
Example #9
0
def get_big_test_cache_info(ibs, cfgx2_qreq_):
    """
    Args:
        ibs (ibeis.IBEISController):
        cfgx2_qreq_ (dict):
    """
    if ut.is_developer():
        import ibeis
        repodir = dirname(ut.get_module_dir(ibeis))
        bt_cachedir = join(repodir, 'BIG_TEST_CACHE2')
    else:
        bt_cachedir = join(ibs.get_cachedir(), 'BIG_TEST_CACHE2')
        #bt_cachedir = './localdata/BIG_TEST_CACHE2'
    ut.ensuredir(bt_cachedir)
    bt_cachestr = ut.hashstr_arr27(
        [qreq_.get_cfgstr(with_input=True) for qreq_ in cfgx2_qreq_],
        ibs.get_dbname() + '_cfgs')
    bt_cachename = 'BIGTESTCACHE2'
    return bt_cachedir, bt_cachename, bt_cachestr
Example #10
0
def get_big_test_cache_info(ibs, cfgx2_qreq_):
    """
    Args:
        ibs (ibeis.IBEISController):
        cfgx2_qreq_ (dict):
    """
    if ut.is_developer():
        import ibeis
        repodir = dirname(ut.get_module_dir(ibeis))
        bt_cachedir = join(repodir, 'BIG_TEST_CACHE2')
    else:
        bt_cachedir = join(ibs.get_cachedir(), 'BIG_TEST_CACHE2')
        #bt_cachedir = './localdata/BIG_TEST_CACHE2'
    ut.ensuredir(bt_cachedir)
    bt_cachestr = ut.hashstr_arr27([
        qreq_.get_cfgstr(with_input=True)
        for qreq_ in cfgx2_qreq_],
        ibs.get_dbname() + '_cfgs')
    bt_cachename = 'BIGTESTCACHE2'
    return bt_cachedir, bt_cachename, bt_cachestr
Example #11
0
def testdata_depc_image():
    """
    Example of local registration
    sudo pip install freetype-py

    CommandLine:
        python -m dtool.example_depcache2 testdata_depc_image --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from dtool.example_depcache2 import *  # NOQA
        >>> depc = testdata_depc_image()
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> depc.show_graph()
        >>> depc['detection'].show_input_graph()
        >>> print(depc['detection'].compute_order)
        >>> ut.show_if_requested()
    """
    import dtool

    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE2')

    root = 'image'

    depc = dtool.DependencyCache(
        root_tablename=root, cache_dpath=cache_dpath, use_globals=False)

    # ----------
    dummy_cols = dict(colnames=['data'], coltypes=[np.ndarray])
    def dummy_func(depc, *args, **kwargs):
        for row_arg in zip(*args):
            yield (np.array([42]),)

    depc.register_preproc(tablename='detector', parents=['image*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='detection', parents=['image', 'detector'], **dummy_cols)(dummy_func)

    depc.initialize()
    return depc
Example #12
0
def testdata_custom_annot_depc(dummy_dependencies, in_memory=True):
    import dtool
    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE5')
    # FIXME: this only puts the sql files in memory
    default_fname = ':memory:' if in_memory else None
    root = 'annot'
    depc = dtool.DependencyCache(root_tablename=root,
                                 get_root_uuid=ut.identity,
                                 default_fname=default_fname,
                                 cache_dpath=cache_dpath,
                                 use_globals=False)
    # ----------
    register_dummy_config = depc_34_helper(depc)

    for dummy in dummy_dependencies:
        register_dummy_config(**dummy)

    depc.initialize()
    return depc
Example #13
0
def write_script_lines(line_list, fname):
    exename = 'python'
    regen_cmd = (exename + ' ' + ' '.join(sys.argv)).replace(expanduser('~'), '~')
    script_lines = []
    script_lines.append('#!/bin/sh')
    script_lines.append('echo << \'EOF\' > /dev/null')
    script_lines.append('RegenCommand:')
    script_lines.append('   ' + regen_cmd)
    script_lines.append('CommandLine:')
    script_lines.append('   sh ' + fname)
    script_lines.append('dont forget to tmuxnew')
    script_lines.append('EOF')
    script_lines.extend(line_list)
    script = '\n'.join(script_lines)
    print(script)
    import ibeis
    from os.path import dirname, join
    dpath = dirname(ut.get_module_dir(ibeis))
    fpath = join(dpath, fname)
    if not ut.get_argflag('--dryrun'):
        ut.writeto(fpath, script)
        ut.chmod_add_executable(fpath)
    return fname, script, line_list
Example #14
0
def testdata_depc2():
    """
    Example of local registration
    sudo pip install freetype-py

    CommandLine:
        python -m dtool.example_depcache2 testdata_depc2 --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from dtool.example_depcache2 import *  # NOQA
        >>> depc = testdata_depc2()
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> depc.show_graph()
        >>> ut.show_if_requested()
    """
    import dtool
    import vtool as vt
    from vtool import fontdemo

    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE2')

    root = 'annot'

    depc = dtool.DependencyCache(
        root_tablename=root, cache_dpath=cache_dpath, use_globals=False)

    # ----------

    class ChipConfig(dtool.Config):
        _param_info_list = [
            ut.ParamInfo('dim_size', 500),
            ut.ParamInfo('ext', '.png'),
        ]

    @depc.register_preproc(
        tablename='chip', parents=[root], colnames=['size', 'img'],
        coltypes=[(int, int), ('extern', vt.imread, vt.imwrite)],
        configclass=ChipConfig)
    def compute_chip(depc, aids, config=None):
        for aid in aids:
            chip = fontdemo.get_text_test_img(str(aid))
            size = vt.get_size(chip)
            yield size, chip

    # ----------

    class TipConfig(dtool.Config):
        _param_info_list = [
            ut.ParamInfo('manual_extract', False, hideif=False),
        ]

    @depc.register_preproc(
        tablename='tip', parents=['chip'],
        colnames=['notch', 'left', 'right'],
        coltypes=[np.ndarray, np.ndarray, np.ndarray],
        configclass=TipConfig,
    )
    def compute_tips(depc, chip_rowids, config=None):
        manual_extract = config['manual_extract']
        chips = depc.get_native('chip', chip_rowids, 'img')
        for chip in chips:
            seed = (chip).sum()
            perb = ((seed % 1000) / 1000) * .25
            w, h = vt.get_size(chip)
            if manual_extract:
                # Make noticable difference between config outputs
                lpb =  np.ceil(w * perb)
                npb =  np.ceil(h * perb)
                rpb = -np.ceil(w * perb)
            else:
                lpb =  np.ceil(w * perb / 2)
                npb = -np.ceil(h * perb)
                rpb = -np.ceil(w * perb)
            wh = np.array([w, h], dtype=np.int32)[None, :]
            rel_base = np.array([[.0, .5], [.5, .5], [1., .5]])
            offset   = np.array([[lpb, 0], [0, npb], [rpb, 0]])
            tip = np.round((wh * rel_base)) + offset
            left, notch, right = tip
            yield left, notch, right

    # ----------

    class CropChipConfig(dtool.Config):
        _param_info_list = [
            ut.ParamInfo('dim_size', 500),
        ]

    @depc.register_preproc(
        tablename='cropchip', parents=['chip', 'tip'],
        colnames=['img'],
        coltypes=[np.ndarray],
        configclass=CropChipConfig,
    )
    def compute_cropchip(depc, cids, tids, config=None):
        print("COMPUTE CROPCHIP")
        print('config = %r' % (config,))
        chips = depc.get_native('chip', cids, 'img')
        tips = depc.get_native('tip', tids)
        print('tips = %r' % (tips,))
        for chip, tip in zip(chips, tips):
            notch, left, right = tip
            lx = left[0]
            rx = right[0]
            cropped_chip = chip[lx:(rx - 1), ...]
            yield (cropped_chip,)

    # ----------

    class TrailingEdgeConfig(dtool.Config):
        _param_info_list = []

    @depc.register_preproc(
        tablename='trailingedge', parents=['cropchip'],
        colnames=['te'],
        coltypes=[np.ndarray],
        configclass=TrailingEdgeConfig,
    )
    def compute_trailing_edge(depc, cropped_chips, config=None):
        for cc in cropped_chips:
            #depc.get_native('chip', cids)
            size = 1
            te = np.arange(size)
            yield (te,)

    depc.initialize()
    return depc
Example #15
0
def testdata_depc(fname=None):
    """
    Example of local registration
    """

    import dtool
    import vtool as vt
    gpath_list = ut.lmap(ut.grab_test_imgpath,
                         ut.get_valid_test_imgkeys(),
                         verbose=False)

    dummy_root = 'dummy_annot'

    def get_root_uuid(aid_list):
        return ut.lmap(ut.hashable_to_uuid, aid_list)

    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE')

    depc = dtool.DependencyCache(
        root_tablename=dummy_root,
        default_fname=fname,
        cache_dpath=cache_dpath,
        get_root_uuid=get_root_uuid,
        #root_asobject=root_asobject,
        use_globals=False)

    @depc.register_preproc(tablename='chip',
                           parents=[dummy_root],
                           colnames=['size', 'chip'],
                           coltypes=[(int, int),
                                     ('extern', vt.imread, vt.imwrite)],
                           configclass=DummyChipConfig)
    def dummy_preproc_chip(depc, annot_rowid_list, config=None):
        """
        TODO: Infer properties from docstr?

        Args:
            depc (dtool.DependencyCache):
            annot_rowid_list (list): list of annot rowids
            config (dict): config dictionary

        Returns:
            tuple : ((int, int), ('extern', vt.imread))
        """
        if config is None:
            config = {}
        # Demonstates using asobject to get input to function as a dictionary
        # of properties
        #for annot in annot_list:
        #print('[preproc] Computing chips of aid=%r' % (aid,))
        print('[preproc] Computing chips')
        for aid in annot_rowid_list:
            #aid = annot['aid']
            #chip_fpath = annot['gpath']
            chip_fpath = gpath_list[aid]
            #w, h = vt.image.open_image_size(chip_fpath)
            chip = vt.imread(chip_fpath)
            size = vt.get_size(chip)
            #size = (w, h)
            print('Dummpy preproc chip yeilds')
            print('* chip_fpath = %r' % (chip_fpath, ))
            print('* size = %r' % (size, ))
            #yield size, chip_fpath
            yield size, chip

    @depc.register_preproc(
        'probchip',
        [dummy_root],
        ['size', 'probchip'],
        coltypes=[(int, int), ('extern', vt.imread, vt.imwrite, '.png')],
        configclass=ProbchipConfig,
    )
    def dummy_preproc_probchip(depc, root_rowids, config):
        print('[preproc] Computing probchip')
        for rowid in root_rowids:
            if config['testerror']:
                if rowid % 2 == 0:
                    # Test error yeilds None on even rowids
                    yield None
                    continue
            rng = np.random.RandomState(rowid)
            probchip = rng.randint(0, 255, size=(64, 64))
            #probchip = np.zeros((64, 64))
            size = (rowid, rowid)
            yield size, probchip

    @depc.register_preproc(
        'keypoint',
        ['chip'],
        ['kpts', 'num'],
        [np.ndarray, int],
        #default_onthefly=True,
        configclass=DummyKptsConfig,
        docstr='Used to store individual chip features (ellipses)',
    )
    def dummy_preproc_kpts(depc, chip_rowids, config=None):
        if config is None:
            config = {}
        print('config = %r' % (config, ))
        adapt_shape = config['adapt_shape']
        print('[preproc] Computing kpts')

        ut.assert_all_not_None(chip_rowids, 'chip_rowids')
        # This is in here to attempt to trigger a failure of the chips dont
        # exist and the feature cache is called.
        chip_fpath_list = depc.get_native('chip',
                                          chip_rowids,
                                          'chip',
                                          read_extern=False)
        print('computing featurse from chip_fpath_list = %r' %
              (chip_fpath_list, ))

        for rowid in chip_rowids:
            if adapt_shape:
                kpts = np.zeros((7 + rowid, 6)) + rowid
            else:
                kpts = np.ones((7 + rowid, 6)) + rowid
            num = len(kpts)
            yield kpts, num

    @depc.register_preproc(
        'descriptor',
        ['keypoint'],
        ['vecs'],
        [np.ndarray],
    )
    def dummy_preproc_vecs(depc, kp_rowid, config=None):
        if config is None:
            config = {}
        print('[preproc] Computing vecs')
        for rowid in kp_rowid:
            yield np.ones((7 + rowid, 8), dtype=np.uint8) + rowid,

    @depc.register_preproc(
        'fgweight',
        ['keypoint', 'probchip'],
        ['fgweight'],
        [np.ndarray],
    )
    def dummy_preproc_fgweight(depc, kpts_rowid, probchip_rowid, config=None):
        if config is None:
            config = {}
        print('[preproc] Computing fgweight')
        for rowid1, rowid2 in zip(kpts_rowid, probchip_rowid):
            yield np.ones(7 + rowid1),

    @depc.register_preproc(tablename='vsmany',
                           colnames='annotmatch',
                           coltypes=DummyAnnotMatch,
                           requestclass=DummyVsManyRequest,
                           configclass=DummyVsManyConfig)
    def vsmany_matching(depc, qaids, config=None):
        """
        CommandLine:
            python -m dtool.base --exec-VsManySimilarityRequest
        """
        print('RUNNING DUMMY VSMANY ALGO')
        daids = config.daids
        qaids = qaids

        sver_on = config.dummy_sver_cfg['sver_on']
        kpts_list = depc.get_property('keypoint', list(qaids))  # NOQA
        #dummy_preproc_kpts
        for qaid in qaids:
            dnid_list = [1, 1, 2, 2]
            unique_nids = [1, 2]
            if sver_on:
                annot_score_list = [.2, .2, .4, .5]
                name_score_list = [.2, .5]
            else:
                annot_score_list = [.3, .3, .6, .9]
                name_score_list = [.1, .7]
            annot_match = DummyAnnotMatch(qaid, daids, dnid_list,
                                          annot_score_list, unique_nids,
                                          name_score_list)
            yield annot_match

    SIMPLE = 0
    if not SIMPLE:

        @depc.register_preproc(tablename='chipmask',
                               parents=[dummy_root],
                               colnames=['size', 'mask'],
                               coltypes=[(int, int),
                                         ('extern', vt.imread, vt.imwrite)])
        def dummy_manual_chipmask(depc, parent_rowids, config=None):
            import vtool as vt
            from plottool import interact_impaint
            mask_dpath = join(depc.cache_dpath, 'ManualChipMask')
            ut.ensuredir(mask_dpath)
            if config is None:
                config = {}
            print('Requesting user defined chip mask')
            for rowid in parent_rowids:
                img = vt.imread(gpath_list[rowid])
                mask = interact_impaint.impaint_mask2(img)
                mask_fpath = join(mask_dpath, 'mask%d.png' % (rowid, ))
                vt.imwrite(mask_fpath, mask)
                w, h = vt.get_size(mask)
                yield (w, h), mask_fpath

        @depc.register_preproc(
            'notch',
            [dummy_root],
            ['notchdata'],
            [np.ndarray],
        )
        def dummy_preproc_notch(depc, parent_rowids, config=None):
            if config is None:
                config = {}
            print('[preproc] Computing notch')
            for rowid in parent_rowids:
                yield np.empty(5 + rowid),

        @depc.register_preproc(
            'spam',
            ['fgweight', 'chip', 'keypoint'],
            ['spam', 'eggs', 'size', 'uuid', 'vector', 'textdata'],
            [
                str, int, (int, int), uuid.UUID, np.ndarray,
                ('extern', ut.readfrom)
            ],
            docstr='I dont like spam',
        )
        def dummy_preproc_spam(depc, *args, **kwargs):
            config = kwargs.get('config', None)
            if config is None:
                config = {}
            print('[preproc] Computing spam')
            ut.writeto('tmp.txt', ut.lorium_ipsum())
            for x in zip(*args):
                size = (42, 21)
                uuid = ut.get_zero_uuid()
                vector = np.ones(3)
                yield ('spam', 3665, size, uuid, vector, 'tmp.txt')

        @depc.register_preproc(
            'nnindexer',
            ['keypoint*'],
            ['flann'],
            [str],  # [('extern', ut.load_data)],
            configclass=DummyIndexerConfig,
        )
        def dummy_preproc_indexer(depc, parent_rowids_list, config=None):
            print('COMPUTING DUMMY INDEXER')
            #assert len(parent_rowids_list) == 1, 'handles only one indexer'
            for parent_rowids in parent_rowids_list:
                yield ('really cool flann object' + str(config.get_cfgstr()) +
                       ' ' + str(parent_rowids), )

        @depc.register_preproc(
            'notchpair',
            ['notch', 'notch'],
            ['pairscore'],
            [int],  # [('extern', ut.load_data)],
            #configclass=DummyIndexerConfig,
        )
        def dummy_notchpair(depc, n1, n2, config=None):
            print('COMPUTING MULTITEST 1 ')
            #assert len(parent_rowids_list) == 1, 'handles only one indexer'
            for nn1, nn2 in zip(n1, n2):
                yield (nn1 + nn2, )

        @depc.register_preproc(
            'multitest',
            [
                'keypoint', 'notch', 'notch', 'fgweight*', 'notchpair*',
                'notchpair*', 'notchpair', 'nnindexer'
            ],
            ['foo'],
            [str],  # [('extern', ut.load_data)],
            #configclass=DummyIndexerConfig,
        )
        def dummy_multitest(depc, *args, **kwargs):
            print('COMPUTING MULTITEST 1 ')
            #assert len(parent_rowids_list) == 1, 'handles only one indexer'
            for x in zip(args):
                yield ('cool multi object' + str(kwargs) + ' ' + str(x), )

        # TEST MULTISET DEPENDENCIES
        @depc.register_preproc(
            'multitest_score',
            ['multitest'],
            ['score'],
            [int],  # [('extern', ut.load_data)],
            #configclass=DummyIndexerConfig,
        )
        def dummy_multitest_score(depc, parent_rowids, config=None):
            print('COMPUTING DEPENDENCY OF MULTITEST 1 ')
            #assert len(parent_rowids_list) == 1, 'handles only one indexer'
            for parent_rowids in zip(parent_rowids):
                yield (parent_rowids, )

        # TEST MULTISET DEPENDENCIES
        @depc.register_preproc(
            'multitest_score_x',
            ['multitest_score', 'multitest_score'],
            ['score'],
            [int],  # [('extern', ut.load_data)],
            #configclass=DummyIndexerConfig,
        )
        def multitest_score_x(depc, *args, **kwargs):
            raise NotImplementedError('hack')

        # REGISTER MATCHING ALGORITHMS

        @depc.register_preproc(
            tablename='neighbs',
            colnames=['qx2_idx', 'qx2_dist'],
            coltypes=[np.ndarray, np.ndarray],
            parents=['keypoint', 'fgweight', 'nnindexer', 'nnindexer'])
        def neighbs(depc, *args, **kwargs):
            """
            CommandLine:
                python -m dtool.base --exec-VsManySimilarityRequest
            """
            #dummy_preproc_kpts
            for qaid in zip(args):
                yield np.array([qaid]), np.array([qaid])

        @depc.register_preproc(tablename='neighbs_score',
                               colnames=['qx2_dist'],
                               coltypes=[np.ndarray],
                               parents=['neighbs'])
        def neighbs_score(depc, *args, **kwargs):
            """
            CommandLine:
                python -m dtool.base --exec-VsManySimilarityRequest
            """
            raise NotImplementedError('hack')

        @depc.register_preproc('vsone', [dummy_root, dummy_root],
                               ['score', 'match_obj', 'fm'],
                               [float, DummyVsOneMatch, np.ndarray],
                               requestclass=DummyVsOneRequest,
                               configclass=DummyVsOneConfig,
                               chunksize=2)
        def compute_vsone_matching(depc, qaids, daids, config):
            """
            CommandLine:
                python -m dtool.base --exec-VsOneSimilarityRequest
            """
            print('RUNNING DUMMY VSONE ALGO')
            for qaid, daid in zip(qaids, daids):
                match = DummyVsOneMatch()
                match.qaid = qaid
                match.daid = daid
                match.fm = np.array([[1, 2], [3, 4]])
                score = match.score = qaid + daid
                yield (score, match, match.fm)

    # table = depc['spam']
    # print(ut.repr2(table.get_addtable_kw(), nl=2))
    depc.initialize()
    # table.print_schemadef()
    # print(table.db.get_schema_current_autogeneration_str())
    return depc
Example #16
0
def testdata_depc(fname=None):
    """
    Example of local registration
    """

    import dtool
    import vtool as vt
    gpath_list = ut.lmap(ut.grab_test_imgpath, ut.get_valid_test_imgkeys(),
                         verbose=False)

    dummy_root = 'dummy_annot'

    def get_root_uuid(aid_list):
        return ut.lmap(ut.hashable_to_uuid, aid_list)

    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE')

    depc = dtool.DependencyCache(
        root_tablename=dummy_root, default_fname=fname,
        cache_dpath=cache_dpath,
        get_root_uuid=get_root_uuid,
        #root_asobject=root_asobject,
        use_globals=False)

    @depc.register_preproc(tablename='chip', parents=[dummy_root],
                           colnames=['size', 'chip'],
                           coltypes=[(int, int), ('extern', vt.imread, vt.imwrite)],
                           configclass=DummyChipConfig)
    def dummy_preproc_chip(depc, annot_rowid_list, config=None):
        """
        TODO: Infer properties from docstr?

        Args:
            depc (dtool.DependencyCache):
            annot_rowid_list (list): list of annot rowids
            config (dict): config dictionary

        Returns:
            tuple : ((int, int), ('extern', vt.imread))
        """
        if config is None:
            config = {}
        # Demonstates using asobject to get input to function as a dictionary
        # of properties
        #for annot in annot_list:
        #print('[preproc] Computing chips of aid=%r' % (aid,))
        print('[preproc] Computing chips')
        for aid in annot_rowid_list:
            #aid = annot['aid']
            #chip_fpath = annot['gpath']
            chip_fpath = gpath_list[aid]
            #w, h = vt.image.open_image_size(chip_fpath)
            chip = vt.imread(chip_fpath)
            size = vt.get_size(chip)
            #size = (w, h)
            #print('* chip_fpath = %r' % (chip_fpath,))
            #print('* size = %r' % (size,))
            #yield size, chip_fpath
            yield size, chip

    @depc.register_preproc(
        'probchip', [dummy_root], ['size', 'probchip'],
        coltypes=[(int, int), ('extern', vt.imread, vt.imwrite, '.png')],
        configclass=ProbchipConfig,
    )
    def dummy_preproc_probchip(depc, root_rowids, config):
        print('[preproc] Computing probchip')
        for rowid in root_rowids:
            if config['testerror']:
                if rowid % 2 == 0:
                    # Test error yeilds None on even rowids
                    yield None
                    continue
            rng = np.random.RandomState(rowid)
            probchip = rng.randint(0, 255, size=(64, 64))
            #probchip = np.zeros((64, 64))
            size = (rowid, rowid)
            yield size, probchip

    @depc.register_preproc(
        'keypoint', ['chip'], ['kpts', 'num'], [np.ndarray, int],
        #default_onthefly=True,
        configclass=DummyKptsConfig,
        docstr='Used to store individual chip features (ellipses)',)
    def dummy_preproc_kpts(depc, chip_rowids, config=None):
        if config is None:
            config = {}
        print('config = %r' % (config,))
        adapt_shape = config['adapt_shape']
        print('[preproc] Computing kpts')
        for rowid in chip_rowids:
            if adapt_shape:
                kpts = np.zeros((7 + rowid, 6)) + rowid
            else:
                kpts = np.ones((7 + rowid, 6)) + rowid
            num = len(kpts)
            yield kpts, num

    @depc.register_preproc('descriptor', ['keypoint'], ['vecs'], [np.ndarray],)
    def dummy_preproc_vecs(depc, kp_rowid, config=None):
        if config is None:
            config = {}
        print('[preproc] Computing vecs')
        for rowid in kp_rowid:
            yield np.ones((7 + rowid, 8), dtype=np.uint8) + rowid,

    @depc.register_preproc('fgweight', ['keypoint', 'probchip'], ['fgweight'], [np.ndarray],)
    def dummy_preproc_fgweight(depc, kpts_rowid, probchip_rowid, config=None):
        if config is None:
            config = {}
        print('[preproc] Computing fgweight')
        for rowid1, rowid2 in zip(kpts_rowid, probchip_rowid):
            yield np.ones(7 + rowid1),

    @depc.register_preproc(
        tablename='vsmany', colnames='annotmatch', coltypes=DummyAnnotMatch,
        requestclass=DummyVsManyRequest, configclass=DummyVsManyConfig)
    def vsmany_matching(depc, qaids, config=None):
        """
        CommandLine:
            python -m dtool.base --exec-VsManySimilarityRequest
        """
        print('RUNNING DUMMY VSMANY ALGO')
        daids = config.daids
        qaids = qaids

        sver_on = config.dummy_sver_cfg['sver_on']
        kpts_list = depc.get_property('keypoint', qaids)  # NOQA
        #dummy_preproc_kpts
        for qaid in qaids:
            dnid_list = [1, 1, 2, 2]
            unique_nids = [1, 2]
            if sver_on:
                annot_score_list = [.2, .2, .4, .5]
                name_score_list = [.2, .5]
            else:
                annot_score_list = [.3, .3, .6, .9]
                name_score_list = [.1, .7]
            annot_match = DummyAnnotMatch(qaid, daids, dnid_list,
                                          annot_score_list, unique_nids,
                                          name_score_list)
            yield annot_match

    SIMPLE = 0
    if not SIMPLE:

        @depc.register_preproc(
            tablename='chipmask', parents=[dummy_root], colnames=['size', 'mask'],
            coltypes=[(int, int), ('extern', vt.imread, vt.imwrite)])
        def dummy_manual_chipmask(depc, parent_rowids, config=None):
            import vtool as vt
            from plottool import interact_impaint
            mask_dpath = join(depc.cache_dpath, 'ManualChipMask')
            ut.ensuredir(mask_dpath)
            if config is None:
                config = {}
            print('Requesting user defined chip mask')
            for rowid in parent_rowids:
                img = vt.imread(gpath_list[rowid])
                mask = interact_impaint.impaint_mask2(img)
                mask_fpath = join(mask_dpath, 'mask%d.png' % (rowid,))
                vt.imwrite(mask_fpath, mask)
                w, h = vt.get_size(mask)
                yield (w, h), mask_fpath

        @depc.register_preproc('notch', [dummy_root], ['notchdata'], [np.ndarray],)
        def dummy_preproc_notch(depc, parent_rowids, config=None):
            if config is None:
                config = {}
            print('[preproc] Computing notch')
            for rowid in parent_rowids:
                yield np.empty(5 + rowid),

        @depc.register_preproc(
            'spam', ['fgweight', 'chip', 'keypoint'],
            ['spam', 'eggs', 'size', 'uuid', 'vector', 'textdata'],
            [str, int, (int, int), uuid.UUID, np.ndarray, ('extern', ut.readfrom)],
            docstr='I dont like spam',)
        def dummy_preproc_spam(depc, *args, **kwargs):
            config = kwargs.get('config', None)
            if config is None:
                config = {}
            print('[preproc] Computing spam')
            ut.writeto('tmp.txt', ut.lorium_ipsum())
            for x in zip(*args):
                size = (42, 21)
                uuid = ut.get_zero_uuid()
                vector = np.ones(3)
                yield ('spam', 3665, size, uuid, vector, 'tmp.txt')

        @depc.register_preproc(
            'nnindexer', ['keypoint*'], ['flann'], [str],  # [('extern', ut.load_data)],
            configclass=DummyIndexerConfig,
        )
        def dummy_preproc_indexer(depc, parent_rowids_list, config=None):
            print('COMPUTING DUMMY INDEXER')
            #assert len(parent_rowids_list) == 1, 'handles only one indexer'
            for parent_rowids in parent_rowids_list:
                yield ('really cool flann object' + str(config.get_cfgstr()) + ' ' + str(parent_rowids),)

        @depc.register_preproc(
            'notchpair', ['notch', 'notch'], ['pairscore'], [int],  # [('extern', ut.load_data)],
            #configclass=DummyIndexerConfig,
        )
        def dummy_notchpair(depc, n1, n2, config=None):
            print('COMPUTING MULTITEST 1 ')
            #assert len(parent_rowids_list) == 1, 'handles only one indexer'
            for nn1, nn2 in zip(n1, n2):
                yield (nn1 + nn2,)

        @depc.register_preproc(
            'multitest', ['keypoint', 'notch', 'notch', 'fgweight*', 'notchpair*', 'notchpair*', 'notchpair', 'nnindexer'], ['foo'], [str],  # [('extern', ut.load_data)],
            #configclass=DummyIndexerConfig,
        )
        def dummy_multitest(depc, *args, **kwargs):
            print('COMPUTING MULTITEST 1 ')
            #assert len(parent_rowids_list) == 1, 'handles only one indexer'
            for x in zip(args):
                yield ('cool multi object' + str(kwargs) + ' ' + str(x),)

        # TEST MULTISET DEPENDENCIES
        @depc.register_preproc(
            'multitest_score', ['multitest'], ['score'], [int],  # [('extern', ut.load_data)],
            #configclass=DummyIndexerConfig,
        )
        def dummy_multitest_score(depc, parent_rowids, config=None):
            print('COMPUTING DEPENDENCY OF MULTITEST 1 ')
            #assert len(parent_rowids_list) == 1, 'handles only one indexer'
            for parent_rowids in zip(parent_rowids):
                yield (parent_rowids,)

        # TEST MULTISET DEPENDENCIES
        @depc.register_preproc(
            'multitest_score_x', ['multitest_score', 'multitest_score'], ['score'], [int],  # [('extern', ut.load_data)],
            #configclass=DummyIndexerConfig,
        )
        def multitest_score_x(depc, *args, **kwargs):
            raise NotImplementedError('hack')
        # REGISTER MATCHING ALGORITHMS

        @depc.register_preproc(tablename='neighbs', colnames=['qx2_idx', 'qx2_dist'],
                               coltypes=[np.ndarray, np.ndarray],
                               parents=['keypoint', 'fgweight', 'nnindexer', 'nnindexer'])
        def neighbs(depc, *args, **kwargs):
            """
            CommandLine:
                python -m dtool.base --exec-VsManySimilarityRequest
            """
            #dummy_preproc_kpts
            for qaid in zip(args):
                yield np.array([qaid]), np.array([qaid])

        @depc.register_preproc(tablename='neighbs_score', colnames=['qx2_dist'],
                               coltypes=[np.ndarray],
                               parents=['neighbs'])
        def neighbs_score(depc, *args, **kwargs):
            """
            CommandLine:
                python -m dtool.base --exec-VsManySimilarityRequest
            """
            raise NotImplementedError('hack')

        @depc.register_preproc(
            'vsone', [dummy_root, dummy_root],
            ['score', 'match_obj', 'fm'],
            [float, DummyVsOneMatch, np.ndarray],
            requestclass=DummyVsOneRequest,
            configclass=DummyVsOneConfig,
            chunksize=2
        )
        def vsone_matching(depc, qaids, daids, config):
            """
            CommandLine:
                python -m dtool.base --exec-VsOneSimilarityRequest
            """
            print('RUNNING DUMMY VSONE ALGO')
            for qaid, daid in zip(qaids, daids):
                match = DummyVsOneMatch()
                match.qaid = qaid
                match.daid = daid
                match.fm = np.array([[1, 2], [3, 4]])
                score = match.score = qaid + daid
                yield (score, match, match.fm)

    # table = depc['spam']
    # print(ut.repr2(table.get_addtable_kw(), nl=2))
    depc.initialize()
    # table.print_schemadef()
    # print(table.db.get_schema_current_autogeneration_str())
    return depc
Example #17
0
def testdata_depc_annot():
    """
    CommandLine:
        python -m dtool.example_depcache2 testdata_depc_annot --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from dtool.example_depcache2 import *  # NOQA
        >>> depc = testdata_depc_annot()
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> depc.show_graph()
        >>> tablename = 'featweight'
        >>> table = depc[tablename]
        >>> table.show_input_graph()
        >>> print(table.compute_order)
        >>> ut.show_if_requested()
    """
    import dtool
    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE2')
    dummy_cols = dict(colnames=['data'], coltypes=[np.ndarray])
    def dummy_func(depc, *args, **kwargs):
        for row_arg in zip(*args):
            yield (np.array([42]),)

    # NOTE: Consider the smk_match.
    # It would be really cool if we could say that the vocab
    # for the input to the parent smk_vec must be the same vocab
    # that was used to compute the inverted index. How do we encode that?

    root = 'annot'
    #vocab_parent = 'annot'
    #vocab_parent = 'chip'
    #vocab_parent = 'feat'
    vocab_parent = 'featweight'
    depc = dtool.DependencyCache(
        root_tablename=root, cache_dpath=cache_dpath, use_globals=False)
    depc.register_preproc(tablename='chip', parents=['annot'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='fgmodel', parents=['chip*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='probchip', parents=['annot', 'fgmodel'], **dummy_cols)(dummy_func)
    #depc.register_preproc(tablename='probchip', parents=['annot'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='feat', parents=['chip'], **dummy_cols)(dummy_func)
    #depc.register_preproc(tablename='feat', parents=['annot'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='featweight', parents=['feat', 'probchip'], **dummy_cols)(dummy_func)

    depc.register_preproc(tablename='indexer', parents=[vocab_parent + '*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='neighbs', parents=[vocab_parent, 'indexer'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='vocab', parents=[vocab_parent + '*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='smk_vec', parents=[vocab_parent, 'vocab'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='inv_index', parents=['smk_vec*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='smk_match', parents=['smk_vec', 'inv_index'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='vsone', parents=[vocab_parent, vocab_parent], **dummy_cols)(dummy_func)

    depc.register_preproc(tablename='viewpoint_model', parents=['annot*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='viewpoint', parents=['annot', 'viewpoint_model'], **dummy_cols)(dummy_func)

    depc.register_preproc(tablename='quality_model', parents=['annot*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='quality', parents=['annot', 'quality_model'], **dummy_cols)(dummy_func)

    depc.initialize()
    return depc
Example #18
0
def testdata_depc2():
    """
    Example of local registration
    sudo pip install freetype-py

    CommandLine:
        python -m dtool.example_depcache2 testdata_depc2 --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from dtool.example_depcache2 import *  # NOQA
        >>> depc = testdata_depc2()
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> depc.show_graph()
        >>> ut.show_if_requested()
    """
    import dtool
    import vtool as vt
    from vtool import fontdemo

    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE2')

    root = 'annot'

    depc = dtool.DependencyCache(
        root_tablename=root, cache_dpath=cache_dpath, use_globals=False)

    # ----------

    class ChipConfig(dtool.Config):
        _param_info_list = [
            ut.ParamInfo('dim_size', 500),
            ut.ParamInfo('ext', '.png'),
        ]

    @depc.register_preproc(
        tablename='chip', parents=[root], colnames=['size', 'img'],
        coltypes=[(int, int), ('extern', vt.imread, vt.imwrite)],
        configclass=ChipConfig)
    def compute_chip(depc, aids, config=None):
        for aid in aids:
            chip = fontdemo.get_text_test_img(str(aid))
            size = vt.get_size(chip)
            yield size, chip

    # ----------

    class TipConfig(dtool.Config):
        _param_info_list = [
            ut.ParamInfo('manual_extract', False, hideif=False),
        ]

    @depc.register_preproc(
        tablename='tip', parents=['chip'],
        colnames=['notch', 'left', 'right'],
        coltypes=[np.ndarray, np.ndarray, np.ndarray],
        configclass=TipConfig,
    )
    def compute_tips(depc, chip_rowids, config=None):
        manual_extract = config['manual_extract']
        chips = depc.get_native('chip', chip_rowids, 'img')
        for chip in chips:
            seed = (chip).sum()
            perb = ((seed % 1000) / 1000) * .25
            w, h = vt.get_size(chip)
            if manual_extract:
                # Make noticable difference between config outputs
                lpb =  np.ceil(w * perb)
                npb =  np.ceil(h * perb)
                rpb = -np.ceil(w * perb)
            else:
                lpb =  np.ceil(w * perb / 2)
                npb = -np.ceil(h * perb)
                rpb = -np.ceil(w * perb)
            wh = np.array([w, h], dtype=np.int32)[None, :]
            rel_base = np.array([[.0, .5], [.5, .5], [1., .5]])
            offset   = np.array([[lpb, 0], [0, npb], [rpb, 0]])
            tip = np.round((wh * rel_base)) + offset
            left, notch, right = tip
            yield left, notch, right

    # ----------

    class CropChipConfig(dtool.Config):
        _param_info_list = [
            ut.ParamInfo('dim_size', 500),
        ]

    @depc.register_preproc(
        tablename='cropchip', parents=['chip', 'tip'],
        colnames=['img'],
        coltypes=[np.ndarray],
        configclass=CropChipConfig,
    )
    def compute_cropchip(depc, cids, tids, config=None):
        print("COMPUTE CROPCHIP")
        print('config = %r' % (config,))
        chips = depc.get_native('chip', cids, 'img')
        tips = depc.get_native('tip', tids)
        print('tips = %r' % (tips,))
        for chip, tip in zip(chips, tips):
            notch, left, right = tip
            lx = left[0]
            rx = right[0]
            cropped_chip = chip[lx:(rx - 1), ...]
            yield (cropped_chip,)

    # ----------

    class TrailingEdgeConfig(dtool.Config):
        _param_info_list = []

    @depc.register_preproc(
        tablename='trailingedge', parents=['cropchip'],
        colnames=['te'],
        coltypes=[np.ndarray],
        configclass=TrailingEdgeConfig,
    )
    def compute_trailing_edge(depc, cropped_chips, config=None):
        for cc in cropped_chips:
            #depc.get_native('chip', cids)
            size = 1
            te = np.arange(size)
            yield (te,)

    depc.initialize()
    return depc
Example #19
0
def testdata_depc_annot():
    """
    CommandLine:
        python -m dtool.example_depcache2 testdata_depc_annot --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from dtool.example_depcache2 import *  # NOQA
        >>> depc = testdata_depc_annot()
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> depc.show_graph()
        >>> tablename = 'featweight'
        >>> table = depc[tablename]
        >>> table.show_input_graph()
        >>> print(table.compute_order)
        >>> ut.show_if_requested()
    """
    import dtool
    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE2')
    dummy_cols = dict(colnames=['data'], coltypes=[np.ndarray])
    def dummy_func(depc, *args, **kwargs):
        return None

    # NOTE: Consider the smk_match.
    # It would be really cool if we could say that the vocab
    # for the input to the parent smk_vec must be the same vocab
    # that was used to compute the inverted index. How do we encode that?

    root = 'annot'
    #vocab_parent = 'annot'
    #vocab_parent = 'chip'
    #vocab_parent = 'feat'
    vocab_parent = 'featweight'
    depc = dtool.DependencyCache(
        root_tablename=root, cache_dpath=cache_dpath, use_globals=False)
    depc.register_preproc(tablename='chip', parents=['annot'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='fgmodel', parents=['chip*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='probchip', parents=['annot', 'fgmodel'], **dummy_cols)(dummy_func)
    #depc.register_preproc(tablename='probchip', parents=['annot'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='feat', parents=['chip'], **dummy_cols)(dummy_func)
    #depc.register_preproc(tablename='feat', parents=['annot'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='featweight', parents=['feat', 'probchip'], **dummy_cols)(dummy_func)

    depc.register_preproc(tablename='indexer', parents=[vocab_parent + '*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='neighbs', parents=[vocab_parent, 'indexer'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='vocab', parents=[vocab_parent + '*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='smk_vec', parents=[vocab_parent, 'vocab'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='inv_index', parents=['smk_vec*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='smk_match', parents=['smk_vec', 'inv_index'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='vsone', parents=[vocab_parent, vocab_parent], **dummy_cols)(dummy_func)

    depc.register_preproc(tablename='viewpoint_model', parents=['annot*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='viewpoint', parents=['annot', 'viewpoint_model'], **dummy_cols)(dummy_func)

    depc.register_preproc(tablename='quality_model', parents=['annot*'], **dummy_cols)(dummy_func)
    depc.register_preproc(tablename='quality', parents=['annot', 'quality_model'], **dummy_cols)(dummy_func)

    depc.initialize()
    return depc
Example #20
0
def testdata_expts(defaultdb='testdb1',
                   default_acfgstr_name_list=['default:qindex=0:10:4,dindex=0:20'],
                   default_test_cfg_name_list=['default'],
                   a=None,
                   t=None,
                   p=None,
                   qaid_override=None,
                   daid_override=None,
                   initial_aids=None,
                   use_cache=None):
    """
    Use this if you want data from an experiment.
    Command line interface to quickly get testdata for test_results.

    Command line flags can be used to specify db, aidcfg, pipecfg, qaid
    override, daid override (and maybe initial aids).


    CommandLine:
        python -m ibeis.init.main_helpers testdata_expts

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.other.dbinfo import *  # NOQA
        >>> import ibeis
        >>> ibs, testres = ibeis.testdata_expts(defaultdb='PZ_MTEST', a='timectrl:qsize=2', t='invar:AI=[False],RI=False', use_cache=False)
        >>> print('testres = %r' % (testres,))
    """
    if ut.VERBOSE:
        print('[main_helpers] testdata_expts')
    import ibeis
    from ibeis.expt import harness
    from ibeis.expt import test_result
    if a is not None:
        default_acfgstr_name_list = a
    if t is not None and p is None:
        p = t
    if p is not None:
        default_test_cfg_name_list = p

    if isinstance(default_acfgstr_name_list, six.string_types):
        default_acfgstr_name_list = [default_acfgstr_name_list]
    if isinstance(default_test_cfg_name_list, six.string_types):
        default_test_cfg_name_list = [default_test_cfg_name_list]

    #from ibeis.expt import experiment_helpers
    ibs = ibeis.opendb(defaultdb=defaultdb)
    acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=list,
                                   default=default_acfgstr_name_list)
    test_cfg_name_list = ut.get_argval(('-t', '-p'), type_=list, default=default_test_cfg_name_list)
    daid_override = ut.get_argval(('--daid-override', '--daids-override'), type_=list, default=daid_override)
    qaid_override = ut.get_argval(('--qaid', '--qaids-override', '--qaid-override'), type_=list, default=qaid_override)

    # Hack a cache here
    use_bigtest_cache3 = not ut.get_argflag(('--nocache', '--nocache-hs'))
    use_bigtest_cache3 &= ut.is_developer()
    if use_cache is not None:
        use_bigtest_cache3 &= use_cache
    use_bigtest_cache3 &= False
    #use_bigtest_cache3 = True
    if use_bigtest_cache3:
        from os.path import dirname, join
        cache_dir = ut.ensuredir(join(dirname(ut.get_module_dir(ibeis)), 'BIG_TESTLIST_CACHE3'))
        _load_testres = ut.cached_func('testreslist', cache_dir=cache_dir)(harness.run_test_configurations2)
    else:
        _load_testres = harness.run_test_configurations2
    testres_list = _load_testres(
        ibs, acfg_name_list, test_cfg_name_list, qaid_override=qaid_override,
        daid_override=daid_override, initial_aids=initial_aids,
        use_cache=use_cache)
    testres = test_result.combine_testres_list(ibs, testres_list)

    if ut.VERBOSE:
        print(testres)
    return ibs, testres
def testdata_depc3(in_memory=True):
    """
    Example of local registration

    CommandLine:
        python -m dtool.example_depcache2 testdata_depc3 --show

    Example:
        >>> # ENABLE_DOCTEST
        >>> from wbia.dtool.example_depcache2 import *  # NOQA
        >>> depc = testdata_depc3()
        >>> data = depc.get('labeler', [1, 2, 3], 'data', _debug=True)
        >>> data = depc.get('indexer', [[1, 2, 3]], 'data', _debug=True)
        >>> depc.print_all_tables()
        >>> # xdoctest: +REQUIRES(--show)
        >>> ut.quit_if_noshow()
        >>> import wbia.plottool as pt
        >>> depc.show_graph()
        >>> from wbia.plottool.interactions import ExpandableInteraction
        >>> inter = ExpandableInteraction(nCols=2)
        >>> depc['smk_match'].show_input_graph(inter)
        >>> depc['vsone'].show_input_graph(inter)
        >>> #depc['vocab'].show_input_graph(inter)
        >>> depc['neighbs'].show_input_graph(inter)
        >>> inter.start()
        >>> #depc['viewpoint_classification'].show_input_graph()
        >>> ut.show_if_requested()
    """
    from wbia import dtool

    # put the test cache in the dtool repo
    dtool_repo = dirname(ut.get_module_dir(dtool))
    cache_dpath = join(dtool_repo, 'DEPCACHE3')

    # FIXME: this only puts the sql files in memory
    default_fname = ':memory:' if in_memory else None

    root = 'annot'
    depc = dtool.DependencyCache(
        root_tablename=root,
        get_root_uuid=ut.identity,
        default_fname=default_fname,
        cache_dpath=cache_dpath,
        use_globals=False,
    )

    # ----------
    # dummy_cols = dict(colnames=['data'], coltypes=[np.ndarray])
    register_dummy_config = depc_34_helper(depc)

    register_dummy_config(tablename='labeler', parents=['annot'])
    register_dummy_config(tablename='meta_labeler', parents=['labeler'])
    register_dummy_config(tablename='indexer', parents=['annot*'])
    # register_dummy_config(tablename='neighbs', parents=['annot', 'indexer'])
    register_dummy_config(tablename='neighbs',
                          parents=['meta_labeler', 'indexer'])
    register_dummy_config(tablename='vocab', parents=['annot*'])
    # register_dummy_config(tablename='smk_vec', parents=['annot', 'vocab'], vectorized=True)
    register_dummy_config(tablename='smk_vec', parents=['annot', 'vocab'])
    # vectorized=True)
    # register_dummy_config(tablename='inv_index', parents=['smk_vec*'])
    register_dummy_config(tablename='inv_index', parents=['smk_vec*', 'vocab'])
    register_dummy_config(tablename='smk_match',
                          parents=['smk_vec', 'inv_index'])
    register_dummy_config(tablename='vsone', parents=['annot', 'annot'])
    # register_dummy_config(tablename='viewpoint_classifier', parents=['annot*'])
    # register_dummy_config(tablename='viewpoint_classification', parents=['annot', 'viewpoint_classifier'])

    depc.initialize()
    return depc
Example #22
0
def testdata_expts(
        defaultdb='testdb1',
        default_acfgstr_name_list=['default:qindex=0:10:4,dindex=0:20'],
        default_test_cfg_name_list=['default'],
        a=None,
        t=None,
        p=None,
        qaid_override=None,
        daid_override=None,
        initial_aids=None,
        use_cache=None,
        dbdir=None,
        ibs=None):
    r"""
    Use this if you want data from an experiment.
    Command line interface to quickly get testdata for test_results.

    Command line flags can be used to specify db, aidcfg, pipecfg, qaid
    override, daid override (and maybe initial aids).


    CommandLine:
        python -m ibeis.init.main_helpers testdata_expts

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.other.dbinfo import *  # NOQA
        >>> import ibeis
        >>> ibs, testres = ibeis.testdata_expts(defaultdb='pz_mtest',
        >>>                                     a='timectrl:qsize=2',
        >>>                                     t='invar:ai=[false],ri=false',
        >>>                                     use_cache=false)
        >>> print('testres = %r' % (testres,))
    """
    if ut.VERBOSE:
        print('[main_helpers] testdata_expts')
    import ibeis
    from ibeis.expt import harness
    if a is not None:
        default_acfgstr_name_list = a
    if t is not None and p is None:
        p = t
    if p is not None:
        default_test_cfg_name_list = p

    if isinstance(default_acfgstr_name_list, six.string_types):
        default_acfgstr_name_list = [default_acfgstr_name_list]
    if isinstance(default_test_cfg_name_list, six.string_types):
        default_test_cfg_name_list = [default_test_cfg_name_list]

    #from ibeis.expt import experiment_helpers
    if dbdir is not None:
        dbdir = ut.truepath(dbdir)
    if ibs is None:
        ibs = ibeis.opendb(defaultdb=defaultdb, dbdir=dbdir)
    acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'),
                                   type_=list,
                                   default=default_acfgstr_name_list)
    test_cfg_name_list = ut.get_argval(('-t', '-p'),
                                       type_=list,
                                       default=default_test_cfg_name_list)
    daid_override = ut.get_argval(('--daid-override', '--daids-override'),
                                  type_=list,
                                  default=daid_override)
    qaid_override = ut.get_argval(
        ('--qaid', '--qaids-override', '--qaid-override'),
        type_=list,
        default=qaid_override)

    # Hack a cache here
    use_bulk_cache = not ut.get_argflag(('--nocache', '--nocache-hs'))
    use_bulk_cache &= ut.is_developer()
    if use_cache is not None:
        use_bulk_cache &= use_cache
    use_bulk_cache &= False
    #use_bulk_cache = True
    if use_bulk_cache:
        from os.path import dirname
        cache_dir = ut.ensuredir(
            (dirname(ut.get_module_dir(ibeis)), 'BULK_TESTRES'))
        _cache_wrp = ut.cached_func('testreslist', cache_dir=cache_dir)
        _load_testres = _cache_wrp(harness.run_expt)
    else:
        _load_testres = harness.run_expt

    testres = _load_testres(ibs,
                            acfg_name_list,
                            test_cfg_name_list,
                            qaid_override=qaid_override,
                            daid_override=daid_override,
                            initial_aids=initial_aids,
                            use_cache=use_cache)
    #testres = test_result.combine_testres_list(ibs, testres_list)

    if ut.VERBOSE:
        print(testres)
    return ibs, testres