Beispiel #1
0
def train_identification_pz():
    r"""

    CommandLine:
        python -m ibeis_cnn.train --test-train_identification_pz

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.train import *  # NOQA
        >>> train_identification_pz()
    """
    print('get_identification_decision_training_data')
    import ibeis
    ibs = ibeis.opendb('NNP_Master3')
    base_size = 128
    #max_examples = 1001
    #max_examples = None
    max_examples = 400
    data_fpath, labels_fpath, training_dpath = ibsplugin.get_identify_training_fpaths(
        ibs, base_size=base_size, max_examples=max_examples)

    model = models.SiameseModel()
    config = dict(
        equal_batch_sizes=True,
        patience=50,
        batch_size=32,
        learning_rate=.03,
        show_confusion=False,
        run_test=None,
    )
    nets_dir = ibs.get_neuralnet_dir()
    ut.ensuredir(nets_dir)
    weights_fpath = join(training_dpath, 'ibeis_cnn_weights.pickle')
    train_harness.train(model, data_fpath, labels_fpath, weights_fpath,
                        training_dpath, **config)
Beispiel #2
0
def load_qcx2_res(ibs, qrid_list, nocache=False):
    'Prefrosm / loads all queries'
    qreq = mc3.quickly_ensure_qreq(ibs, qrids=qrid_list)
    # Build query big cache rowid
    query_rowid = qreq.get_rowid()
    hs_rowid    = ibs.get_db_name()
    qcxs_rowid  = utool.hashstr_arr(qrid_list, lbl='_qcxs')
    qres_rowid  = hs_rowid + query_rowid + qcxs_rowid
    cache_dir = join(ibs.dirs.cache_dir, 'query_results_bigcache')
    print('[rr2] load_qcx2_res(): %r' % qres_rowid)
    io_kwargs = dict(dpath=cache_dir, fname='query_results', rowid=qres_rowid, ext='.cPkl')
    # Return cache if available
    if not params.args.nocache_query and (not nocache):
        qrid2_qres = io.smart_load(**io_kwargs)
        if qrid2_qres is not None:
            print('[rr2]  *  cache hit')
            return qrid2_qres
        print('[rr2]  *  cache miss')
    else:
        print('[rr2]  *  cache off')
    # Individually load / compute queries
    if isinstance(qrid_list, list):
        qcx_set = set(qrid_list)
    else:
        qcx_set = set(qrid_list.tolist())
    qcx_max = max(qrid_list) + 1
    qrid2_qres = [ibs.query(qrid) if qrid in qcx_set else None for qrid in xrange(qcx_max)]
    # Save to the cache
    print('[rr2] Saving query_results to bigcache: %r' % qres_rowid)
    utool.ensuredir(cache_dir)
    io.smart_save(qrid2_qres, **io_kwargs)
    return qrid2_qres
Beispiel #3
0
    def get_smart_patrol_dir(ibs, ensure=True):
        """
        Args:
            ensure (bool):

        Returns:
            str smart_patrol_dpath

        CommandLine:
            python -m wbia.control.IBEISControl --test-get_smart_patrol_dir

        Example:
            >>> # ENABLE_DOCTEST
            >>> from wbia.control.IBEISControl import *  # NOQA
            >>> import wbia
            >>> # build test data
            >>> ibs = wbia.opendb('testdb1')
            >>> ensure = True
            >>> # execute function
            >>> smart_patrol_dpath = ibs.get_smart_patrol_dir(ensure)
            >>> # verify results
            >>> ut.assertpath(smart_patrol_dpath, verbose=True)
        """
        smart_patrol_dpath = join(ibs.dbdir, const.PATH_NAMES.smartpatrol)
        if ensure:
            ut.ensuredir(smart_patrol_dpath)
        return smart_patrol_dpath
Beispiel #4
0
def get_global_species_scorenorm_cachedir(ibs, species_text, ensure=True):
    """
    Args:
        species_text (str):
        ensure       (bool):

    Returns:
        str: species_cachedir

    CommandLine:
        python -m ibeis.control.IBEISControl --test-get_global_species_scorenorm_cachedir

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.control.IBEISControl import *  # NOQA
        >>> import ibeis  # NOQA
        >>> ibs = ibeis.opendb('testdb1')
        >>> species_text = ibeis.const.TEST_SPECIES.ZEB_GREVY
        >>> ensure = True
        >>> species_cachedir = ibs.get_global_species_scorenorm_cachedir(species_text, ensure)
        >>> resourcedir = ibs.get_ibeis_resource_dir()
        >>> result = ut.relpath_unix(species_cachedir, resourcedir)
        >>> print(result)
        scorenorm/zebra_grevys
    """
    scorenorm_cachedir = join(ibs.get_ibeis_resource_dir(),
                              const.PATH_NAMES.scorenormdir)
    species_cachedir = join(scorenorm_cachedir, species_text)
    if ensure:
        ut.ensurepath(scorenorm_cachedir)
        ut.ensuredir(species_cachedir)
    return species_cachedir
Beispiel #5
0
    def get_smart_patrol_dir(ibs, ensure=True):
        """
        Args:
            ensure (bool):

        Returns:
            str smart_patrol_dpath

        CommandLine:
            python -m ibeis.control.IBEISControl --test-get_smart_patrol_dir

        Example:
            >>> # ENABLE_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import ibeis
            >>> # build test data
            >>> ibs = ibeis.opendb('testdb1')
            >>> ensure = True
            >>> # execute function
            >>> smart_patrol_dpath = ibs.get_smart_patrol_dir(ensure)
            >>> # verify results
            >>> ut.assertpath(smart_patrol_dpath, verbose=True)
        """
        smart_patrol_dpath = join(ibs.dbdir, const.PATH_NAMES.smartpatrol)
        if ensure:
            ut.ensuredir(smart_patrol_dpath)
        return smart_patrol_dpath
Beispiel #6
0
def set_logdir(log_dir):
    from os.path import realpath, expanduser
    log_dir = realpath(expanduser(log_dir))
    ut.ensuredir(log_dir, verbose=True)
    ut.stop_logging()
    _ibeis_cache_write(LOGDIR_CACHEID, log_dir)
    ut.start_logging(appname=__APPNAME__)
Beispiel #7
0
def create_shortcut(r, what, where=''):
    # TODO Move to windows helpers
    print('\n\n+---- Creating Shortcut ----')
    print('What = %s\n Where=%s' % (what, where))
    run_in = ''
    what_args = ''
    if isinstance(what, tuple):
        tup = what
        what = tup[0]
        what_args = tup[1]
        run_in = tup[2]
        if run_in == ' ':
            run_in = ''
        if what_args == ' ':
            what_args = ''
    if where == '':
        target = what + '.lnk'
    else:
        import utool as ut
        ut.ensuredir(where)
        base_what = os.path.basename(what)
        if len(base_what) > 0:
            if base_what[-1] in ['"', "'"]:
                base_what = base_what[0:-1]

        target = where + '/' + base_what + '.lnk'
    helpers_vbs = r.f.create_shortcut_vbs
    cmd = 'cscript "%s" "%s" "%s" "%s" "%s"' % (helpers_vbs, target, what,
                                                what_args, run_in)
    print(cmd)
    call(cmd)
Beispiel #8
0
def get_global_distinctiveness_modeldir(ensure=True):
    # DEPRICATE
    resource_dir = get_wbia_resource_dir()
    global_distinctdir = join(resource_dir, const.PATH_NAMES.distinctdir)
    if ensure:
        ut.ensuredir(global_distinctdir)
    return global_distinctdir
Beispiel #9
0
def vizualize_vocabulary(ibs, invindex):
    """
    cleaned up version of dump_word_patches. Makes idf scatter plots and dumps
    the patches that contributed to each word.

    CommandLine:
        python -m ibeis.algo.hots.smk.smk_plots --test-vizualize_vocabulary
        python -m ibeis.algo.hots.smk.smk_plots --test-vizualize_vocabulary --vf

    Example:
        >>> from ibeis.algo.hots.smk.smk_plots import *  # NOQA
        >>> from ibeis.algo.hots.smk import smk_debug
        >>> from ibeis.algo.hots.smk import smk_repr
        >>> #tup = smk_debug.testdata_raw_internals0(db='GZ_ALL', nWords=64000)
        >>> #tup = smk_debug.testdata_raw_internals0(db='GZ_ALL', nWords=8000)
        >>> tup = smk_debug.testdata_raw_internals0(db='PZ_Master0', nWords=64000)
        >>> #tup = smk_debug.testdata_raw_internals0(db='PZ_Mothers', nWords=8000)
        >>> ibs, annots_df, daids, qaids, invindex, qreq_ = tup
        >>> smk_repr.compute_data_internals_(invindex, qreq_.qparams, delete_rawvecs=False)
        >>> vizualize_vocabulary(ibs, invindex)
    """
    invindex.idx2_wxs = np.array(invindex.idx2_wxs)

    print('[smk_plots] Vizualizing vocabulary')

    # DUMPING PART --- dumps patches to disk
    figdir = ibs.get_fig_dir()
    ut.ensuredir(figdir)
    if ut.get_argflag('--vf'):
        ut.view_directory(figdir)

    # Compute Word Statistics
    metrics = compute_word_metrics(invindex)
    wx2_nMembers, wx2_pdist_stats, wx2_wdist_stats = metrics
    #(wx2_pdist, wx2_wdist, wx2_nMembers, wx2_pdist_stats, wx2_wdist_stats) = metrics

    #wx2_prad = {wx: pdist_stats['max'] for wx, pdist_stats in six.iteritems(wx2_pdist_stats) if 'max' in pdist_stats}
    #wx2_wrad = {wx: wdist_stats['max'] for wx, wdist_stats in six.iteritems(wx2_wdist_stats) if 'max' in wdist_stats}

    wx2_prad = {wx: stats['max'] for wx, stats in wx2_pdist_stats.items() if 'max' in stats}
    wx2_wrad = {wx: stats['max'] for wx, stats in wx2_wdist_stats.items() if 'max' in stats}
    #wx2_prad = get_metric(metrics, 'wx2_pdist_stats', 'max')
    #wx2_wrad = get_metric(metrics, 'wx2_wdist_stats', 'max')

    wx_sample1 = select_by_metric(wx2_nMembers)
    wx_sample2 = select_by_metric(wx2_prad)
    wx_sample3 = select_by_metric(wx2_wrad)

    wx_sample = wx_sample1 + wx_sample2 + wx_sample3
    overlap123 = len(wx_sample) - len(set(wx_sample))
    print('overlap123 = %r' % overlap123)
    wx_sample  = set(wx_sample)
    print('len(wx_sample) = %r' % len(wx_sample))

    #make_scatterplots(ibs, figdir, invindex, metrics)

    vocabdir = join(figdir, 'vocab_patches2')
    wx2_dpath = get_word_dpaths(vocabdir, wx_sample, metrics)

    make_wordfigures(ibs, metrics, invindex, figdir, wx_sample, wx2_dpath)
Beispiel #10
0
    def get_global_species_scorenorm_cachedir(ibs, species_text, ensure=True):
        """
        Args:
            species_text (str):
            ensure       (bool):

        Returns:
            str: species_cachedir

        CommandLine:
            python -m ibeis.control.IBEISControl --test-get_global_species_scorenorm_cachedir

        Example:
            >>> # ENABLE_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import ibeis  # NOQA
            >>> ibs = ibeis.opendb('testdb1')
            >>> species_text = ibeis.const.TEST_SPECIES.ZEB_GREVY
            >>> ensure = True
            >>> species_cachedir = ibs.get_global_species_scorenorm_cachedir(species_text, ensure)
            >>> resourcedir = ibs.get_ibeis_resource_dir()
            >>> result = ut.relpath_unix(species_cachedir, resourcedir)
            >>> print(result)
            scorenorm/zebra_grevys
        """
        scorenorm_cachedir = join(ibs.get_ibeis_resource_dir(),
                                  const.PATH_NAMES.scorenormdir)
        species_cachedir = join(scorenorm_cachedir, species_text)
        if ensure:
            ut.ensurepath(scorenorm_cachedir)
            ut.ensuredir(species_cachedir)
        return species_cachedir
Beispiel #11
0
def get_numpy_dataset(data_fpath, labels_fpath, training_dpath):
    """
    """
    import numpy as np
    # hack for caching num_labels
    data = np.load(data_fpath)
    data_shape = data.shape[1:]
    labels = np.load(labels_fpath)
    num_labels = len(labels)

    alias_key = 'temp'
    ut.ensuredir(training_dpath)

    dataset = DataSet.new_training_set(
        alias_key=alias_key,
        data_fpath=data_fpath,
        labels_fpath=labels_fpath,
        metadata_fpath=None,
        training_dpath=training_dpath,
        data_shape=data_shape,
        data_per_label=1,
        output_dims=1,
        num_labels=num_labels,
    )
    return dataset
Beispiel #12
0
    def _setup_links(self, cfg_prefix, config=None):
        """
        Called only when setting up an experiment to make a measurement.

        Creates symlinks such that all data is written to a directory that
        depends on a computer name, cfg_prefix and an arbitrary configuration
        dict.

        Then force the link in the basic directory to point to abs_dpath.
        """
        # Setup directory
        from os.path import expanduser
        assert self.dname is not None

        computer_id = ut.get_argval('--comp', default=ut.get_computer_name())

        conf_dpath = ut.ensuredir((expanduser(self.base_dpath), 'configured'))
        comp_dpath = ut.ensuredir((join(conf_dpath, computer_id)))

        link_dpath = ut.ensuredir((self.base_dpath, 'link'))

        # if True:
        #     # move to new system
        #     old_dpath = join(conf_dpath, self.dbname + '_' + computer_id)
        #     if exists(old_dpath):
        #         ut.move(old_dpath, join(comp_dpath, self.dbname))

        try:
            cfgstr = ut.repr3(config.getstate_todict_recursive())
        except AttributeError:
            cfgstr = ut.repr3(config)

        hashid = ut.hash_data(cfgstr)[0:6]
        suffix = '_'.join([cfg_prefix, hashid])
        dbcode = self.dbname + '_' + suffix

        abs_dpath = ut.ensuredir(join(comp_dpath, dbcode))

        self.dname = dbcode
        self.dpath = abs_dpath
        self.abs_dpath = abs_dpath

        # Place a basic link in the base link directory
        links = []
        links.append(expanduser(join(link_dpath, self.dbname)))
        # # Make a configured but computer agnostic link
        # links.append(expanduser(join(conf_dpath, self.dbname)))

        for link in links:
            try:
                # Overwrite any existing link so the most recently used is
                # the default
                self.link = ut.symlink(abs_dpath, link, overwrite=True)
            except Exception:
                if exists(abs_dpath):
                    newpath = ut.non_existing_path(abs_dpath, suffix='_old')
                    ut.move(link, newpath)
                    self.link = ut.symlink(abs_dpath, link)

        ut.writeto(join(abs_dpath, 'info.txt'), cfgstr)
Beispiel #13
0
def visualize_bounding_boxes(
    ibs,
    config,
    version,
    gid_list=None,
    randomize=False,
    num_images=10,
    t_width=500,
    output_path=None,
):
    if gid_list is None:
        gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **config)
    else:
        num_images = None

    if randomize:
        random.shuffle(gid_list)

    if num_images not in [-1, None]:
        num_images = min(num_images, len(gid_list))
        gid_list = gid_list[:num_images]

    uuid_list = ibs.get_image_uuids(gid_list)

    assert version is not None
    version = version.lower()
    if version == 'prediction':
        logger.info('\tGather Predictions')
        val_dict = localizer_parse_pred(ibs, test_gid_list=gid_list, **config)
    elif version == 'ground_truth':
        logger.info('\tGather Ground-Truth')
        val_dict = general_parse_gt(ibs, test_gid_list=gid_list, **config)

    if output_path is None:
        output_path = abspath(expanduser(join('~', 'Desktop', 'bboxes')))
        ut.ensuredir(output_path)

    filepath_dict = {}
    for gid, image_uuid in zip(gid_list, uuid_list):
        image = ibs.get_images(gid)
        image = _resize(image, t_width=t_width)
        h, w, c = image.shape

        val_list = val_dict[image_uuid]

        for val in val_list:
            xbr = int(np.around(val['xbr'] * w))
            ybr = int(np.around(val['ybr'] * h))
            xtl = int(np.around(val['xtl'] * w))
            ytl = int(np.around(val['ytl'] * h))
            cv2.rectangle(image, (xtl, ytl), (xbr, ybr), (0, 140, 255), 4)

        write_filename = 'bboxes_%d_%s.png' % (gid, version)
        write_filepath = join(output_path, write_filename)
        logger.info(write_filepath)
        cv2.imwrite(write_filepath, image)

        filepath_dict[gid] = write_filepath

    return filepath_dict
Beispiel #14
0
def get_annot_probchip_fpath_list(ibs, aid_list, config2_=None, species=None):
    """ Build probability chip file paths based on the current IBEIS configuration

    Args:
        ibs (IBEISController):
        aid_list (list):
        suffix (None):

    Returns:
        probchip_fpath_list

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.algo.preproc.preproc_probchip import *  # NOQA
        >>> from os.path import basename
        >>> ibs, aid_list = preproc_chip.testdata_ibeis()
        >>> config2_ = ibs.new_query_params(dict(fg_on=False))
        >>> probchip_fpath_list = get_annot_probchip_fpath_list(ibs, aid_list, config2_=config2_)
        >>> result = ut.relpath_unix(probchip_fpath_list[1], ibs.get_dbdir())
        >>> print(result)
        _ibsdb/_ibeis_cache/prob_chips/probchip_avuuid=5a1a53ba-fd44-b113-7f8c-fcf248d7047f_CHIP(sz450)_FEATWEIGHT(OFF).png

        _ibsdb/_ibeis_cache/prob_chips/probchip_avuuid=5a1a53ba-fd44-b113-7f8c-fcf248d7047f_CHIP(sz450)_FEATWEIGHT(ON,uselabel,rf).png

    probchip_aid=5_bbox=(0,0,1072,804)_theta=0.0tau_gid=5_CHIP(sz450)_FEATWEIGHT(ON,uselabel,rf)_CHIP().png
    """
    ibs.probchipdir = ibs.get_probchip_dir()
    cachedir = ibs.get_probchip_dir()
    ut.ensuredir(cachedir)
    probchip_fname_fmt = get_probchip_fname_fmt(ibs, config2_=config2_, species=species)
    annot_visual_uuid_list  = ibs.get_annot_visual_uuids(aid_list)
    probchip_fpath_list = [ut.unixjoin(cachedir, probchip_fname_fmt.format(avuuid=avuuid))
                           for avuuid in annot_visual_uuid_list]
    return probchip_fpath_list
Beispiel #15
0
def get_flann_fpath(dpts,
                    cache_dir='default',
                    cfgstr='',
                    flann_params={},
                    use_params_hash=True,
                    use_data_hash=True,
                    appname='vtool',
                    verbose=True):
    """ returns filepath for flann index """
    if cache_dir == 'default':
        if verbose:
            print('[flann] using default cache dir')
        cache_dir = utool.get_app_resource_dir(appname)
        utool.ensuredir(cache_dir)
    flann_cfgstr = get_flann_cfgstr(dpts,
                                    flann_params,
                                    cfgstr,
                                    use_params_hash=use_params_hash,
                                    use_data_hash=use_data_hash)
    if verbose:
        print('...flann_cache cfgstr = %r: ' % flann_cfgstr)
    # Append any user labels
    flann_fname = 'flann_index' + flann_cfgstr + '.flann'
    flann_fpath = normpath(join(cache_dir, flann_fname))
    return flann_fpath
Beispiel #16
0
def TEST_SQL_NAMES():
    # -------- INIT DATABASE ------------
    #
    # Create new temp database
    sqldb_fname = 'temp_test_sql_names.sqlite3'
    sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
    utool.ensuredir(sqldb_dpath)
    print('Remove Old Temp Database')
    utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
    print('New Temp Database')
    db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
                                                  sqldb_fname=sqldb_fname)
    #
    # Define the schema
    __define_schema(db)
    #
    # -------- RUN INSERTS --------------
    print('[TEST] --- INSERT NAMES --- ')
    test_names = [
        'fred',
        'sue',
        'Robert\');DROP TABLE Students;--',
        'joe',
        'rob',
    ]
    __insert_names(db, test_names)
    __insert_names(db, test_names[2:4])
    #
    # -------- RUN SELECT NAMES --------------
    print('[TEST] --- SELECT NAMES ---')
    name_text_results = db.executeone('SELECT name_text FROM names', [])
    print(' * name_text_results=%r' % name_text_results)
    #assert name_text_results == test_names, 'unexpected results from select names'
    #
    # -------- RUN SELECT NIDS --------------
    print('[TEST] --- SELECT NIDS ---')
    query_names = test_names[::2] + ['missingno']
    nid_list = db.executemany(
        operation='''
               SELECT name_rowid
               FROM names
               WHERE name_text=?
               ''',
        params_iter=((name,) for name in query_names))

    # Get the parameter indexes that failed
    failx_list = [count for count, nid in enumerate(nid_list) if nid is None]
    assert failx_list == [3]
    failed_names = [query_names[failx] for failx in failx_list]  # NOQA
    utool.printvar2('failed_names')

    # We selected a name not in the table.
    # Its return index is an empty list
    print('[TEST] nid_list=%r' % nid_list)
    print('[TEST] query_names=%r' % query_names)
    print('[TEST] test_names=%r' % test_names)
    # SQL INTEGERS START AT 1 APPARENTLY
    #expected_names = [test_names[nid - 1] for nid in nid_list]
    #assert expected_names == query_names, 'unexpected results from select names'
    return locals()
Beispiel #17
0
 def new_database(back, new_dbdir=None):
     """ File -> New Database"""
     if new_dbdir is None:
         new_dbname = back.user_input(
             msg='What do you want to name the new database?',
             title='New Database')
         if new_dbname is None or len(new_dbname) == 0:
             print('Abort new database. new_dbname=%r' % new_dbname)
             return
         reply = back.user_option(
             msg='Where should I put the new database?',
             title='Import Images',
             options=['Choose Directory', 'My Work Dir'],
             use_cache=False)
         if reply == 'Choose Directory':
             print('[back] new_database(): SELECT A DIRECTORY')
             putdir = guitool.select_directory('Select new database directory')
         elif reply == 'My Work Dir':
             putdir = back.get_work_directory()
         else:
             print('Abort new database')
             return
         new_dbdir = join(putdir, new_dbname)
         if not exists(putdir):
             raise ValueError('Directory %r does not exist.' % putdir)
         if exists(new_dbdir):
             raise ValueError('New DB %r already exists.' % new_dbdir)
     utool.ensuredir(new_dbdir)
     print('[back] new_database(new_dbdir=%r)' % new_dbdir)
     back.open_database(dbdir=new_dbdir)
Beispiel #18
0
def try_download_baseline_ibeis_normalizer(ibs, qreq_):
    """
    tries to download a baseline normalizer for some species.
    creates an empty normalizer if it cannot
    """
    baseline_url_dict = {
        # TODO: Populate
    }
    species_text = '_'.join(qreq_.get_unique_species())  # HACK
    query_cfgstr = qreq_.qparams.query_cfgstr
    cachedir = qreq_.ibs.get_global_species_scorenorm_cachedir(species_text)
    key = species_text + query_cfgstr
    baseline_url = baseline_url_dict.get(key, None)
    if baseline_url is not None:
        try:
            cachedir = qreq_.ibs.get_global_species_scorenorm_cachedir(species_text)
            baseline_cachedir = join(cachedir, 'baseline')
            ut.ensuredir(baseline_cachedir)
            encoder = vt.ScoreNormalizer(cfgstr=query_cfgstr, prefix=species_text)
            encoder.load(baseline_cachedir)
        except Exception:
            encoder = None
    else:
        encoder = None
    if encoder is None:
        if False and ut.is_developer(['hyrule']):
            # train new encoder. only do this on hyrule
            print('Baseline does not exist and cannot be downlaoded. Training baseline')
            encoder = train_baseline_ibeis_normalizer(qreq_.ibs)
        else:
            # return empty score encoder
            encoder = vt.ScoreNormalizer(cfgstr=query_cfgstr, prefix=species_text)
            print('returning empty encoder')
            #raise NotImplementedError('return the nodata noramlizer with 1/2 default')
    return encoder
Beispiel #19
0
def set_logdir(log_dir):
    from os.path import realpath, expanduser
    log_dir = realpath(expanduser(log_dir))
    ut.ensuredir(log_dir, verbose=True)
    ut.stop_logging()
    _ibeis_cache_write(LOGDIR_CACHEID, log_dir)
    ut.start_logging(appname=__APPNAME__)
Beispiel #20
0
def TEST_SQL_NAMES():
    # -------- INIT DATABASE ------------
    #
    # Create new temp database
    sqldb_fname = 'temp_test_sql_names.sqlite3'
    sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
    utool.ensuredir(sqldb_dpath)
    print('Remove Old Temp Database')
    utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
    print('New Temp Database')
    db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
                                                  sqldb_fname=sqldb_fname)
    #
    # Define the schema
    __define_schema(db)
    #
    # -------- RUN INSERTS --------------
    print('[TEST] --- INSERT NAMES --- ')
    test_names = [
        'fred',
        'sue',
        'Robert\');DROP TABLE names;--',
        'joe',
        'rob',
    ]
    __insert_names(db, test_names)
    __insert_names(db, test_names[2:3])
    #
    # -------- RUN SELECT NAMES --------------
    print('[TEST] --- SELECT NAMES ---')
    name_text_results = db.executeone('SELECT name_text FROM names', [])
    print(' * name_text_results=%r' % name_text_results)
    #assert name_text_results == test_names, 'unexpected results from select names'

    return locals()
Beispiel #21
0
def copy_ibeisdb(source_dbdir, dest_dbdir):
    # TODO: rectify with rsync, script, and merge script.
    from os.path import normpath
    import ibeis
    exclude_dirs_ = (ibeis.const.EXCLUDE_COPY_REL_DIRS +
                     ['_hsdb', '.hs_internals'])
    exclude_dirs = [ut.ensure_unixslash(normpath(rel))
                    for rel in exclude_dirs_]

    rel_tocopy = ut.glob(source_dbdir, '*', exclude_dirs=exclude_dirs,
                         recursive=True, with_files=True, with_dirs=False,
                         fullpath=False)
    rel_tocopy_dirs = ut.glob(source_dbdir, '*', exclude_dirs=exclude_dirs,
                              recursive=True, with_files=False, with_dirs=True,
                              fullpath=False)

    src_list = [join(source_dbdir, relpath) for relpath in rel_tocopy]
    dst_list = [join(dest_dbdir, relpath) for relpath in rel_tocopy]

    # ensure directories exist
    rel_tocopy_dirs = [dest_dbdir] + [join(dest_dbdir, dpath_)
                                      for dpath_ in rel_tocopy_dirs]
    for dpath in rel_tocopy_dirs:
        ut.ensuredir(dpath)
    # copy files
    ut.copy(src_list, dst_list)
Beispiel #22
0
def TEST_SQL_NAMES():
    # -------- INIT DATABASE ------------
    #
    # Create new temp database
    sqldb_fname = 'temp_test_sql_names.sqlite3'
    sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
    utool.ensuredir(sqldb_dpath)
    print('Remove Old Temp Database')
    utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
    print('New Temp Database')
    db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
                                                  sqldb_fname=sqldb_fname)
    #
    # Define the schema
    __define_schema(db)
    #
    # -------- RUN INSERTS --------------
    print('[TEST] --- INSERT NAMES --- ')
    test_names = [
        'fred',
        'sue',
        'Robert\');DROP TABLE Students;--',
        'joe',
        'rob',
    ]
    __insert_names(db, test_names)
    __insert_names(db, test_names[2:4])
    #
    # -------- RUN SELECT NAMES --------------
    print('[TEST] --- SELECT NAMES ---')
    name_text_results = db.executeone('SELECT name_text FROM names', [])
    print(' * name_text_results=%r' % name_text_results)
    #assert name_text_results == test_names, 'unexpected results from select names'
    #
    # -------- RUN SELECT NIDS --------------
    print('[TEST] --- SELECT NIDS ---')
    query_names = test_names[::2] + ['missingno']
    nid_list = db.executemany(operation='''
               SELECT name_rowid
               FROM names
               WHERE name_text=?
               ''',
                              params_iter=((name, ) for name in query_names))

    # Get the parameter indexes that failed
    failx_list = [count for count, nid in enumerate(nid_list) if nid is None]
    assert failx_list == [3]
    failed_names = [query_names[failx] for failx in failx_list]  # NOQA
    utool.printvar2('failed_names')

    # We selected a name not in the table.
    # Its return index is an empty list
    print('[TEST] nid_list=%r' % nid_list)
    print('[TEST] query_names=%r' % query_names)
    print('[TEST] test_names=%r' % test_names)
    # SQL INTEGERS START AT 1 APPARENTLY
    #expected_names = [test_names[nid - 1] for nid in nid_list]
    #assert expected_names == query_names, 'unexpected results from select names'
    return locals()
Beispiel #23
0
    def train(dark, voc_path, weight_path, **kwargs):
        """
            Train a new forest with the given positive chips and negative chips.

            Args:
                train_pos_chip_path_list (list of str): list of positive training chips
                train_neg_chip_path_list (list of str): list of negative training chips
                trees_path (str): string path of where the newly trained trees are to be saved

            Kwargs:
                chips_norm_width (int, optional): Chip normalization width for resizing;
                    the chip is resized to have a width of chips_norm_width and
                    whatever resulting height in order to best match the original
                    aspect ratio; defaults to 128

                    If both chips_norm_width and chips_norm_height are specified,
                    the original aspect ratio of the chip is not respected
                chips_norm_height (int, optional): Chip normalization height for resizing;
                    the chip is resized to have a height of chips_norm_height and
                    whatever resulting width in order to best match the original
                    aspect ratio; defaults to None

                    If both chips_norm_width and chips_norm_height are specified,
                    the original aspect ratio of the chip is not respected
                verbose (bool, optional): verbose flag; defaults to object's verbose or
                    selectively enabled for this function

            Returns:
                None
        """
        # Default values
        params = odict(
            [
                ("weight_filepath", None),  # This value always gets overwritten
                ("verbose", dark.verbose),
                ("quiet", dark.quiet),
            ]
        )
        # params.update(kwargs)
        ut.update_existing(params, kwargs)

        # Make the tree path absolute
        weight_path = abspath(weight_path)
        ut.ensuredir(weight_path)

        # Setup training files and folder structures
        results = dark._train_setup(voc_path, weight_path)
        manifest_filename, num_images, config_filepath, class_filepath = results

        # Run training algorithm
        params_list = [dark.net, manifest_filename, weight_path, num_images] + list(params.values())
        DARKNET_CLIB.train(*params_list)
        weight_filepath = params["weight_filepath"]

        if not params["quiet"]:
            print("\n\n[pydarknet py] *************************************")
            print("[pydarknet py] Training Completed")
            print("[pydarknet py] Weight file saved to: %s" % (weight_filepath,))
        return weight_filepath, config_filepath, class_filepath
Beispiel #24
0
def labeler_train(ibs,
                  species_list=None,
                  species_mapping=None,
                  viewpoint_mapping=None,
                  ensembles=3,
                  **kwargs):
    from ibeis_cnn.ingest_ibeis import get_cnn_labeler_training_images_pytorch
    from ibeis.algo.detect import densenet

    species = '-'.join(species_list)
    args = (species, )
    data_path = join(ibs.get_cachedir(), 'extracted-labeler-%s' % args)
    extracted_path = get_cnn_labeler_training_images_pytorch(
        ibs,
        category_list=species_list,
        category_mapping=species_mapping,
        viewpoint_mapping=viewpoint_mapping,
        dest_path=data_path,
        **kwargs)

    weights_path_list = []
    for ensemble_num in range(ensembles):
        args = (
            species,
            ensemble_num,
        )
        output_path = join(ibs.get_cachedir(), 'training',
                           'labeler-%s-ensemble-%d' % args)
        if exists(output_path):
            ut.delete(output_path)
        weights_path = densenet.train(extracted_path,
                                      output_path,
                                      blur=False,
                                      flip=False)
        weights_path_list.append(weights_path)

    args = (species, )
    output_name = 'labeler.%s' % args
    ensemble_path = join(ibs.get_cachedir(), 'training', output_name)
    ut.ensuredir(ensemble_path)

    archive_path = '%s.zip' % (ensemble_path)
    ensemble_weights_path_list = []

    for index, weights_path in enumerate(sorted(weights_path_list)):
        assert exists(weights_path)
        ensemble_weights_path = join(ensemble_path,
                                     'labeler.%d.weights' % (index, ))
        ut.copy(weights_path, ensemble_weights_path)
        ensemble_weights_path_list.append(ensemble_weights_path)

    ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list
    ut.archive_files(archive_path,
                     ensemble_weights_path_list,
                     overwrite=True,
                     common_prefix=True)

    return archive_path
Beispiel #25
0
def db_to_dbdir(db, allow_newdir=False, extra_workdirs=[], use_sync=False):
    """ Implicitly gets dbdir. Searches for db inside of workdir """
    work_dir = get_workdir()
    dbalias_dict = get_dbalias_dict()

    workdir_list = []
    for extra_dir in extra_workdirs:
        if exists(extra_dir):
            workdir_list.append(extra_dir)
    if use_sync:
        sync_dir = join(work_dir, '../sync')
        if exists(sync_dir):
            workdir_list.append(sync_dir)
    workdir_list.append(work_dir)  # TODO: Allow multiple workdirs

    # Check all of your work directories for the database
    for _dir in workdir_list:
        dbdir = realpath(join(_dir, db))
        # Use db aliases
        if not exists(dbdir) and db.upper() in dbalias_dict:
            dbdir = join(_dir, dbalias_dict[db.upper()])
        if exists(dbdir):
            break

    # Create the database if newdbs are allowed in the workdir
    #print('allow_newdir=%r' % allow_newdir)
    if allow_newdir:
        utool.ensuredir(dbdir, verbose=True)

    # Complain if the implicit dbdir does not exist
    if not exists(dbdir):
        print('!!!')
        print('[sysres] WARNING: db=%r not found in work_dir=%r' %
              (db, work_dir))
        fname_list = os.listdir(work_dir)
        lower_list = [fname.lower() for fname in fname_list]
        index = util_list.listfind(lower_list, db.lower())
        if index is not None:
            print('[sysres] WARNING: db capitalization seems to be off')
            if not utool.STRICT:
                print('[sysres] attempting to fix it')
                db = fname_list[index]
                dbdir = join(work_dir, db)
                print('[sysres] dbdir=%r' % dbdir)
                print('[sysres] db=%r' % db)
        if not exists(dbdir):
            msg = '[sysres!] ERROR: Database does not exist'
            print('<!!!>')
            print(msg)
            print('[sysres!] Here is a list of valid dbs: ' +
                  utool.indentjoin(fname_list, '\n  * '))
            print('[sysres!] dbdir=%r' % dbdir)
            print('[sysres!] db=%r' % db)
            print('[sysres!] work_dir=%r' % work_dir)
            print('</!!!>')
            raise AssertionError(msg)
        print('!!!')
    return dbdir
Beispiel #26
0
def TEST_SQL_NUMPY():
    sqldb_fname = 'temp_test_sql_numpy.sqlite3'
    sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
    utool.ensuredir(sqldb_dpath)
    utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
    db = sqldbc.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
                                      sqldb_fname=sqldb_fname)

    db.add_table('temp', [
        ('temp_id', 'INTEGER PRIMARY KEY'),
        ('temp_hash', 'NUMPY'),
    ])

    tt = utool.tic()
    feats_list = grab_numpy_testdata(shape=(3e3, 128), dtype=np.uint8)
    print(' * numpy.new time=%r sec' % utool.toc(tt))

    print('[TEST] insert numpy arrays')
    tt = utool.tic()
    feats_iter = ((feats, ) for feats in feats_list)
    db.executemany(operation='''
        INSERT
        INTO temp
        (
            temp_hash
        )
        VALUES (?)
        ''',
                   params_iter=feats_iter)
    print(' * execute insert time=%r sec' % utool.toc(tt))

    print('[TEST] save sql database')
    tt = utool.tic()
    #db.cur.commit()
    db.connection.commit()
    print(' * commit time=%r sec' % utool.toc(tt))

    print('[TEST] read from sql database')

    tt = utool.tic()
    db.cur.execute('SELECT temp_hash FROM temp', [])
    print(' * execute select time=%r sec' % utool.toc(tt))

    tt = utool.tic()
    result_list = _results_gen(db.cur)
    print(' * iter results time=%r sec' % utool.toc(tt))
    print(' * memory(result_list) = %s' %
          utool.byte_str2(utool.get_object_size(result_list)))
    del result_list
    #print('[TEST] result_list=%r' % result_list)

    print('[TEST] dump sql database')
    tt = utool.tic()
    db.dump('temp.dump.txt')
    print(' * dump time=%r sec' % utool.toc(tt))
    #with open('temp.dump.txt') as file_:
    #    print(file_.read())
    return locals()
Beispiel #27
0
def ingest_standard_database(db, force_delete=False):
    print('[ingest] Ingest Standard Database: db=%r' % (db,))
    ingestable = get_standard_ingestable(db)
    dbdir = ibeis.sysres.db_to_dbdir(ingestable.db, allow_newdir=True, use_sync=False)
    utool.ensuredir(dbdir, verbose=True)
    if force_delete:
        ibsfuncs.delete_ibeis_database(dbdir)
    ibs = IBEISControl.IBEISController(dbdir)
    ingest_rawdata(ibs, ingestable)
Beispiel #28
0
def _make_empty_controller():
    print('make_empty_controller')
    sqldb_fname = 'temp_test_sql_control.sqlite3'
    sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
    utool.ensuredir(sqldb_dpath)
    utool.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
    db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
                                                  sqldb_fname=sqldb_fname)
    return db
Beispiel #29
0
def get_local_species_scorenorm_cachedir(ibs, species_text, ensure=True):
    """"""
    scorenorm_cachedir = join(ibs.get_cachedir(),
                              const.PATH_NAMES.scorenormdir)
    species_cachedir = join(scorenorm_cachedir, species_text)
    if ensure:
        ut.ensuredir(scorenorm_cachedir)
        ut.ensuredir(species_cachedir)
    return species_cachedir
Beispiel #30
0
def canonical_classifier_train(ibs,
                               species,
                               ensembles=3,
                               extracted_path=None,
                               **kwargs):
    from wbia.other.detectexport import (
        get_cnn_classifier_canonical_training_images_pytorch, )
    from wbia.algo.detect import densenet

    args = (species, )
    data_path = join(ibs.get_cachedir(),
                     'extracted-classifier-canonical-%s' % args)
    if extracted_path is None:
        extracted_path = get_cnn_classifier_canonical_training_images_pytorch(
            ibs,
            species,
            dest_path=data_path,
        )

    weights_path_list = []
    for ensemble_num in range(ensembles):
        args = (
            species,
            ensemble_num,
        )
        output_path = join(ibs.get_cachedir(), 'training',
                           'classifier-canonical-%s-ensemble-%d' % args)
        if exists(output_path):
            ut.delete(output_path)
        weights_path = densenet.train(extracted_path,
                                      output_path,
                                      blur=False,
                                      flip=False)
        weights_path_list.append(weights_path)

    args = (species, )
    output_name = 'classifier.canonical.%s' % args
    ensemble_path = join(ibs.get_cachedir(), 'training', output_name)
    ut.ensuredir(ensemble_path)

    archive_path = '%s.zip' % (ensemble_path)
    ensemble_weights_path_list = []

    for index, weights_path in enumerate(sorted(weights_path_list)):
        assert exists(weights_path)
        ensemble_weights_path = join(
            ensemble_path, 'classifier.canonical.%d.weights' % (index, ))
        ut.copy(weights_path, ensemble_weights_path)
        ensemble_weights_path_list.append(ensemble_weights_path)

    ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list
    ut.archive_files(archive_path,
                     ensemble_weights_path_list,
                     overwrite=True,
                     common_prefix=True)

    return archive_path
Beispiel #31
0
def localizer_train(ibs, species_list=None, **kwargs):
    from pydarknet import Darknet_YOLO_Detector
    data_path = ibs.export_to_xml(species_list=species_list, **kwargs)
    output_path = join(ibs.get_cachedir(), 'training', 'localizer')
    ut.ensuredir(output_path)
    dark = Darknet_YOLO_Detector()
    model_path = dark.train(data_path, output_path)
    del dark
    return model_path
Beispiel #32
0
def _make_empty_controller():
    print('make_empty_controller')
    sqldb_fname = 'temp_test_sql_control.sqlite3'
    sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
    utool.ensuredir(sqldb_dpath)
    utool.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
    db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
                                                  sqldb_fname=sqldb_fname)
    return db
Beispiel #33
0
 def dump_tables_to_csv(db):
     """ Convenience: Dumps all csv database files to disk """
     dump_dir = join(db.dir_, 'CSV_DUMP')
     utool.ensuredir(dump_dir)
     for tablename in six.iterkeys(db.table_columns):
         table_fname = tablename + '.csv'
         table_csv = db.get_table_csv(tablename)
         with open(join(dump_dir, table_fname), 'w') as file_:
             file_.write(table_csv)
Beispiel #34
0
def TEST_SQL_NUMPY():
    sqldb_fname = 'temp_test_sql_numpy.sqlite3'
    sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
    utool.ensuredir(sqldb_dpath)
    utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
    db = sqldbc.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
                                      sqldb_fname=sqldb_fname)

    db.schema('temp',    [
        ('temp_id',      'INTEGER PRIMARY KEY'),
        ('temp_hash',    'NUMPY'),
    ])

    tt = utool.tic()
    feats_list = grab_numpy_testdata(shape=(3e3, 128), dtype=np.uint8)
    print(' * numpy.new time=%r sec' % utool.toc(tt))

    print('[TEST] insert numpy arrays')
    tt = utool.tic()
    feats_iter = ((feats, ) for feats in feats_list)
    db.executemany(operation='''
        INSERT
        INTO temp
        (
            temp_hash
        )
        VALUES (?)
        ''', params_iter=feats_iter)
    print(' * execute insert time=%r sec' % utool.toc(tt))

    print('[TEST] save sql database')
    tt = utool.tic()
    #db.cur.commit()
    db.connection.commit()
    print(' * commit time=%r sec' % utool.toc(tt))

    print('[TEST] read from sql database')

    tt = utool.tic()
    db.cur.execute('SELECT temp_hash FROM temp', [])
    print(' * execute select time=%r sec' % utool.toc(tt))

    tt = utool.tic()
    result_list = _results_gen(db.cur)
    print(' * iter results time=%r sec' % utool.toc(tt))
    print(' * memory(result_list) = %s' % utool.byte_str2(utool.get_object_size(result_list)))
    del result_list
    #print('[TEST] result_list=%r' % result_list)

    print('[TEST] dump sql database')
    tt = utool.tic()
    db.dump('temp.dump.txt')
    print(' * dump time=%r sec' % utool.toc(tt))
    #with open('temp.dump.txt') as file_:
    #    print(file_.read())
    return locals()
Beispiel #35
0
def test_akmeans(full_test=False,
                 plot_test=False,
                 num_pca_dims=2,
                 data_dim=2,
                 nump=1000):
    import numpy as np
    from vtool import clustering
    nump = nump
    dims = data_dim  # 128
    dtype = np.uint8
    print('Make %d random %d-dimensional %s points.' % (nump, dims, dtype))
    # Seed for a determenistic test
    np.random.seed(42)
    data = np.array(np.random.randint(0, 255, (nump, dims)), dtype=dtype)

    num_clusters = 10
    max_iters = 2
    ave_unchanged_thresh = 0
    ave_unchanged_iterwin = 10
    flann_params = {}

    cache_dir = utool.get_app_resource_dir('vtool', 'test_cache')
    utool.ensuredir(cache_dir)

    # Test precomputing
    dx2_label, centers = clustering.precompute_akmeans(data,
                                                       num_clusters,
                                                       max_iters=max_iters,
                                                       cache_dir=cache_dir)
    # internal names
    datax2_clusterx, centroids = dx2_label, centers

    if plot_test:
        clustering.plot_clusters(data,
                                 datax2_clusterx,
                                 centroids,
                                 num_pca_dims=num_pca_dims)

    assert centers.shape == (num_clusters, dims), 'sanity check'
    assert dx2_label.shape == (nump, ), 'sanity check'

    # Test regular computing
    if full_test:
        dx2_label, centers = clustering.akmeans(data,
                                                num_clusters,
                                                max_iters=max_iters)
        assert centers.shape == (num_clusters, dims), 'sanity check'
        assert dx2_label.shape == (nump, ), 'sanity check'

    if False:
        # other test (development)
        import pyflann
        flann_lib_inst = pyflann.flann
        flann_class_inst = pyflann.FLANN()
        flann_class_inst.build_index(data)
    return locals()
Beispiel #36
0
def export_ibeis_to_hotspotter(ibs):
    print('export to hsdb')
    ibs.inject_func(get_hsdb_image_gpaths)
    hsdb_dir = ibs.get_dbdir()
    internal_hsdb = join(hsdb_dir, '_hsdb')
    utool.ensuredir(internal_hsdb)

    # Build Image Table
    gid_list        = ibs.get_valid_gids()
    gpath_list      = ibs.get_hsdb_image_gpaths(gid_list)
    reviewed_list        = ibs.get_image_reviewed(gid_list)
    # aif in hotspotter is equivilant to reviewed in IBEIS
    image_table_csv = utool.make_csv_table(
        [gid_list, gpath_list, reviewed_list],
        ['gid', 'gname', 'aif'],
        '# image table')

    # Build Name Table
    nid_list       =  ibs.get_valid_nids()
    name_list      = ibs.get_name_text(nid_list)
    name_table_csv = utool.make_csv_table(
        [nid_list, name_list],
        ['nid', 'name'],
        '# name table')

    # Build Chip Table
    aid_list        = ibs.get_valid_aids()
    annotationgid_list     = ibs.get_annot_gids(aid_list)
    annotationnid_list     = ibs.get_annot_nids(aid_list)
    bbox_list       = map(list, ibs.get_annot_bboxes(aid_list))
    theta_list      = ibs.get_annot_thetas(aid_list)
    notes_list      = ibs.get_annot_notes(aid_list)

    chip_column_list = [aid_list, annotationgid_list, annotationnid_list, bbox_list, theta_list, notes_list]
    chip_column_lbls = ['cid', 'gid', 'nid', '[tlx tly w h]', 'theta', 'notes']
    chip_column_types = [int, int, int, list, float, str]
    chip_table_csv = utool.make_csv_table(
        chip_column_list,
        chip_column_lbls,
        '# chip table', chip_column_types)

    if utool.VERBOSE:
        if len(aid_list) < 87:
            print(chip_table_csv)
        if len(nid_list) < 87:
            print(name_table_csv)
        if len(gid_list) < 87:
            print(image_table_csv)

    # Write Tables
    with open(join(internal_hsdb, 'HSDB_image_table.csv'), 'wb') as imgtbl_file:
        imgtbl_file.write(image_table_csv)
    with open(join(internal_hsdb, 'HSDB_name_table.csv'), 'wb') as nametbl_file:
        nametbl_file.write(name_table_csv)
    with open(join(internal_hsdb, 'HSDB_chip_table.csv'), 'wb') as chiptbl_file:
        chiptbl_file.write(chip_table_csv)
Beispiel #37
0
def imwrite_theano_symbolic_graph(thean_expr):
    import theano
    graph_dpath = '.'
    graph_fname = 'symbolic_graph.png'
    graph_fpath = ut.unixjoin(graph_dpath, graph_fname)
    ut.ensuredir(graph_dpath)
    theano.printing.pydotprint(
        thean_expr, outfile=graph_fpath, var_with_name_simple=True)
    ut.startfile(graph_fpath)
    return graph_fpath
Beispiel #38
0
def get_word_dpaths(vocabdir, wx_sample, metrics):
    """
    Gets word folder names and ensure they exist
    """
    ut.ensuredir(vocabdir)
    wx2_dpath = {wx: join(vocabdir, get_word_dname(wx, metrics)) for wx in wx_sample}
    iter_ = ut.progiter(six.itervalues(wx2_dpath), lbl='Ensuring word_dpath: ', freq=200)
    for dpath in iter_:
        ut.ensuredir(dpath)
    return wx2_dpath
Beispiel #39
0
 def get_local_species_scorenorm_cachedir(ibs, species_text, ensure=True):
     """
     """
     scorenorm_cachedir = join(ibs.get_cachedir(),
                               const.PATH_NAMES.scorenormdir)
     species_cachedir = join(scorenorm_cachedir, species_text)
     if ensure:
         ut.ensuredir(scorenorm_cachedir)
         ut.ensuredir(species_cachedir)
     return species_cachedir
Beispiel #40
0
 def __init__(self):
     import utool as ut
     self.modname = None
     code_dpath = ut.truepath(ut.get_argval('--code-dir', default='~/code'))
     self.code_dpath = ut.unexpanduser(code_dpath)
     self.repo_fname = (ut.get_argval(('--repo', '--repo-name'), type_=str))
     self.repo_dpath = join(code_dpath, self.repo_fname)
     self.modname = ut.get_argval('--modname', default=self.repo_fname)
     self.regenfmt = 'python -m utool SetupRepo.{cmd} --modname={modname} --repo={repo_fname} --codedir={code_dpath}'
     ut.ensuredir(self.repo_dpath, verbose=True)
Beispiel #41
0
def detector_train(ibs):
    results = ibs.localizer_train()
    localizer_weight_path, localizer_config_path, localizer_class_path = results
    classifier_model_path = ibs.classifier_binary_train()
    labeler_model_path = ibs.labeler_train()
    output_path = join(ibs.get_cachedir(), 'training', 'detector')
    ut.ensuredir(output_path)
    ut.copy(localizer_weight_path, join(output_path, 'localizer.weights'))
    ut.copy(localizer_config_path, join(output_path, 'localizer.config'))
    ut.copy(localizer_class_path, join(output_path, 'localizer.classes'))
    ut.copy(classifier_model_path, join(output_path, 'classifier.npy'))
    ut.copy(labeler_model_path, join(output_path, 'labeler.npy'))
Beispiel #42
0
    def build_sphinx_apidoc_cmdstr():
        print('')
        print('if this fails try: sudo pip install sphinx')
        print('')
        apidoc = 'sphinx-apidoc'
        if ut.WIN32:
            winprefix = 'C:/Python27/Scripts/'
            sphinx_apidoc_exe = winprefix + apidoc + '.exe'
        else:
            sphinx_apidoc_exe = apidoc
        apidoc_argfmt_list = [
            sphinx_apidoc_exe,
            '--force',
            '--full',
            '--maxdepth="{maxdepth}"',
            '--doc-author="{author}"',
            '--doc-version="{doc_version}"',
            '--doc-release="{doc_release}"',
            '--output-dir="_doc"',
            #'--separate',  # Put documentation for each module on its own page
            '--private',  # Include "_private" modules
            '{pkgdir}',
        ]
        outputdir = '_doc'
        author = ut.parse_author()
        packages = ut.find_packages(maxdepth=1)
        assert len(
            packages) != 0, 'directory must contain at least one package'
        if len(packages) > 1:
            assert len(packages) == 1,\
                ('FIXME I dont know what to do with more than one root package: %r'
                 % (packages,))
        pkgdir = packages[0]
        version = ut.parse_package_for_version(pkgdir)
        modpath = dirname(ut.truepath(pkgdir))

        apidoc_fmtdict = {
            'author': author,
            'maxdepth': '8',
            'pkgdir': pkgdir,
            'doc_version': version,
            'doc_release': version,
            'outputdir': outputdir,
        }
        ut.assert_exists('setup.py')
        ut.ensuredir('_doc')
        apidoc_fmtstr = ' '.join(apidoc_argfmt_list)
        apidoc_cmdstr = apidoc_fmtstr.format(**apidoc_fmtdict)
        print('[util_setup] autogenerate sphinx docs for %r' % (pkgdir, ))
        if ut.VERBOSE:
            print(ut.dict_str(apidoc_fmtdict))
        return apidoc_cmdstr, modpath, outputdir
Beispiel #43
0
def export_ibeis_to_hotspotter(ibs):
    # Dumps the files
    hsdb_dir = ibs.get_dbdir()
    internal_hsdb = join(hsdb_dir, "_hsdb")
    ut.ensuredir(internal_hsdb)
    image_table_csv, name_table_csv, chip_table_csv = get_hots_table_strings(ibs)
    # Write Tables
    with open(join(internal_hsdb, "HSDB_image_table.csv"), "wb") as imgtbl_file:
        imgtbl_file.write(image_table_csv)
    with open(join(internal_hsdb, "HSDB_name_table.csv"), "wb") as nametbl_file:
        nametbl_file.write(name_table_csv)
    with open(join(internal_hsdb, "HSDB_chip_table.csv"), "wb") as chiptbl_file:
        chiptbl_file.write(chip_table_csv)
Beispiel #44
0
    def build_sphinx_apidoc_cmdstr():
        print('')
        print('if this fails try: sudo pip install sphinx')
        print('')
        apidoc = 'sphinx-apidoc'
        if ut.WIN32:
            winprefix = 'C:/Python27/Scripts/'
            sphinx_apidoc_exe = winprefix + apidoc + '.exe'
        else:
            sphinx_apidoc_exe = apidoc
        apidoc_argfmt_list = [
            sphinx_apidoc_exe,
            '--force',
            '--full',
            '--maxdepth="{maxdepth}"',
            '--doc-author="{author}"',
            '--doc-version="{doc_version}"',
            '--doc-release="{doc_release}"',
            '--output-dir="_doc"',
            #'--separate',  # Put documentation for each module on its own page
            '--private',  # Include "_private" modules
            '{pkgdir}',
        ]
        outputdir = '_doc'
        author = ut.parse_author()
        packages = ut.find_packages(maxdepth=1)
        assert len(packages) != 0, 'directory must contain at least one package'
        if len(packages) > 1:
            assert len(packages) == 1,\
                ('FIXME I dont know what to do with more than one root package: %r'
                 % (packages,))
        pkgdir = packages[0]
        version = ut.parse_package_for_version(pkgdir)
        modpath = dirname(ut.truepath(pkgdir))

        apidoc_fmtdict = {
            'author': author,
            'maxdepth': '8',
            'pkgdir': pkgdir,
            'doc_version': version,
            'doc_release': version,
            'outputdir': outputdir,
        }
        ut.assert_exists('setup.py')
        ut.ensuredir('_doc')
        apidoc_fmtstr = ' '.join(apidoc_argfmt_list)
        apidoc_cmdstr = apidoc_fmtstr.format(**apidoc_fmtdict)
        print('[util_setup] autogenerate sphinx docs for %r' % (pkgdir,))
        if ut.VERBOSE:
            print(ut.dict_str(apidoc_fmtdict))
        return apidoc_cmdstr, modpath, outputdir
Beispiel #45
0
def test_labeler(output_path):
    print('[labeler] Loading the labeler training data')
    data_list, label_list = load_images()

    print('[mnist] Loading the data into a JPCNN_Data')
    data = JPCNN_Data()
    data.set_data_list(data_list)
    data.set_label_list(label_list)

    print('[labeler] Create the JPCNN_Model used for testing')
    model = Labeler_Model('model.npy')

    print('[mnist] Create the JPCNN_network and start testing')
    net = JPCNN_Network(model, data)
    test_results = net.test(output_path, best_weights=True)
    prediction_list = test_results['label_list']
    confidence_list = test_results['confidence_list']

    best_errors = np.inf
    conf_list = [ _ / 100.0 for _ in range(0, 101) ]
    # conf_list = [ 0.81 ]  # FOR MODEL.5.NPY
    for conf in conf_list:
        failure_path = join(output_path, 'failures')
        ut.ensuredir(failure_path)
        error_list = [0, 0, 0, 0]
        zipped = zip(data_list, label_list, prediction_list, confidence_list)
        for index, (data, label, prediction, confidence) in enumerate(zipped):
            if prediction == 'negative' and confidence < conf:
                prediction = 'positive'
                confidence == 1.0 - confidence
            if label == prediction and label == 'positive':
                error_list[0] += 1
            elif label == prediction and label == 'negative':
                error_list[1] += 1
            elif label != prediction:
                if label == 'positive':
                    error_list[2] += 1
                elif label == 'negative':
                    error_list[3] += 1
                args = (confidence, index, label, prediction)
                failure_filename = 'failure_%0.05f_%06d_%s_%s.png' % args
                failure_filepath = join(failure_path, failure_filename)
                cv2.imwrite(failure_filepath, data)
        errors = error_list[2] + error_list[3]
        total = sum(error_list)
        if errors < best_errors:
            best_errors = errors
            print(error_list)
            args = (conf, errors / total, errors, total, )
            print('Error rate %0.2f: %0.03f [ %d / %d ]' % args)
Beispiel #46
0
 def dummy_manual_chipmask(depc, parent_rowids, config=None):
     import vtool as vt
     from plottool import interact_impaint
     mask_dpath = join(depc.cache_dpath, 'ManualChipMask')
     ut.ensuredir(mask_dpath)
     if config is None:
         config = {}
     print('Requesting user defined chip mask')
     for rowid in parent_rowids:
         img = vt.imread(gpath_list[rowid])
         mask = interact_impaint.impaint_mask2(img)
         mask_fpath = join(mask_dpath, 'mask%d.png' % (rowid, ))
         vt.imwrite(mask_fpath, mask)
         w, h = vt.get_size(mask)
         yield (w, h), mask_fpath
Beispiel #47
0
 def extract_zipfile_images(ibs, ingestable):
     import utool as ut  # NOQA
     zipfile_list = ut.glob(ingestable.img_dir, '*.zip', recursive=True)
     if len(zipfile_list) > 0:
         print('Found zipfile_list = %r' % (zipfile_list,))
         ut.ensuredir(unzipped_file_base_dir)
         for zipfile in zipfile_list:
             unziped_file_relpath = dirname(relpath(relpath(realpath(zipfile), realpath(ingestable.img_dir))))
             unzipped_file_dir = join(unzipped_file_base_dir, unziped_file_relpath)
             ut.ensuredir(unzipped_file_dir)
             ut.unzip_file(zipfile, output_dir=unzipped_file_dir, overwrite=False)
         gpath_list = ut.list_images(unzipped_file_dir, fullpath=True, recursive=True)
     else:
         gpath_list = []
     return gpath_list
Beispiel #48
0
def get_word_dpaths(vocabdir, wx_sample, metrics):
    """
    Gets word folder names and ensure they exist
    """
    ut.ensuredir(vocabdir)
    wx2_dpath = {
        wx: join(vocabdir, get_word_dname(wx, metrics))
        for wx in wx_sample
    }
    iter_ = ut.progiter(six.itervalues(wx2_dpath),
                        lbl='Ensuring word_dpath: ',
                        freq=200)
    for dpath in iter_:
        ut.ensuredir(dpath)
    return wx2_dpath
Beispiel #49
0
 def dummy_manual_chipmask(depc, parent_rowids, config=None):
     import vtool as vt
     from plottool import interact_impaint
     mask_dpath = ut.unixjoin(depc.cache_dpath, 'ManualChipMask')
     ut.ensuredir(mask_dpath)
     if config is None:
         config = {}
     print('Requesting user defined chip mask')
     for rowid in parent_rowids:
         img = vt.imread(gpath_list[rowid])
         mask = interact_impaint.impaint_mask2(img)
         mask_fpath = ut.unixjoin(mask_dpath, 'mask%d.png' % (rowid,))
         vt.imwrite(mask_fpath, mask)
         w, h = vt.get_size(mask)
         yield (w, h), mask_fpath
    def __init__(qreq_, ibs=None, qaids=None, daids=None, config=None):
        super(SMKRequest, qreq_).__init__()
        if config is None:
            config = {}

        qreq_.ibs = ibs
        qreq_.qaids = qaids
        qreq_.daids = daids

        qreq_.config = config

        # qreq_.vocab = None
        # qreq_.dinva = None

        qreq_.qinva = None
        qreq_.dinva = None
        qreq_.smk = SMK()

        # Hack to work with existing hs code
        qreq_.stack_config = SMKRequestConfig(**config)
        # Flat config
        qreq_.qparams = dtool.base.StackedConfig(
            [dict(qreq_.stack_config.parse_items())])
        #    # TODO: add vocab, inva, features
        qreq_.cachedir = ut.ensuredir((ibs.cachedir, 'smk'))
Beispiel #51
0
def test_akmeans(full_test=False, plot_test=False, num_pca_dims=2, data_dim=2,
                 nump=1000):
    import numpy as np
    from vtool import clustering
    nump = nump
    dims = data_dim  # 128
    dtype = np.uint8
    print('Make %d random %d-dimensional %s points.' % (nump, dims, dtype))
    # Seed for a determenistic test
    np.random.seed(42)
    data = np.array(np.random.randint(0, 255, (nump, dims)), dtype=dtype)

    num_clusters = 10
    max_iters = 2
    ave_unchanged_thresh = 0
    ave_unchanged_iterwin = 10
    flann_params = {}

    cache_dir = utool.get_app_resource_dir('vtool', 'test_cache')
    utool.ensuredir(cache_dir)

    # Test precomputing
    dx2_label, centers = clustering.precompute_akmeans(data, num_clusters,
                                                       max_iters=max_iters,
                                                       cache_dir=cache_dir)
    # internal names
    datax2_clusterx, centroids = dx2_label, centers

    if plot_test:
        clustering.plot_clusters(data, datax2_clusterx, centroids, num_pca_dims=num_pca_dims)

    assert centers.shape == (num_clusters, dims), 'sanity check'
    assert dx2_label.shape == (nump,), 'sanity check'

    # Test regular computing
    if full_test:
        dx2_label, centers = clustering.akmeans(data, num_clusters, max_iters=max_iters)
        assert centers.shape == (num_clusters, dims), 'sanity check'
        assert dx2_label.shape == (nump,), 'sanity check'

    if False:
        # other test (development)
        import pyflann
        flann_lib_inst = pyflann.flann
        flann_class_inst = pyflann.FLANN()
        flann_class_inst.build_index(data)
    return locals()
Beispiel #52
0
def image_upload(cleanup=True, **kwargs):
    r"""
    Returns the gid for an uploaded image.

    Args:
        image (image binary): the POST variable containing the binary
            (multi-form) image data
        **kwargs: Arbitrary keyword arguments; the kwargs are passed down to
            the add_images function

    Returns:
        gid (rowids): gid corresponding to the image submitted.
            lexigraphical order.

    RESTful:
        Method: POST
        URL:    /api/upload/image/
    """
    ibs = current_app.ibs
    logger.info('request.files = %s' % (request.files, ))

    filestore = request.files.get('image', None)
    if filestore is None:
        raise controller_inject.WebMissingInput(
            'Missing required image parameter', 'image')
        # raise IOError('Image not given')

    uploads_path = ibs.get_uploadsdir()
    ut.ensuredir(uploads_path)
    current_time = time.strftime('%Y_%m_%d_%H_%M_%S')

    modifier = 1
    upload_filename = 'upload_%s.png' % (current_time)
    while exists(upload_filename):
        upload_filename = 'upload_%s_%04d.png' % (current_time, modifier)
        modifier += 1

    upload_filepath = join(uploads_path, upload_filename)
    filestore.save(upload_filepath)

    gid_list = ibs.add_images([upload_filepath], **kwargs)
    gid = gid_list[0]

    if cleanup and exists(upload_filepath):
        ut.delete(upload_filepath)

    return gid
Beispiel #53
0
def canonical_localizer_train(ibs, species, ensembles=3, **kwargs):
    from ibeis_cnn.ingest_ibeis import get_cnn_localizer_canonical_training_images_pytorch
    from ibeis.algo.detect import canonical

    args = (species, )
    data_path = join(ibs.get_cachedir(),
                     'extracted-localizer-canonical-%s' % args)
    extracted_path = get_cnn_localizer_canonical_training_images_pytorch(
        ibs,
        species,
        dest_path=data_path,
    )

    weights_path_list = []
    for ensemble_num in range(ensembles):
        args = (
            species,
            ensemble_num,
        )
        output_path = join(ibs.get_cachedir(), 'training',
                           'localizer-canonical-%s-ensemble-%d' % args)
        weights_path = canonical.train(extracted_path, output_path)
        weights_path_list.append(weights_path)

    args = (species, )
    output_name = 'localizer.canonical.%s' % args
    ensemble_path = join(ibs.get_cachedir(), 'training', output_name)
    ut.ensuredir(ensemble_path)

    archive_path = '%s.zip' % (ensemble_path)
    ensemble_weights_path_list = []

    for index, weights_path in enumerate(sorted(weights_path_list)):
        assert exists(weights_path)
        ensemble_weights_path = join(
            ensemble_path, 'localizer.canonical.%d.weights' % (index, ))
        ut.copy(weights_path, ensemble_weights_path)
        ensemble_weights_path_list.append(ensemble_weights_path)

    ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list
    ut.archive_files(archive_path,
                     ensemble_weights_path_list,
                     overwrite=True,
                     common_prefix=True)

    return archive_path
Beispiel #54
0
def export_wbia_to_hotspotter(ibs):
    # Dumps the files
    hsdb_dir = ibs.get_dbdir()
    internal_hsdb = join(hsdb_dir, '_hsdb')
    ut.ensuredir(internal_hsdb)
    image_table_csv, name_table_csv, chip_table_csv = get_hots_table_strings(
        ibs)
    # Write Tables
    with open(join(internal_hsdb, 'HSDB_image_table.csv'),
              'wb') as imgtbl_file:
        imgtbl_file.write(image_table_csv)
    with open(join(internal_hsdb, 'HSDB_name_table.csv'),
              'wb') as nametbl_file:
        nametbl_file.write(name_table_csv)
    with open(join(internal_hsdb, 'HSDB_chip_table.csv'),
              'wb') as chiptbl_file:
        chiptbl_file.write(chip_table_csv)
Beispiel #55
0
def image_upload(cleanup=True, **kwargs):
    r"""
    Returns the gid for an uploaded image.

    Args:
        image (image binary): the POST variable containing the binary
            (multi-form) image data
        **kwargs: Arbitrary keyword arguments; the kwargs are passed down to
            the add_images function

    Returns:
        gid (rowids): gid corresponding to the image submitted.
            lexigraphical order.

    RESTful:
        Method: POST
        URL:    /api/image/
    """
    ibs = current_app.ibs
    print('request.files = %s' % (request.files,))

    filestore = request.files.get('image', None)
    if filestore is None:
        raise IOError('Image not given')

    uploads_path = ibs.get_uploadsdir()
    ut.ensuredir(uploads_path)
    current_time = time.strftime('%Y_%m_%d_%H_%M_%S')

    modifier = 1
    upload_filename = 'upload_%s.png' % (current_time)
    while exists(upload_filename):
        upload_filename = 'upload_%s_%04d.png' % (current_time, modifier)
        modifier += 1

    upload_filepath = join(uploads_path, upload_filename)
    filestore.save(upload_filepath)

    gid_list = ibs.add_images([upload_filepath], **kwargs)
    gid = gid_list[0]

    if cleanup:
        ut.remove_dirs(upload_filepath)

    return gid
Beispiel #56
0
def get_flann_fpath(dpts, cache_dir='default', cfgstr='', flann_params={},
                    use_params_hash=True, use_data_hash=True, appname='vtool',
                    verbose=True):
    """ returns filepath for flann index """
    if cache_dir == 'default':
        if verbose:
            print('[flann] using default cache dir')
        cache_dir = utool.get_app_resource_dir(appname)
        utool.ensuredir(cache_dir)
    flann_cfgstr = get_flann_cfgstr(dpts, flann_params, cfgstr,
                                    use_params_hash=use_params_hash,
                                    use_data_hash=use_data_hash)
    if verbose:
        print('...flann_cache cfgstr = %r: ' % flann_cfgstr)
    # Append any user labels
    flann_fname = 'flann_index' + flann_cfgstr + '.flann'
    flann_fpath = normpath(join(cache_dir, flann_fname))
    return flann_fpath
Beispiel #57
0
def testdata_expts(defaultdb='testdb1',
                   default_acfgstr_name_list=['default:qindex=0:10:4,dindex=0:20'],
                   default_test_cfg_name_list=['default'],
                   a=None,
                   t=None,
                   qaid_override=None,
                   daid_override=None,
                   initial_aids=None,
                   ):
    """
    Use this if you want data from an experiment.
    Command line interface to quickly get testdata for test_results.

    Command line flags can be used to specify db, aidcfg, pipecfg, qaid
    override, daid override (and maybe initial aids).

    """
    print('[main_helpers] testdata_expts')
    import ibeis
    from ibeis.expt import experiment_harness
    from ibeis.expt import test_result
    if a is not None:
        default_acfgstr_name_list = a
    if t is not None:
        default_test_cfg_name_list = t

    if isinstance(default_acfgstr_name_list, six.string_types):
        default_acfgstr_name_list = [default_acfgstr_name_list]
    if isinstance(default_test_cfg_name_list, six.string_types):
        default_test_cfg_name_list = [default_test_cfg_name_list]

    #from ibeis.expt import experiment_helpers
    ibs = ibeis.opendb(defaultdb=defaultdb)
    acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=list,
                                   default=default_acfgstr_name_list)
    test_cfg_name_list = ut.get_argval(('-t', '-p'), type_=list, default=default_test_cfg_name_list)
    daid_override = ut.get_argval(('--daid-override', '--daids-override'), type_=list, default=daid_override)
    qaid_override = ut.get_argval(('--qaid', '--qaids-override', '--qaid-override'), type_=list, default=qaid_override)

    # Hack a cache here
    use_bigtest_cache3 = not ut.get_argflag(('--nocache', '--nocache-hs'))
    use_bigtest_cache3 &= ut.is_developer()
    use_bigtest_cache3 &= False
    if use_bigtest_cache3:
        from os.path import dirname, join
        cache_dir = ut.ensuredir(join(dirname(ut.get_module_dir(ibeis)), 'BIG_TESTLIST_CACHE3'))
        load_testres = ut.cached_func('testreslist', cache_dir=cache_dir)(experiment_harness.run_test_configurations2)
    else:
        load_testres = experiment_harness.run_test_configurations2
    testres_list = load_testres(
        ibs, acfg_name_list, test_cfg_name_list, qaid_override=qaid_override,
        daid_override=daid_override, initial_aids=initial_aids)
    testres = test_result.combine_testres_list(ibs, testres_list)

    print(testres)

    return ibs, testres
Beispiel #58
0
def copy_ibeisdb(source_dbdir, dest_dbdir):
    # TODO; rectify with rsycn script
    from os.path import normpath
    import ibeis
    exclude_dirs = [ut.ensure_unixslash(normpath(rel)) for rel in ibeis.const.EXCLUDE_COPY_REL_DIRS + ['_hsdb', '.hs_internals']]

    rel_tocopy = ut.glob(source_dbdir, '*', exclude_dirs=exclude_dirs, recursive=True, with_files=True, with_dirs=False, fullpath=False)
    rel_tocopy_dirs = ut.glob(source_dbdir, '*', exclude_dirs=exclude_dirs, recursive=True, with_files=False, with_dirs=True, fullpath=False)

    src_list = [join(source_dbdir, relpath) for relpath in rel_tocopy]
    dst_list = [join(dest_dbdir, relpath) for relpath in rel_tocopy]

    # ensure directories exist
    rel_tocopy_dirs = [dest_dbdir] + [join(dest_dbdir, dpath_) for dpath_ in rel_tocopy_dirs]
    for dpath in rel_tocopy_dirs:
        ut.ensuredir(dpath)
    # copy files
    ut.copy(src_list, dst_list)