def download_tomcat():
"""
Put tomcat into a directory controlled by ibeis
CommandLine:
# Reset
python -c "import utool as ut; ut.delete(ut.unixjoin(ut.get_app_resource_dir('ibeis'), 'tomcat'))"
"""
from os.path import splitext, dirname
print('Grabbing tomcat')
# FIXME: need to make a stable link
if ut.WIN32:
tomcat_binary_url = 'http://mirrors.advancedhosters.com/apache/tomcat/tomcat-8/v8.0.24/bin/apache-tomcat-8.0.24-windows-x86.zip'
else:
tomcat_binary_url = 'http://mirrors.advancedhosters.com/apache/tomcat/tomcat-8/v8.0.24/bin/apache-tomcat-8.0.24.zip'
zip_fpath = ut.grab_file_url(tomcat_binary_url, appname='ibeis')
# Download tomcat into the IBEIS resource directory
tomcat_dpath = join(dirname(zip_fpath), 'tomcat')
if not ut.checkpath(tomcat_dpath, verbose=True):
# hack because unzipping is still weird
ut.unzip_file(zip_fpath)
tomcat_dpath_tmp = splitext(zip_fpath)[0]
ut.move(tomcat_dpath_tmp, tomcat_dpath)
if ut.checkpath(join(tomcat_dpath, 'bin'), verbose=True):
scriptnames = ['catalina.sh', 'startup.sh', 'shutdown.sh']
for fname in scriptnames:
fpath = join(tomcat_dpath, 'bin', fname)
if not ut.is_file_executable(fpath):
print('Adding executable bits to script %r' % (fpath,))
ut.chmod_add_executable(fpath)
return tomcat_dpath
def reset_testdbs(**kwargs):
# Step 0) Parse Args
default_args = {'reset_' + key: False
for key in six.iterkeys(TEST_DBNAMES_MAP)}
default_args['reset_all'] = False
default_args.update(kwargs)
argdict = ut.parse_dict_from_argv(default_args)
if not any(list(six.itervalues(argdict))):
# Default behavior is to reset the small dbs
argdict['reset_testdb0'] = True
argdict['reset_testdb1'] = True
argdict['reset_testdb_guiall'] = True
# Step 1) Delete DBs to be Reset
for key, dbname in six.iteritems(TEST_DBNAMES_MAP):
if argdict.get('reset_' + key, False) or argdict['reset_all']:
delete_dbdir(dbname)
# Step 3) Ensure DBs that dont exist
ensure_smaller_testingdbs()
workdir = ibeis.sysres.get_workdir()
if not ut.checkpath(join(workdir, 'PZ_MTEST'), verbose=True):
ibeis.ensure_pz_mtest()
if not ut.checkpath(join(workdir, 'NAUT_test'), verbose=True):
ibeis.ensure_nauts()
if not ut.checkpath(join(workdir, 'testdb2'), verbose=True):
ibeis.init.sysres.ensure_testdb2()
# Step 4) testdb1 becomes the main database
workdir = ibeis.sysres.get_workdir()
TESTDB1 = join(workdir, 'testdb1')
sysres.set_default_dbdir(TESTDB1)
def TEST_DELETE_IMAGE(ibs, back):
gpath_list = grabdata.get_test_gpaths(ndata=None)[0:4]
gid_list = ibs.add_images(gpath_list)
bbox_list = [(0, 0, 100, 100)] * len(gid_list)
name_list = ['a', 'b', 'a', 'd']
aid_list = ibs.add_annots(gid_list, bbox_list=bbox_list, name_list=name_list)
gid = gid_list[0]
assert gid is not None, "gid is None"
aid_list = ibs.get_image_aids(gid)
assert len(aid_list) == 1, "Length of aid_list=%r" % (len(aid_list),)
aid = aid_list[0]
assert aid is not None, "aid is None"
cid = ibs.get_annot_cids(aid, ensure=False)
fid = ibs.get_annot_fids(aid, ensure=False)
assert cid is None, "cid=%r should be None" % (cid,)
assert fid is None, "fid=%r should be None" % (fid,)
cid = ibs.get_annot_cids(aid, ensure=True)
fid = ibs.get_annot_fids(aid, ensure=True)
assert cid is not None, "cid should be computed"
assert fid is not None, "fid should be computed"
gthumbpath = ibs.get_image_thumbpath(gid)
athumbpath = ibs.get_annot_chip_thumbpath(aid)
ibs.delete_images(gid)
all_gids = ibs.get_valid_gids()
all_aids = ibs.get_valid_aids()
all_cids = ibs.get_valid_cids()
all_fids = ibs.get_valid_fids()
assert gid not in all_gids, "gid still exists"
assert aid not in all_aids, "rid %r still exists" % aid
assert fid not in all_fids, "fid %r still exists" % fid
assert cid not in all_cids, "cid %r still exists" % cid
assert not utool.checkpath(gthumbpath), "Thumbnail still exists"
assert not utool.checkpath(athumbpath), "ANNOTATION Thumbnail still exists"
return locals()
def ensure_smaller_testingdbs():
"""
Makes the smaller test databases
"""
def make_testdb0():
""" makes testdb0 """
def get_test_gpaths(ndata=None, names=None, **kwargs):
# Read ndata from args or command line
""" DEPRICATE """
ndata_arg = ut.get_argval('--ndata', type_=int, default=None, help_='use --ndata to specify bigger data')
if ndata_arg is not None:
ndata = ndata_arg
imgdir = get_testdata_dir(**kwargs)
gpath_list = sorted(list(ut.list_images(imgdir, full=True, recursive=True)))
# Get only the gpaths of certain names
if names is not None:
gpath_list = [gpath for gpath in gpath_list if
ut.basename_noext(gpath) in names]
# Get a some number of test images
if ndata is not None:
gpath_cycle = cycle(gpath_list)
if six.PY2:
gpath_list = [gpath_cycle.next() for _ in range(ndata)]
else:
gpath_list = [next(gpath_cycle) for _ in range(ndata)]
return gpath_list
workdir = ibeis.sysres.get_workdir()
TESTDB0 = join(workdir, 'testdb0')
main_locals = ibeis.main(dbdir=TESTDB0, gui=False, allow_newdir=True)
ibs = main_locals['ibs']
assert ibs is not None, str(main_locals)
gpath_list = list(map(ut.unixpath, get_test_gpaths()))
#print('[RESET] gpath_list=%r' % gpath_list)
gid_list = ibs.add_images(gpath_list) # NOQA
valid_gids = ibs.get_valid_gids()
valid_aids = ibs.get_valid_aids()
try:
assert len(valid_aids) == 0, 'there are more than 0 annotations in an empty database!'
except Exception as ex:
ut.printex(ex, key_list=['valid_aids'])
raise
gid_list = valid_gids[0:1]
bbox_list = [(0, 0, 100, 100)]
aid = ibs.add_annots(gid_list, bbox_list=bbox_list)[0]
#print('[RESET] NEW RID=%r' % aid)
aids = ibs.get_image_aids(gid_list)[0]
try:
assert aid in aids, ('bad annotation adder: aid = %r, aids = %r' % (aid, aids))
except Exception as ex:
ut.printex(ex, key_list=['aid', 'aids'])
raise
get_testdata_dir(True)
if not ut.checkpath(join(ibeis.sysres.get_workdir(), 'testdb0'), verbose=True):
print("\n\nMAKE TESTDB0\n\n")
make_testdb0()
if not ut.checkpath(join(ibeis.sysres.get_workdir(), 'testdb1'), verbose=True):
print("\n\nMAKE TESTDB1\n\n")
ingest_database.ingest_standard_database('testdb1')
def build_win32_inno_installer():
inno_dir = r'C:\Program Files (x86)\Inno Setup 5'
inno_fname = 'ISCC.exe'
inno_fpath = join(inno_dir, inno_fname)
cwd = get_setup_dpath()
iss_script = join(cwd, '_installers', 'win_installer_script.iss')
assert utool.checkpath(inno_fpath, verbose=True)
assert utool.checkpath(iss_script, verbose=True)
utool.cmd([inno_fpath, iss_script])
import shutil
installer_src = join(cwd, '_installers', 'Output', 'ibeis-win32-setup.exe')
installer_dst = join(cwd, 'dist', 'ibeis-win32-setup.exe')
shutil.move(installer_src, installer_dst)
def _get_models(ibs, species, modeldir="default", cfg_override=True, verbose=VERBOSE_RF):
r"""
Args:
ibs (IBEISController): ibeis controller object
species (?):
modeldir (str): (default = 'default')
cfg_override (bool): (default = True)
verbose (bool): verbosity flag(default = False)
Returns:
?: fpath_list
CommandLine:
python -m ibeis.algo.detect.randomforest --test-_get_models
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.detect.randomforest import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> species = ibeis.const.TEST_SPECIES.ZEB_PLAIN
>>> modeldir = 'default'
>>> cfg_override = True
>>> verbose = False
>>> fpath_list = _get_models(ibs, species, modeldir, cfg_override, verbose)
>>> result = ('fpath_list = %s' % (str(fpath_list),))
>>> print(result)
"""
# with ut.embed_on_exception_context:
if cfg_override and len(ibs.cfg.detect_cfg.trees_path) > 0:
trees_path = ibs.cfg.detect_cfg.trees_path
else:
# Ensure all models downloaded and accounted for
assert species is not None, "[_get_models] Cannot detect without specifying a species"
grabmodels.ensure_models(modeldir=modeldir, verbose=verbose)
trees_path = grabmodels.get_species_trees_paths(species, modeldir=modeldir)
# Load tree paths
if ut.checkpath(trees_path, verbose=verbose):
fpath_list = ut.ls(trees_path, "*.txt")
# direct = Directory(trees_path, include_extensions=['txt'])
# files = direct.files()
else:
# If the models do not exist, return None
fpath_list = None
if fpath_list is None or len(fpath_list) == 0:
msg = (
ut.codeblock(
"""
[_get_models] Error loading trees, either directory or fpath_list not found
* trees_path = %r
* fpath_list = %r
* species = %r
* model_dir = %r
* cfg_override = %r
"""
)
% (trees_path, fpath_list, species, modeldir, cfg_override)
)
raise AssertionError(msg)
return fpath_list
def ensure_text(fname, text, repo_dpath='.', force=None, locals_={}, chmod=None):
"""
Args:
fname (str): file name
text (str):
repo_dpath (str): directory path string(default = '.')
force (bool): (default = False)
locals_ (dict): (default = {})
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_git import * # NOQA
>>> import utool as ut
>>> result = setup_repo()
>>> print(result)
"""
import utool as ut
ut.colorprint('Ensuring fname=%r' % (fname), 'yellow')
if force is None and ut.get_argflag('--force-%s' % (fname,)):
force = True
fpath = join(repo_dpath, fname)
if force or not ut.checkpath(fpath, verbose=2, n=5):
text_ = ut.remove_codeblock_syntax_sentinals(text)
fmtkw = locals_.copy()
fmtkw['fname'] = fname
text_ = text_.format(**fmtkw) + '\n'
ut.writeto(fpath, text_)
try:
if chmod:
ut.chmod(fpath, chmod)
except Exception as ex:
ut.printex(ex, iswarning=True)
def ensure_inno_isinstalled():
""" Ensures that the current machine has INNO installed. returns path to the
executable """
assert ut.WIN32, 'Can only build INNO on windows'
inno_fpath = ut.search_in_dirs('Inno Setup 5\ISCC.exe', ut.get_install_dirs())
# Make sure INNO is installed
if inno_fpath is None:
print('WARNING: cannot find inno_fpath')
AUTO_FIXIT = ut.WIN32
print('Inno seems to not be installed. AUTO_FIXIT=%r' % AUTO_FIXIT)
if AUTO_FIXIT:
print('Automaticaly trying to downoad and install INNO')
# Download INNO Installer
inno_installer_url = 'http://www.jrsoftware.org/download.php/ispack.exe'
inno_installer_fpath = ut.download_url(inno_installer_url)
print('Automaticaly trying to install INNO')
# Install INNO Installer
ut.cmd(inno_installer_fpath)
else:
inno_homepage_url = 'http://www.jrsoftware.org/isdl.php'
ut.open_url_in_browser(inno_homepage_url)
raise AssertionError('Cannot find INNO and AUTOFIX it is false')
# Ensure that it has now been installed
inno_fpath = ut.search_in_dirs('Inno Setup 5\ISCC.exe', ut.get_install_dirs())
assert ut.checkpath(inno_fpath, verbose=True, info=True), 'inno installer is still not installed!'
return inno_fpath
def TEST_DELETE_ANNOTATION(ibs, back):
gpath_list = grabdata.get_test_gpaths(ndata=None)[0:4]
gid_list = ibs.add_images(gpath_list)
bbox_list = [(0, 0, 100, 100)] * len(gid_list)
name_list = ['a', 'b', 'a', 'd']
aid_list = ibs.add_annots(gid_list, bbox_list=bbox_list, name_list=name_list)
aid = aid_list[0]
assert aid is not None, "aid is None"
cid = ibs.get_annot_cids(aid, ensure=False)
fid = ibs.get_annot_fids(aid, ensure=False)
assert cid is None, "cid should be None"
assert fid is None, "fid should be None"
cid = ibs.get_annot_cids(aid, ensure=True)
fid = ibs.get_annot_fids(aid, ensure=True)
assert cid is not None, "cid should be computed"
assert fid is not None, "fid should be computed"
thumbpath = ibs.get_annot_chip_thumbpath(aid)
ibs.delete_annots(aid)
aid_list = ibs.get_valid_aids()
cid_list = ibs.get_valid_cids()
fid_list = ibs.get_valid_fids()
assert aid not in aid_list, "RID still exists"
assert cid not in cid_list, "CID still exists"
assert fid not in fid_list, "FID still exists"
assert not utool.checkpath(thumbpath), "Thumbnail still exists"
return locals()
def template(template_directory=None, template_filename=None, **kwargs):
global_args = {
'NAVBAR': NavbarClass(),
'YEAR': date.today().year,
'URL': flask.request.url,
'REFER_SRC_STR': flask.request.url.replace(flask.request.url_root, ''),
'__wrapper__' : True,
}
global_args['REFER_SRC_ENCODED'] = encode_refer_url(global_args['REFER_SRC_STR'])
if 'refer' in flask.request.args.keys():
refer = flask.request.args['refer']
print('[web] REFER: %r' % (refer, ))
global_args['REFER_DST_ENCODED'] = refer
global_args['REFER_DST_STR'] = decode_refer_url(refer)
if template_directory is None:
template_directory = ''
#template_directory = abspath(join(dirname(__file__), 'templates'))
#template_directory = join(dirname(dirname(__file__)))
if template_filename is None:
template_filename = 'index'
template_ = join(template_directory, template_filename + '.html')
# Update global args with the template's args
_global_args = dict(global_args)
_global_args.update(kwargs)
print('[appfuncs] template()')
from ibeis.control import controller_inject
app = controller_inject.get_flask_app()
# flask hates windows apparently
template_ = template_.replace('\\', '/')
print('[appfuncs.template] * app.template_folder = %r' % (app.template_folder,))
print('[appfuncs.template] * template_directory = %r' % (template_directory,))
print('[appfuncs.template] * template_filename = %r' % (template_filename,))
print('[appfuncs.template] * template_ = %r' % (template_,))
try:
ret = flask.render_template(template_, **_global_args)
#ret = flask.render_template(full_template_fpath, **_global_args)
except jinja2.exceptions.TemplateNotFound as ex:
print('Error template not found')
full_template_fpath = join(app.template_folder, template_)
print('[appfuncs.template] * full_template_fpath = %r' % (full_template_fpath,))
ut.checkpath(full_template_fpath, verbose=True)
ut.printex(ex, 'Template error in appfuncs', tb=True)
raise
except Exception as ex:
ut.printex(ex, 'Error in appfuncs', tb=True)
raise
return ret
def TEST_DELETE_ANNOTATION_CHIPS(ibs, back):
gpath_list = grabdata.get_test_gpaths(ndata=None)[0:4]
gid_list = ibs.add_images(gpath_list)
bbox_list = [(0, 0, 100, 100)] * len(gid_list)
name_list = ['a', 'b', 'a', 'd']
aid_list = ibs.add_annots(gid_list, bbox_list=bbox_list, name_list=name_list)
assert len(aid_list) != 0, "No annotations"
aid = aid_list[0]
gid = ibs.get_annot_gids(aid)
assert gid is not None, "gid for aid=%r is None" % (aid,)
gthumbpath = ibs.get_image_thumbpath(gid)
annotation_thumbpath = ibs.get_annot_chip_thumbpath(aid)
ibs.delete_annot_chips(aid)
aid_list = ibs.get_valid_aids()
assert aid in aid_list, "Error: Annotation deleted"
assert not utool.checkpath(gthumbpath), "Image Thumbnail not deleted"
assert not utool.checkpath(annotation_thumbpath), "Roi Thumbnail not deleted"
return locals()
def ensure_inno_script():
""" writes inno script to disk for win32 installer build """
cwd = get_setup_dpath()
iss_script_fpath = join(cwd, '_installers', 'win_installer_script.iss')
# THE ISS USES {} AS SYNTAX. CAREFUL
#app_publisher = 'Rensselaer Polytechnic Institute'
#app_name = 'IBEIS'
import ibeis
iss_script_code = ut.codeblock(
r'''
; Script generated by the Inno Setup Script Wizard.
; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
; http://www.jrsoftware.org/isdl.php
[Setup]
; NOTE: The value of AppId uniquely identifies this application.
; Do not use the same AppId value in installers for other applications.
; (To generate a new GUID, click Tools | Generate GUID inside the IDE.)
; Also it seems like the off-balanced curly brace is necessary
AppId={{47BE3DA2-261D-4672-9849-18BB2EB382FC}
AppName=IBEIS
AppVersion=''' + str(ibeis.__version__) + '''
;AppVerName=IBEIS 1
AppPublisher=Rensselaer Polytechnic Institute
AppPublisherURL=ibeis.org ;www.rpi.edu/~crallj/
AppSupportURL=ibeis.org ;ww.rpi.edu/~crallj/
AppUpdatesURL=ibeis.org ;www.rpi.edu/~crallj/
DefaultDirName={pf}\IBEIS
DefaultGroupName=IBEIS
OutputBaseFilename=ibeis-win32-setup
SetupIconFile=ibsicon.ico
Compression=lzma
SolidCompression=yes
[Languages]
Name: "english"; MessagesFile: "compiler:Default.isl"
[Tasks]
Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
[Files]
Source: "..\dist\ibeis\IBEISApp.exe"; DestDir: "{app}"; Flags: ignoreversion
Source: "..\dist\ibeis\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
; NOTE: Don't use "Flags: ignoreversion" on any shared system files
[Icons]
Name: "{group}\ibeis"; Filename: "{app}\IBEISApp.exe"
Name: "{commondesktop}\ibeis"; Filename: "{app}\IBEISApp.exe"; Tasks: desktopicon
[Run]
Filename: "{app}\IBEISApp.exe"; Description: "{cm:LaunchProgram,IBEIS}"; Flags: nowait postinstall skipifsilent
'''
)
ut.write_to(iss_script_fpath, iss_script_code, onlyifdiff=True)
assert ut.checkpath(iss_script_fpath, verbose=True, info=True), 'cannot find iss_script_fpath'
return iss_script_fpath
def test_file_hash():
resdir = utool.get_app_resource_dir('utool')
test_fpath = join(resdir, 'lorium_ipsum.txt')
if not utool.checkpath(test_fpath, verbose=True, n=100):
utool.write_to(test_fpath, lorium_text)
hash_ = utool.get_file_hash(test_fpath)
target_hash_ = b'\xd1Y\xe5\xa2\xc1\xd8\xb8\nS\xb1?\x16\xfe\xc5\x88\xbd\x9e\xb4\xe3\xda'
print(repr(hash_))
print(repr(target_hash_))
assert hash_ == target_hash_
def ensure_inno_script():
""" writes inno script to disk for win32 installer build """
cwd = get_setup_dpath()
iss_script_fpath = join(cwd, '_installers', 'win_installer_script.iss')
# THE ISS USES {} AS SYNTAX. CAREFUL
#app_publisher = 'Rensselaer Polytechnic Institute'
#app_name = 'IBEIS'
import ibeis
iss_script_code = ut.codeblock(r'''
; Script generated by the Inno Setup Script Wizard.
; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
; http://www.jrsoftware.org/isdl.php
[Setup]
; NOTE: The value of AppId uniquely identifies this application.
; Do not use the same AppId value in installers for other applications.
; (To generate a new GUID, click Tools | Generate GUID inside the IDE.)
; Also it seems like the off-balanced curly brace is necessary
AppId={{47BE3DA2-261D-4672-9849-18BB2EB382FC}
AppName=IBEIS
AppVersion=''' + str(ibeis.__version__) + '''
;AppVerName=IBEIS 1
AppPublisher=Rensselaer Polytechnic Institute
AppPublisherURL=ibeis.org ;www.rpi.edu/~crallj/
AppSupportURL=ibeis.org ;ww.rpi.edu/~crallj/
AppUpdatesURL=ibeis.org ;www.rpi.edu/~crallj/
DefaultDirName={pf}\IBEIS
DefaultGroupName=IBEIS
OutputBaseFilename=ibeis-win32-setup
SetupIconFile=ibsicon.ico
Compression=lzma
SolidCompression=yes
[Languages]
Name: "english"; MessagesFile: "compiler:Default.isl"
[Tasks]
Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
[Files]
Source: "..\dist\ibeis\IBEISApp.exe"; DestDir: "{app}"; Flags: ignoreversion
Source: "..\dist\ibeis\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
; NOTE: Don't use "Flags: ignoreversion" on any shared system files
[Icons]
Name: "{group}\ibeis"; Filename: "{app}\IBEISApp.exe"
Name: "{commondesktop}\ibeis"; Filename: "{app}\IBEISApp.exe"; Tasks: desktopicon
[Run]
Filename: "{app}\IBEISApp.exe"; Description: "{cm:LaunchProgram,IBEIS}"; Flags: nowait postinstall skipifsilent
''')
ut.write_to(iss_script_fpath, iss_script_code, onlyifdiff=True)
assert ut.checkpath(iss_script_fpath, verbose=True,
info=True), 'cannot find iss_script_fpath'
return iss_script_fpath
def get_regen_cmd():
try:
if len(sys.argv) > 0 and ut.checkpath(sys.argv[0]):
# Check if running python command
if ut.is_python_module(sys.argv[0]):
python_exe = ut.python_executable(check=False)
modname = ut.get_modname_from_modpath(sys.argv[0])
new_argv = [python_exe, '-m', modname] + sys.argv[1:]
return ' '.join(new_argv)
except Exception as ex:
ut.printex(ex, iswarning=True)
return ' '.join(sys.argv)
def TEST_DELETE_ANNOTATION_CHIPS(ibs, back):
gpath_list = grabdata.get_test_gpaths(ndata=None)[0:4]
gid_list = ibs.add_images(gpath_list)
bbox_list = [(0, 0, 100, 100)] * len(gid_list)
name_list = ['a', 'b', 'a', 'd']
aid_list = ibs.add_annots(gid_list,
bbox_list=bbox_list,
name_list=name_list)
assert len(aid_list) != 0, "No annotations"
aid = aid_list[0]
gid = ibs.get_annot_gids(aid)
assert gid is not None, "gid for aid=%r is None" % (aid, )
gthumbpath = ibs.get_image_thumbpath(gid)
annotation_thumbpath = ibs.get_annot_chip_thumbpath(aid)
ibs.delete_annot_chips(aid)
aid_list = ibs.get_valid_aids()
assert aid in aid_list, "Error: Annotation deleted"
assert not utool.checkpath(gthumbpath), "Image Thumbnail not deleted"
assert not utool.checkpath(
annotation_thumbpath), "Roi Thumbnail not deleted"
return locals()
def TEST_DELETE_ANNOTATION_ALL(ibs, back):
aid_list = ibs.get_valid_aids()
thumbpath_list = ibs.get_annot_chip_thumbpath(aid_list)
ibs.delete_annots(aid_list)
aid_list = ibs.get_valid_aids()
cid_list = ibs.get_valid_cids()
fid_list = ibs.get_valid_fids()
assert len(aid_list) == 0, "Didn't delete all ANNOTATIONs"
assert len(cid_list) == 0, "Didn't delete all chips"
assert len(fid_list) == 0, "Didn't delete all features"
for thumbpath in thumbpath_list:
assert not utool.checkpath(thumbpath), "Thumbnail still exists"
return locals()
def get_regen_cmd():
# TODO: move to utool
try:
if len(sys.argv) > 0 and ut.checkpath(sys.argv[0]):
# Check if running python command
if ut.is_python_module(sys.argv[0]):
python_exe = ut.python_executable(check=False)
modname = ut.get_modname_from_modpath(sys.argv[0])
new_argv = [python_exe, '-m', modname] + sys.argv[1:]
return ' '.join(new_argv)
except Exception as ex:
ut.printex(ex, iswarning=True)
return ' '.join(sys.argv)
def TEST_DELETE_ANNOTATION_ALL(ibs, back):
aid_list = ibs.get_valid_aids()
thumbpath_list = ibs.get_annot_chip_thumbpath(aid_list)
ibs.delete_annots(aid_list)
aid_list = ibs.get_valid_aids()
cid_list = ibs.get_valid_cids()
fid_list = ibs.get_valid_fids()
assert len(aid_list) == 0, "Didn't delete all ANNOTATIONs"
assert len(cid_list) == 0, "Didn't delete all chips"
assert len(fid_list) == 0, "Didn't delete all features"
for thumbpath in thumbpath_list:
assert not utool.checkpath(thumbpath), "Thumbnail still exists"
return locals()
def reset_testdbs(**kwargs):
# Step 0) Parse Args
import ibeis
ibeis.ENABLE_WILDBOOK_SIGNAL = False
default_args = {
'reset_' + key: False
for key in six.iterkeys(TEST_DBNAMES_MAP)
}
default_args['reset_all'] = False
default_args.update(kwargs)
argdict = ut.parse_dict_from_argv(default_args)
if not any(list(six.itervalues(argdict))):
# Default behavior is to reset the small dbs
argdict['reset_testdb0'] = True
argdict['reset_testdb1'] = True
argdict['reset_testdb_guiall'] = True
# Step 1) Delete DBs to be Reset
for key, dbname in six.iteritems(TEST_DBNAMES_MAP):
if argdict.get('reset_' + key, False) or argdict['reset_all']:
delete_dbdir(dbname)
# Step 3) Ensure DBs that dont exist
ensure_smaller_testingdbs()
workdir = ibeis.sysres.get_workdir()
if not ut.checkpath(join(workdir, 'PZ_MTEST'), verbose=True):
ibeis.ensure_pz_mtest()
if not ut.checkpath(join(workdir, 'NAUT_test'), verbose=True):
ibeis.ensure_nauts()
if not ut.checkpath(join(workdir, 'wd_peter2'), verbose=True):
ibeis.ensure_wilddogs()
if not ut.checkpath(join(workdir, 'testdb2'), verbose=True):
ibeis.init.sysres.ensure_testdb2()
# Step 4) testdb1 becomes the main database
workdir = ibeis.sysres.get_workdir()
TESTDB1 = join(workdir, 'testdb1')
sysres.set_default_dbdir(TESTDB1)
def grab_selenium_chromedriver(redownload=False):
r"""
Automatically download selenium chrome driver if needed
CommandLine:
python -m utool.util_grabdata --test-grab_selenium_chromedriver:1
Example:
>>> # DISABLE_DOCTEST
>>> ut.grab_selenium_chromedriver()
>>> import selenium.webdriver
>>> driver = selenium.webdriver.Chrome()
>>> driver.get('http://www.google.com')
>>> search_field = driver.find_element_by_name('q')
>>> search_field.send_keys('puppies')
>>> search_field.send_keys(selenium.webdriver.common.keys.Keys.ENTER)
Example1:
>>> # DISABLE_DOCTEST
>>> import selenium.webdriver
>>> driver = selenium.webdriver.Firefox()
>>> driver.get('http://www.google.com')
>>> search_field = driver.find_element_by_name('q')
>>> search_field.send_keys('puppies')
>>> search_field.send_keys(selenium.webdriver.common.keys.Keys.ENTER)
"""
import utool as ut
import os
import stat
# TODO: use a better download dir (but it must be in the PATh or selenium freaks out)
chromedriver_dpath = ut.ensuredir(ut.truepath('~/bin'))
chromedriver_fpath = join(chromedriver_dpath, 'chromedriver')
if not ut.checkpath(chromedriver_fpath) or redownload:
assert chromedriver_dpath in os.environ['PATH'].split(os.pathsep)
# TODO: make this work for windows as well
if ut.LINUX and ut.util_cplat.is64bit_python():
import requests
rsp = requests.get('http://chromedriver.storage.googleapis.com/LATEST_RELEASE')
assert rsp.status_code == 200
url = 'http://chromedriver.storage.googleapis.com/' + rsp.text.strip() + '/chromedriver_linux64.zip'
ut.grab_zipped_url(url, download_dir=chromedriver_dpath, redownload=True)
else:
raise AssertionError('unsupported chrome driver getter script')
if not ut.WIN32:
st = os.stat(chromedriver_fpath)
os.chmod(chromedriver_fpath, st.st_mode | stat.S_IEXEC)
ut.assert_exists(chromedriver_fpath)
os.environ['webdriver.chrome.driver'] = chromedriver_fpath
return chromedriver_fpath
def init_theanorc():
theanorc_fpath = join(os.getenv('HOME'), '.theanorc')
theanorc_text = ut.codeblock('''
[global]
floatX = float32
device = gpu0
openmp = True
[nvcc]
fastmath = True
''')
if ut.checkpath(theanorc_fpath, verbose=True):
if not ut.arg_you_sure('overwrite?'):
return
ut.write_to(theanorc_fpath, theanorc_text)
def __setstate__(ibs, state):
# Hack to allow for ibeis objects to be pickled
import ibeis
dbdir = state['dbdir']
machine_name = state.pop('machine_name')
try:
assert machine_name == ut.get_computer_name(), (
'ibeis objects can only be picked and unpickled on the same machine')
except AssertionError as ex:
iswarning = ut.checkpath(dbdir)
ut.printex(ex, iswarning=iswarning)
if not iswarning:
raise
ibs2 = ibeis.opendb(dbdir=dbdir, web=False)
ibs.__dict__.update(**ibs2.__dict__)
def load_or_build_flann(dstcnvs_normer, cachedir=None, verbose=True, *args, **kwargs):
from vtool._pyflann_backend import pyflann as pyflann
flann_fpath = dstcnvs_normer.get_flann_fpath(cachedir)
if ut.checkpath(flann_fpath, verbose=ut.VERBOSE):
try:
dstcnvs_normer.flann = pyflann.FLANN()
dstcnvs_normer.flann.load_index(flann_fpath, dstcnvs_normer.vecs)
assert dstcnvs_normer.flann._FLANN__curindex is not None
# load_success = True
except Exception as ex:
ut.printex(ex, '... cannot load distinctiveness flann', iswarning=True)
dstcnvs_normer.rebuild(cachedir)
else:
dstcnvs_normer.ensure_flann(cachedir)
def load_or_build_flann(dstcnvs_normer, cachedir=None, verbose=True, *args,
**kwargs):
import pyflann
flann_fpath = dstcnvs_normer.get_flann_fpath(cachedir)
if ut.checkpath(flann_fpath, verbose=ut.VERBOSE):
try:
dstcnvs_normer.flann = pyflann.FLANN()
dstcnvs_normer.flann.load_index(flann_fpath, dstcnvs_normer.vecs)
assert dstcnvs_normer.flann._FLANN__curindex is not None
#load_success = True
except Exception as ex:
ut.printex(ex, '... cannot load distinctiveness flann',
iswarning=True)
dstcnvs_normer.rebuild(cachedir)
else:
dstcnvs_normer.ensure_flann(cachedir)
def download_image_urls(image_url_info_list):
# Find ones that we already have
print('Requested %d downloaded images' % (len(image_url_info_list)))
full_gpath_list = [join(image_dir, basename(gpath)) for gpath in image_url_info_list]
exists_list = [ut.checkpath(gpath) for gpath in full_gpath_list]
image_url_info_list_ = ut.compress(image_url_info_list, ut.not_list(exists_list))
print('Already have %d/%d downloaded images' % (
len(image_url_info_list) - len(image_url_info_list_), len(image_url_info_list)))
print('Need to download %d images' % (len(image_url_info_list_)))
#import sys
#sys.exit(0)
# Download the rest
imgurl_prefix = 'https://snapshotserengeti.s3.msi.umn.edu/'
image_url_list = [imgurl_prefix + suffix for suffix in image_url_info_list_]
for img_url in ut.ProgressIter(image_url_list, lbl='Downloading image'):
ut.grab_file_url(img_url, download_dir=image_dir)
return full_gpath_list
def TEST_DELETE_IMAGE_THUMBTUPS(ibs, back):
gpath_list = grabdata.get_test_gpaths(ndata=None)[0:4]
gid_list = ibs.add_images(gpath_list)
bbox_list = [(0, 0, 100, 100)]*len(gid_list)
name_list = ['a', 'b', 'a', 'd']
aid_list = ibs.add_annots(gid_list, bbox_list=bbox_list, name_list=name_list)
assert len(aid_list) !=0, "No annotations added"
thumbpath_list = ibs.get_image_thumbpath(gid_list)
gpath_list = ibs.get_image_paths(gid_list)
ibs.delete_image_thumbtups(gid_list)
assert utool.is_list(thumbpath_list), "thumbpath_list is not a list"
assert utool.is_list(gpath_list), "gpath_list is not a list"
for path in thumbpath_list:
assert not utool.checkpath(path), "Thumbnail not deleted"
for path in gpath_list:
utool.assertpath(path)
return locals()
def dump_autogen_code(fpath, autogen_text, codetype='python', fullprint=None):
"""
Helper that write a file if -w is given on command line, otherwise
it just prints it out. It has the opption of comparing a diff to the file.
"""
import utool as ut
dowrite = ut.get_argflag(('-w', '--write'))
show_diff = ut.get_argflag('--diff')
num_context_lines = ut.get_argval('--diff', type_=int, default=None)
show_diff = show_diff or num_context_lines is not None
num_context_lines = ut.get_argval('--diff', type_=int, default=None)
if fullprint is None:
fullprint = True
if fullprint is False:
fullprint = ut.get_argflag('--print')
print('[autogen] Autogenerated %s...\n+---\n' % (fpath,))
if not dowrite:
if fullprint:
ut.print_code(autogen_text, lexer_name=codetype)
print('\nL___')
else:
print('specify --print to write to stdout')
pass
print('specify -w to write, or --diff to compare')
print('...would write to: %s' % fpath)
if show_diff:
if ut.checkpath(fpath, verbose=True):
prev_text = ut.read_from(fpath)
textdiff = ut.get_textdiff(prev_text, autogen_text,
num_context_lines=num_context_lines)
try:
ut.print_difftext(textdiff)
except UnicodeDecodeError:
import unicodedata
textdiff = unicodedata.normalize('NFKD', textdiff).encode('ascii', 'ignore')
ut.print_difftext(textdiff)
if dowrite:
print('WARNING: Not writing. Remove --diff from command line')
elif dowrite:
ut.write_to(fpath, autogen_text)
def convert_pdf_to_image(pdf_fpath, ext='.jpg', verbose=1, dpi=300,
quality=90):
import utool as ut
if verbose:
print('[ut] convert_pdf_to_image.')
img_fpath = ut.ensure_ext(pdf_fpath, ext)
if ut.UNIX:
convert_fpath = ut.cmd2('which convert')['out'].strip()
if not convert_fpath:
raise Exception('ImageMagik convert was not found')
args = ' '.join(['convert', '-density', str(dpi), pdf_fpath, '-quality',
str(quality), img_fpath])
info = ut.cmd2(args, verbose=verbose > 1) # NOQA
if not ut.checkpath(img_fpath, verbose=verbose > 1):
print('Failed to convert pdf to ' + ext)
print(info['out'])
raise Exception('ImageMagik failed to convert pdf to ' + ext)
return img_fpath
def ensure_local_war(verbose=ut.NOT_QUIET):
"""
Ensures tomcat has been unpacked and the war is localized
CommandLine:
ibeis ensure_local_war
Example:
>>> # SCRIPT
>>> from ibeis.control.wildbook_manager import * # NOQA
>>> result = ensure_local_war()
>>> print(result)
"""
# TODO: allow custom specified tomcat directory
try:
output = subprocess.check_output(['java', '-version'],
stderr=subprocess.STDOUT)
_java_version = output.split('\n')[0]
_java_version = _java_version.replace('java version ', '')
java_version = _java_version.replace('"', '')
print('java_version = %r' % (java_version,))
if not java_version.startswith('1.7'):
print('Warning wildbook is only supported for java 1.7')
except OSError:
output = None
if output is None:
raise ImportError(
'Cannot find java on this machine. '
'Please install java: http://www.java.com/en/download/')
tomcat_dpath = find_or_download_tomcat()
assert tomcat_dpath is not None, 'Could not find tomcat'
redownload = ut.get_argflag('--redownload-war')
war_fpath = find_or_download_wilbook_warfile(redownload=redownload)
war_fname = basename(war_fpath)
# Move the war file to tomcat webapps if not there
webapps_dpath = join(tomcat_dpath, 'webapps')
deploy_war_fpath = join(webapps_dpath, war_fname)
if not ut.checkpath(deploy_war_fpath, verbose=verbose):
ut.copy(war_fpath, deploy_war_fpath)
wb_target = splitext(war_fname)[0]
return tomcat_dpath, webapps_dpath, wb_target
def ensure_local_war(verbose=ut.NOT_QUIET):
"""
Ensures tomcat has been unpacked and the war is localized
CommandLine:
ibeis ensure_local_war
Example:
>>> # SCRIPT
>>> from ibeis.control.wildbook_manager import * # NOQA
>>> result = ensure_local_war()
>>> print(result)
"""
# TODO: allow custom specified tomcat directory
try:
output = subprocess.check_output(['java', '-version'],
stderr=subprocess.STDOUT)
_java_version = output.split('\n')[0]
_java_version = _java_version.replace('java version ', '')
java_version = _java_version.replace('"', '')
print('java_version = %r' % (java_version, ))
if not java_version.startswith('1.7'):
print('Warning wildbook is only supported for java 1.7')
except OSError:
output = None
if output is None:
raise ImportError(
'Cannot find java on this machine. '
'Please install java: http://www.java.com/en/download/')
tomcat_dpath = find_or_download_tomcat()
assert tomcat_dpath is not None, 'Could not find tomcat'
redownload = ut.get_argflag('--redownload-war')
war_fpath = find_or_download_wilbook_warfile(redownload=redownload)
war_fname = basename(war_fpath)
# Move the war file to tomcat webapps if not there
webapps_dpath = join(tomcat_dpath, 'webapps')
deploy_war_fpath = join(webapps_dpath, war_fname)
if not ut.checkpath(deploy_war_fpath, verbose=verbose):
ut.copy(war_fpath, deploy_war_fpath)
wb_target = splitext(war_fname)[0]
return tomcat_dpath, webapps_dpath, wb_target
def publish(dstcnvs_normer, cachedir=None):
"""
Sets this as the default normalizer available for download
ONLY DEVELOPERS CAN PERFORM THIS OPERATION
Args:
cachedir (str):
CommandLine:
python -m wbia.algo.hots.distinctiveness_normalizer --test-publish
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.algo.hots.distinctiveness_normalizer import * # NOQA
>>> dstcnvs_normer = testdata_distinctiveness()[0]
>>> dstcnvs_normer.rebuild()
>>> dstcnvs_normer.save()
>>> result = dstcnvs_normer.publish(cachedir)
>>> # verify results
>>> print(result)
"""
from os.path import basename, join
assert ut.is_developer(), 'ONLY DEVELOPERS CAN PERFORM THIS OPERATION'
cachedir = dstcnvs_normer.cachedir if cachedir is None else cachedir
archive_fpath = dstcnvs_normer.archive(cachedir, overwrite=True)
archive_fname = basename(archive_fpath)
publish_dpath = PUBLISH_DIR
publish_fpath = join(publish_dpath, archive_fname)
if ut.checkpath(publish_fpath, verbose=True):
logger.info('Overwriting model')
logger.info(
'old nBytes(publish_fpath) = %s'
% (ut.get_file_nBytes_str(publish_fpath),)
)
logger.info(
'new nBytes(archive_fpath) = %s'
% (ut.get_file_nBytes_str(archive_fpath),)
)
else:
logger.info('Publishing model')
logger.info('publish_fpath = %r' % (publish_fpath,))
ut.copy(archive_fpath, publish_fpath)
def fix_importlib_hook():
""" IMPORTLIB FIX
References:
http://stackoverflow.com/questions/18596410/importerror-no-module-named-mpl-toolkits-with-maptlotlib-1-3-0-and-py2exe
"""
try:
dpath_ = importlib.import_module('mpl_toolkits').__path__
if isinstance(dpath_, (list, tuple)):
for dpath in dpath_:
fpath = join(dpath, '__init__.py')
break
else:
dpath = dpath_
if not ut.checkpath(dpath, verbose=True, info=True):
ut.touch(fpath)
except ImportError as ex:
ut.printex(ex, 'pip install mpl_toolkits?')
def fix_importlib_hook():
""" IMPORTLIB FIX
References:
http://stackoverflow.com/questions/18596410/importerror-no-module-named-mpl-toolkits-with-maptlotlib-1-3-0-and-py2exe
"""
try:
dpath_ = importlib.import_module('mpl_toolkits').__path__
if isinstance(dpath_, (list, tuple)):
for dpath in dpath_:
fpath = join(dpath, '__init__.py')
break
else:
dpath = dpath_
if not ut.checkpath(dpath, verbose=True, info=True):
ut.touch(fpath)
except ImportError as ex:
ut.printex(ex, 'pip install mpl_toolkits?')
def get_local_dist_packages_dir():
"""
Attempts to work around virtualenvs and find the system dist_pacakges.
Essentially this is implmenented as a lookuptable
"""
import utool as ut
if not ut.in_virtual_env():
# Non venv case
return get_site_packages_dir()
else:
candidates = []
if ut.LINUX:
candidates += [
'/usr/local/lib/python2.7/dist-packages',
]
else:
raise NotImplementedError()
for path in candidates:
if ut.checkpath(path):
return path
def load(nnindexer, cachedir=None, fpath=None, verbose=True):
r"""
Loads a cached flann neighbor indexer from disk (not the data)
"""
load_success = False
if fpath is None:
flann_fpath = nnindexer.get_fpath(cachedir)
else:
flann_fpath = fpath
nnindexer.flann_fpath = flann_fpath
if ut.checkpath(flann_fpath, verbose=verbose):
idx2_vec = nnindexer.idx2_vec
# Warning: Loading a FLANN index with old headers may silently fail.
try:
nnindexer.flann.load_index(flann_fpath, idx2_vec)
except (IOError, pyflann.FLANNException) as ex:
ut.printex(ex, '... cannot load nnindex flann', iswarning=True)
else:
load_success = True
return load_success
def get_local_dist_packages_dir():
"""
Attempts to work around virtualenvs and find the system dist_pacakges.
Essentially this is implmenented as a lookuptable
"""
import utool as ut
if not ut.in_virtual_env():
# Non venv case
return get_site_packages_dir()
else:
candidates = []
if ut.LINUX:
candidates += [
'/usr/local/lib/python2.7/dist-packages',
]
else:
raise NotImplementedError()
for path in candidates:
if ut.checkpath(path):
return path
def load(nnindexer, cachedir=None, fpath=None, verbose=True):
r"""
Loads a cached flann neighbor indexer from disk (not the data)
"""
load_success = False
if fpath is None:
flann_fpath = nnindexer.get_fpath(cachedir)
else:
flann_fpath = fpath
nnindexer.flann_fpath = flann_fpath
if ut.checkpath(flann_fpath, verbose=verbose):
idx2_vec = nnindexer.idx2_vec
# Warning: Loading a FLANN index with old headers may silently fail.
try:
nnindexer.flann.load_index(flann_fpath, idx2_vec)
except (IOError, pyflann.FLANNException) as ex:
ut.printex(ex, '... cannot load nnindex flann', iswarning=True)
else:
load_success = True
return load_success
def ensure_text(fname,
text,
repo_dpath='.',
force=None,
locals_={},
chmod=None):
"""
Args:
fname (str): file name
text (str):
repo_dpath (str): directory path string(default = '.')
force (bool): (default = False)
locals_ (dict): (default = {})
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_git import * # NOQA
>>> import utool as ut
>>> result = setup_repo()
>>> print(result)
"""
import utool as ut
ut.colorprint('Ensuring fname=%r' % (fname), 'yellow')
if force is None and ut.get_argflag('--force-%s' % (fname, )):
force = True
fpath = join(repo_dpath, fname)
if force or not ut.checkpath(fpath, verbose=2, n=5):
text_ = ut.remove_codeblock_syntax_sentinals(text)
fmtkw = locals_.copy()
fmtkw['fname'] = fname
text_ = text_.format(**fmtkw) + '\n'
ut.writeto(fpath, text_)
try:
if chmod:
ut.chmod(fpath, chmod)
except Exception as ex:
ut.printex(ex, iswarning=True)
def publish(dstcnvs_normer, cachedir=None):
"""
Sets this as the default normalizer available for download
ONLY DEVELOPERS CAN PERFORM THIS OPERATION
Args:
cachedir (str):
CommandLine:
python -m ibeis.algo.hots.distinctiveness_normalizer --test-publish
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.distinctiveness_normalizer import * # NOQA
>>> dstcnvs_normer = testdata_distinctiveness()[0]
>>> dstcnvs_normer.rebuild()
>>> dstcnvs_normer.save()
>>> result = dstcnvs_normer.publish(cachedir)
>>> # verify results
>>> print(result)
"""
from os.path import basename, join
assert ut.is_developer(), 'ONLY DEVELOPERS CAN PERFORM THIS OPERATION'
cachedir = dstcnvs_normer.cachedir if cachedir is None else cachedir
archive_fpath = dstcnvs_normer.archive(cachedir, overwrite=True)
archive_fname = basename(archive_fpath)
publish_dpath = PUBLISH_DIR
publish_fpath = join(publish_dpath, archive_fname)
if ut.checkpath(publish_fpath, verbose=True):
print('Overwriting model')
print('old nBytes(publish_fpath) = %s' %
(ut.get_file_nBytes_str(publish_fpath),))
print('new nBytes(archive_fpath) = %s' %
(ut.get_file_nBytes_str(archive_fpath),))
else:
print('Publishing model')
print('publish_fpath = %r' % (publish_fpath,))
ut.copy(archive_fpath, publish_fpath)
def read_thumb_size(thumb_path):
import vtool as vt
if VERBOSE_THUMB:
print('[ThumbDelegate] Reading thumb size')
# npimg = vt.imread(thumb_path, delete_if_corrupted=True)
# (height, width) = npimg.shape[0:2]
# del npimg
try:
width, height = vt.open_image_size(thumb_path)
except IOError as ex:
if ut.checkpath(thumb_path, verbose=True):
ut.printex(
ex,
'image=%r seems corrupted. Needs deletion' % (thumb_path, ),
iswarning=True,
)
ut.delete(thumb_path)
else:
ut.printex(ex,
'image=%r does not exist', (thumb_path, ),
iswarning=True)
raise
return width, height
def load_oxford_2007():
"""
Loads data from
http://www.robots.ox.ac.uk:5000/~vgg/publications/2007/Philbin07/philbin07.pdf
>>> from wbia.algo.smk.script_smk import * # NOQA
"""
from os.path import join, basename, splitext
import pandas as pd
import vtool as vt
dbdir = ut.truepath('/raid/work/Oxford/')
data_fpath0 = join(dbdir, 'data_2007.pkl')
if ut.checkpath(data_fpath0):
data = ut.load_data(data_fpath0)
return data
else:
word_dpath = join(dbdir, 'word_oxc1_hesaff_sift_16M_1M')
_word_fpath_list = ut.ls(word_dpath)
imgid_to_word_fpath = {
splitext(basename(word_fpath))[0]: word_fpath
for word_fpath in _word_fpath_list
}
readme_fpath = join(dbdir, 'README2.txt')
imgid_order = ut.readfrom(readme_fpath).split('\n')[20:-1]
imgid_order = imgid_order
data_uri_order = [x.replace('oxc1_', '') for x in imgid_order]
imgid_to_df = {}
for imgid in ut.ProgIter(imgid_order, label='reading kpts'):
word_fpath = imgid_to_word_fpath[imgid]
row_gen = (map(float,
line.strip('\n').split(' '))
for line in ut.read_lines_from(word_fpath)[2:])
rows = [(int(word_id), x, y, e11, e12, e22)
for (word_id, x, y, e11, e12, e22) in row_gen]
df = pd.DataFrame(
rows, columns=['word_id', 'x', 'y', 'e11', 'e12', 'e22'])
imgid_to_df[imgid] = df
df_list = ut.take(imgid_to_df, imgid_order)
nfeat_list = [len(df_) for df_ in df_list]
offset_list = [0] + ut.cumsum(nfeat_list)
shape = (offset_list[-1], 128)
# shape = (16334970, 128)
sift_fpath = join(dbdir, 'OxfordSIFTDescriptors',
'feat_oxc1_hesaff_sift.bin')
try:
file_ = open(sift_fpath, 'rb')
with ut.Timer('Reading SIFT binary file'):
nbytes = np.prod(shape)
all_vecs = np.fromstring(file_.read(nbytes), dtype=np.uint8)
all_vecs = all_vecs.reshape(shape)
finally:
file_.close()
kpts_list = [
df_.loc[:, ('x', 'y', 'e11', 'e12', 'e22')].values
for df_ in df_list
]
wordid_list = [df_.loc[:, 'word_id'].values for df_ in df_list]
kpts_Z = np.vstack(kpts_list)
idx_to_wx = np.hstack(wordid_list)
# assert len(np.unique(idx_to_wx)) == 1E6
# Reqd standard query order
query_files = sorted(
ut.glob(dbdir + '/oxford_groundtruth', '*_query.txt'))
query_uri_order = []
for qpath in query_files:
text = ut.readfrom(qpath, verbose=0)
query_uri = text.split(' ')[0].replace('oxc1_', '')
query_uri_order.append(query_uri)
logger.info('converting to invV')
all_kpts = vt.convert_kptsZ_to_kpts(kpts_Z)
data = {
'offset_list': offset_list,
'all_kpts': all_kpts,
'all_vecs': all_vecs,
'idx_to_wx': idx_to_wx,
'data_uri_order': data_uri_order,
'query_uri_order': query_uri_order,
}
ut.save_data(data_fpath0, data)
return data
def install_wildbook(verbose=ut.NOT_QUIET):
"""
Script to setup wildbook on a unix based system
(hopefully eventually this will generalize to win32)
CommandLine:
# Reset
ibeis purge_local_wildbook
ibeis ensure_wb_mysql
ibeis ensure_local_war
# Setup
ibeis install_wildbook
# ibeis install_wildbook --nomysql
# Startup
ibeis startup_wildbook_server --show
Alternates:
ibeis install_wildbook --redownload-war
ibeis install_wildbook --assets
ibeis startup_wildbook_server --show
Example:
>>> # SCRIPT
>>> from ibeis.control.wildbook_manager import * # NOQA
>>> verbose = True
>>> result = install_wildbook()
>>> print(result)
"""
import requests
# Ensure that the war file has been unpacked
tomcat_dpath, webapps_dpath, wb_target = ensure_local_war()
unpacked_war_dpath = join(webapps_dpath, wb_target)
tomcat_startup_dir = get_tomcat_startup_tmpdir()
fresh_install = not ut.checkpath(unpacked_war_dpath, verbose=verbose)
if fresh_install:
# Need to make sure you start catalina in the same directory otherwise
# the derby databsae gets put in in cwd
with ut.ChdirContext(tomcat_startup_dir):
# Starting and stoping catalina should be sufficient to unpack the
# war
startup_fpath = join(tomcat_dpath, 'bin', 'startup.sh')
#shutdown_fpath = join(tomcat_dpath, 'bin', 'shutdown.sh')
ut.cmd(ut.quote_single_command(startup_fpath))
print('It is NOT ok if the startup.sh fails\n')
# wait for the war to be unpacked
for retry_count in range(0, 6):
time.sleep(1)
if ut.checkpath(unpacked_war_dpath, verbose=True):
break
else:
print('Retrying')
# ensure that the server is ruuning
print('Checking if we can ping the server')
response = requests.get('http://localhost:8080')
if response is None or response.status_code != 200:
print('There may be an error starting the server')
else:
print('Seem able to ping the server')
# assert tht the war was unpacked
ut.assertpath(unpacked_war_dpath, (
'Wildbook war might have not unpacked correctly. This may '
'be ok. Try again. If it fails a second time, then there is a '
'problem.'),
verbose=True)
# Don't shutdown just yet. Need to create assets
update_wildbook_install_config(webapps_dpath, unpacked_war_dpath)
asset_flag_fpath = join(tomcat_startup_dir, 'made_assets.flag')
# Pinging the server to create asset store
# Ensureing that createAssetStore exists
if not ut.checkpath(asset_flag_fpath):
if not fresh_install:
startup_wildbook_server()
#web_url = startup_wildbook_server(verbose=False)
print('Creating asset store')
wb_url = 'http://localhost:8080/' + wb_target
response = requests.get(wb_url + '/createAssetStore.jsp')
if response is None or response.status_code != 200:
print('There may be an error starting the server')
#if response.status_code == 500:
print(response.text)
assert False, 'response error'
else:
print('Created asset store')
# Create file signaling we did this
ut.writeto(asset_flag_fpath, 'True')
shutdown_wildbook_server(verbose=False)
print('It is ok if the shutdown fails')
elif fresh_install:
shutdown_wildbook_server(verbose=False)
#127.0.0.1:8080/wildbook_data_dir/test.txt
print('Wildbook is installed and waiting to be started')
def ensure_smaller_testingdbs():
"""
Makes the smaller test databases
"""
def make_testdb0():
""" makes testdb0 """
def get_test_gpaths(ndata=None, names=None, **kwargs):
# Read ndata from args or command line
""" DEPRICATE """
ndata_arg = ut.get_argval(
'--ndata',
type_=int,
default=None,
help_='use --ndata to specify bigger data')
if ndata_arg is not None:
ndata = ndata_arg
imgdir = get_testdata_dir(**kwargs)
gpath_list = sorted(
list(ut.list_images(imgdir, full=True, recursive=True)))
# Get only the gpaths of certain names
if names is not None:
gpath_list = [
gpath for gpath in gpath_list
if ut.basename_noext(gpath) in names
]
# Get a some number of test images
if ndata is not None:
gpath_cycle = cycle(gpath_list)
if six.PY2:
gpath_list = [gpath_cycle.next() for _ in range(ndata)]
else:
gpath_list = [next(gpath_cycle) for _ in range(ndata)]
return gpath_list
workdir = ibeis.sysres.get_workdir()
TESTDB0 = join(workdir, 'testdb0')
main_locals = ibeis.main(dbdir=TESTDB0, gui=False, allow_newdir=True)
ibs = main_locals['ibs']
assert ibs is not None, str(main_locals)
gpath_list = list(map(ut.unixpath, get_test_gpaths()))
#print('[RESET] gpath_list=%r' % gpath_list)
gid_list = ibs.add_images(gpath_list) # NOQA
valid_gids = ibs.get_valid_gids()
valid_aids = ibs.get_valid_aids()
try:
assert len(
valid_aids
) == 0, 'there are more than 0 annotations in an empty database!'
except Exception as ex:
ut.printex(ex, key_list=['valid_aids'])
raise
gid_list = valid_gids[0:1]
bbox_list = [(0, 0, 100, 100)]
aid = ibs.add_annots(gid_list, bbox_list=bbox_list)[0]
#print('[RESET] NEW RID=%r' % aid)
aids = ibs.get_image_aids(gid_list)[0]
try:
assert aid in aids, ('bad annotation adder: aid = %r, aids = %r' %
(aid, aids))
except Exception as ex:
ut.printex(ex, key_list=['aid', 'aids'])
raise
get_testdata_dir(True)
if not ut.checkpath(join(ibeis.sysres.get_workdir(), 'testdb0'),
verbose=True):
print("\n\nMAKE TESTDB0\n\n")
make_testdb0()
if not ut.checkpath(join(ibeis.sysres.get_workdir(), 'testdb1'),
verbose=True):
print("\n\nMAKE TESTDB1\n\n")
ingest_database.ingest_standard_database('testdb1')
def _get_models(ibs,
species,
modeldir='default',
cfg_override=True,
verbose=VERBOSE_RF):
r"""
Args:
ibs (IBEISController): wbia controller object
species (?):
modeldir (str): (default = 'default')
cfg_override (bool): (default = True)
verbose (bool): verbosity flag(default = False)
Returns:
?: fpath_list
CommandLine:
python -m wbia.algo.detect.randomforest --test-_get_models
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.detect.randomforest import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='testdb1')
>>> species = wbia.const.TEST_SPECIES.ZEB_PLAIN
>>> modeldir = 'default'
>>> cfg_override = True
>>> verbose = False
>>> fpath_list = _get_models(ibs, species, modeldir, cfg_override, verbose)
>>> result = ('fpath_list = %s' % (str(fpath_list),))
>>> print(result)
"""
# with ut.embed_on_exception_context:
if cfg_override and len(ibs.cfg.detect_cfg.trees_path) > 0:
trees_path = ibs.cfg.detect_cfg.trees_path
else:
# Ensure all models downloaded and accounted for
assert (species is not None
), '[_get_models] Cannot detect without specifying a species'
grabmodels.ensure_models(modeldir=modeldir, verbose=verbose)
trees_path = grabmodels.get_species_trees_paths(species,
modeldir=modeldir)
# Load tree paths
if ut.checkpath(trees_path, verbose=verbose):
fpath_list = ut.ls(trees_path, '*.txt')
# direct = Directory(trees_path, include_extensions=['txt'])
# files = direct.files()
else:
# If the models do not exist, return None
fpath_list = None
if fpath_list is None or len(fpath_list) == 0:
msg = (ut.codeblock("""
[_get_models] Error loading trees, either directory or fpath_list not found
* trees_path = %r
* fpath_list = %r
* species = %r
* model_dir = %r
* cfg_override = %r
""") % (trees_path, fpath_list, species, modeldir, cfg_override))
raise AssertionError(msg)
return fpath_list
def ensure_pz_mtest_batchworkflow_test():
r"""
CommandLine:
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test --reset
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test --reset
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest_batchworkflow_test()
"""
import ibeis
ibeis.ensure_pz_mtest()
workdir = ibeis.sysres.get_workdir()
mtest_dbpath = join(workdir, 'PZ_MTEST')
source_dbdir = mtest_dbpath
dest_dbdir = join(workdir, 'PZ_BATCH_WORKFLOW_MTEST')
if ut.get_argflag('--reset'):
ut.delete(dest_dbdir)
if ut.checkpath(dest_dbdir):
return
else:
copy_ibeisdb(source_dbdir, dest_dbdir)
ibs = ibeis.opendb('PZ_BATCH_WORKFLOW_MTEST')
assert len(ibs.get_valid_aids()) == 119
assert len(ibs.get_valid_nids()) == 41
ibs.delete_all_imagesets()
aid_list = ibs.get_valid_aids()
unixtime_list = ibs.get_annot_image_unixtimes(aid_list)
untimed_aids = ut.compress(aid_list, [t == -1 for t in unixtime_list])
ibs.get_annot_groundtruth(untimed_aids, aid_list)
aids_list, nid_list = ibs.group_annots_by_name(aid_list)
hourdiffs_list = ibs.get_name_hourdiffs(nid_list)
imageset_aids_list = [[] for _ in range(4)]
imageset_idx = 0
for hourdiffs, aids in zip(hourdiffs_list, aids_list):
#import scipy.spatial.distance as spdist
if len(aids) == 1:
imageset_aids_list[imageset_idx].extend(aids)
imageset_idx = (imageset_idx + 1) % len(imageset_aids_list)
else:
for chunk in list(ut.ichunks(aids, 2)):
imageset_aids_list[imageset_idx].extend(chunk)
imageset_idx = (imageset_idx + 1) % len(imageset_aids_list)
#import vtool as vt
#import networkx as netx
#nodes = list(range(len(aids)))
#edges_pairs = vt.pdist_argsort(hourdiffs)
#edge_weights = -hourdiffs[hourdiffs.argsort()]
#netx_graph = make_netx_graph(edges_pairs, nodes, edge_weights)
#cut_edges = netx.minimum_edge_cut(netx_graph)
#netx_graph.remove_edges_from(cut_edges)
#components = list(netx.connected_components(netx_graph))
#components = ut.sortedby(components, list(map(len, components)), reverse=True)
#print(components)
#imageset_aids_list[0].extend(components[0])
#for compoment in components:
# TODO do max-nway cut
#day_diffs = spdist.squareform(hourdiffs) / 24.0
#print(ut.numpy_str(day_diffs, precision=2, suppress_small=True))
#import itertools
#compare_idxs = [(r, c) for r, c in itertools.product(range(len(aids)), range(len(aids))) if (c > r)]
#print(len(aids))
#def make_netx_graph(edges_pairs, nodes=None, edge_weights=None):
# import networkx as netx
# node_lbls = [('id_', 'int')]
# edge_lbls = [('weight', 'float')]
# edges = [(pair[0], pair[1], weight) for pair, weight in zip(edges_pairs, edge_weights)]
# print('make_netx_graph')
# # Make a graph between the chips
# netx_nodes = [(ntup[0], {key[0]: val for (key, val) in zip(node_lbls, ntup[1:])})
# for ntup in iter(zip(nodes))]
# netx_edges = [(etup[0], etup[1], {key[0]: val for (key, val) in zip(edge_lbls, etup[2:])})
# for etup in iter(edges)]
# netx_graph = netx.Graph()
# netx_graph.add_nodes_from(netx_nodes)
# netx_graph.add_edges_from(netx_edges)
# return netx_graph
# Group into imagesets based on old names
gids_list = ibs.unflat_map(ibs.get_annot_image_rowids, imageset_aids_list)
imgsetid_list = ibs.new_imagesets_from_images(gids_list) # NOQA
# Remove all names
ibs.delete_annot_nids(aid_list)
# FIX THIS TO POINT TO THE CORRECT DIRECTORY
# prefix = expanduser(join('~', 'Desktop'))
prefix = '/Volumes/EXTERNAL/BACKUPS/Dan_2014-03-26_Ol_Pejeta__100GB/Ol_pejeta_zebra_stuff__2GB/'
print("""
=====================
PROCESSING ACTIVITIES
=====================
""")
activities = {}
columns = [3, 9, 10, 11, 12, 13, 14, 15]
csv_fpath = join(prefix, 'OPC Zebra database all [exported]', 'csv')
activity_csv_fpath = join(csv_fpath, 'Group-Habitat-Activity table.csv')
exportedmdb_fpath = join(csv_fpath, 'Individual sightings.csv')
utool.checkpath(activity_csv_fpath, verbose=True)
utool.checkpath(exportedmdb_fpath, verbose=True)
with open(join(activity_csv_fpath), 'r') as file_:
lines = file_.read()
for line in lines.splitlines()[1:]:
line = [item.strip() for item in line.strip().split(',')]
_id = line[2]
if _id not in activities:
activities[_id] = [line[col] for col in columns]
originals = join(prefix, 'Ol_pejeta_zebra_photos2__1GB')
images = Directory(originals)
image_set = set(images.files())
print(images)
exts = []
def ensure_pz_mtest_mergesplit_test():
r"""
Make a test database for MERGE and SPLIT cases
CommandLine:
python -m ibeis.init.sysres --test-ensure_pz_mtest_mergesplit_test
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest_mergesplit_test()
"""
import ibeis
ibeis.ensure_pz_mtest()
workdir = ibeis.sysres.get_workdir()
mtest_dbpath = join(workdir, 'PZ_MTEST')
source_dbdir = mtest_dbpath
dest_dbdir = join(workdir, 'PZ_MERGESPLIT_MTEST')
if ut.get_argflag('--reset'):
ut.delete(dest_dbdir)
if ut.checkpath(dest_dbdir):
return
copy_ibeisdb(source_dbdir, dest_dbdir)
ibs = ibeis.opendb('PZ_MERGESPLIT_MTEST')
assert len(ibs.get_valid_aids()) == 119
assert len(ibs.get_valid_nids()) == 41
aid_list = ibs.get_valid_aids()
aids_list, nid_list = ibs.group_annots_by_name(aid_list)
num_aids = list(map(len, aids_list))
# num cases wanted
num_merge = 3
num_split = 1
num_combo = 1
# num inputs needed
num_merge_names = num_merge
num_split_names = num_split * 2
num_combo_names = num_combo * 3
total_names = num_merge_names + num_split_names + num_combo_names
modify_aids = ut.take(aids_list, ut.list_argsort(num_aids, reverse=True)[0:total_names])
merge_nids1 = ibs.make_next_nids(num_merge, location_text='XMERGE')
merge_nids2 = ibs.make_next_nids(num_merge, location_text='XMERGE')
split_nid = ibs.make_next_nids(num_split, location_text='XSPLIT')[0]
combo_nids = ibs.make_next_nids(num_combo * 2, location_text='XCOMBO')
# the first 3 become merge cases
#left = 0
#right = left + num_merge
for aids, nid1, nid2 in zip(modify_aids[0:3], merge_nids1, merge_nids2):
#ibs.get_annot_nids(aids)
aids_ = aids[::2]
ibs.set_annot_name_rowids(aids_, [nid1] * len(aids_))
ibs.set_annot_name_rowids(aids_, [nid2] * len(aids_))
# the next 2 become split cases
#left = right
#right = left + num_split_names
for aids in modify_aids[3:5]:
ibs.set_annot_name_rowids(aids, [split_nid] * len(aids))
#left = right
#right = left + num_combo_names
# The final 3 are a combination case
for aids in modify_aids[5:8]:
aids_even = aids[::2]
aids_odd = aids[1::2]
ibs.set_annot_name_rowids(aids_even, [combo_nids[0]] * len(aids_even))
ibs.set_annot_name_rowids(aids_odd, [combo_nids[1]] * len(aids_odd))
final_result = ibs.unflat_map(ibs.get_annot_nids, modify_aids)
print('final_result = %s' % (ut.list_str(final_result),))
def get_data_list():
r"""
CommandLine:
python ~/code/ibeis/_installers/ibeis_pyinstaller_data_helper.py --test-get_data_list
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis_pyinstaller_data_helper import * # NOQA
>>> result = get_data_list()
>>> DATATUP_LIST, BINARYTUP_LIST, iconfile = result
>>> print('DATATUP_LIST = ' + ut.list_str(DATATUP_LIST))
>>> print('BINARYTUP_LIST = ' + ut.list_str(BINARYTUP_LIST))
>>> print(len(DATATUP_LIST))
>>> print(len(BINARYTUP_LIST))
>>> print(iconfile)
"""
# Build data before running analysis for quick debugging
DATATUP_LIST = []
BINARYTUP_LIST = []
#import pyhesaff
#pyhesaff.HESAFF_CLIB.__LIB_FPATH__
#import pyrf
#pyrf.RF_CLIB.__LIB_FPATH__
# Hesaff
libhesaff_fname = 'libhesaff' + LIB_EXT
libhesaff_src = realpath(
join(root_dir, '..', 'hesaff', 'pyhesaff', libhesaff_fname))
libhesaff_dst = join(ibsbuild, 'pyhesaff', 'lib', libhesaff_fname)
DATATUP_LIST.append((libhesaff_dst, libhesaff_src))
# PyRF
libpyrf_fname = 'libpyrf' + LIB_EXT
libpyrf_src = realpath(join(root_dir, '..', 'pyrf', 'pyrf', libpyrf_fname))
libpyrf_dst = join(ibsbuild, 'pyrf', 'lib', libpyrf_fname)
DATATUP_LIST.append((libpyrf_dst, libpyrf_src))
# FLANN
libflann_fname = 'libflann' + LIB_EXT
#try:
# #import pyflann
# #pyflann.__file__
# #join(dirname(dirname(pyflann.__file__)), 'build')
#except ImportError as ex:
# print('PYFLANN IS NOT IMPORTABLE')
# raise
#if WIN32 or LINUX:
# FLANN
#libflann_src = join_SITE_PACKAGES('pyflann', 'lib', libflann_fname)
#libflann_dst = join(ibsbuild, libflann_fname)
#elif APPLE:
# # libflann_src = '/pyflann/lib/libflann.dylib'
# # libflann_dst = join(ibsbuild, libflann_fname)
# libflann_src = join_SITE_PACKAGES('pyflann', 'lib', libflann_fname)
# libflann_dst = join(ibsbuild, libflann_fname)
# This path is when pyflann was built using setup.py develop
libflann_src = realpath(
join(root_dir, '..', 'flann', 'build', 'lib', libflann_fname))
libflann_dst = join(ibsbuild, 'pyflann', 'lib', libflann_fname)
DATATUP_LIST.append((libflann_dst, libflann_src))
# VTool
vtool_libs = ['libsver']
for libname in vtool_libs:
lib_fname = libname + LIB_EXT
vtlib_src = realpath(join(root_dir, '..', 'vtool', 'vtool', lib_fname))
vtlib_dst = join(ibsbuild, 'vtool', lib_fname)
DATATUP_LIST.append((vtlib_dst, vtlib_src))
linux_lib_dpaths = [
'/usr/lib/x86_64-linux-gnu', '/usr/lib', '/usr/local/lib'
]
# OpenMP
if APPLE:
# BSDDB, Fix for the modules that PyInstaller needs and (for some reason)
# are not being added by PyInstaller
libbsddb_src = '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-dynload/_bsddb.so'
libbsddb_dst = join(ibsbuild, '_bsddb.so')
DATATUP_LIST.append((libbsddb_dst, libbsddb_src))
#libgomp_src = '/opt/local/lib/libgomp.dylib'
libgomp_src = '/opt/local/lib/gcc48/libgomp.dylib'
BINARYTUP_LIST.append(('libgomp.1.dylib', libgomp_src, 'BINARY'))
# very hack
libiomp_src = '/Users/bluemellophone/code/libomp_oss/exports/mac_32e/lib.thin/libiomp5.dylib'
BINARYTUP_LIST.append(('libiomp5.dylib', libiomp_src, 'BINARY'))
if LINUX:
libgomp_src = ut.search_in_dirs('libgomp.so.1', linux_lib_dpaths)
ut.assertpath(libgomp_src)
BINARYTUP_LIST.append(('libgomp.so.1', libgomp_src, 'BINARY'))
# MinGW
if WIN32:
mingw_root = r'C:\MinGW\bin'
mingw_dlls = [
'libgcc_s_dw2-1.dll', 'libstdc++-6.dll', 'libgomp-1.dll',
'pthreadGC2.dll'
]
for lib_fname in mingw_dlls:
lib_src = join(mingw_root, lib_fname)
lib_dst = join(ibsbuild, lib_fname)
DATATUP_LIST.append((lib_dst, lib_src))
# We need to add these 4 opencv libraries because pyinstaller does not find them.
#OPENCV_EXT = {'win32': '248.dll',
# 'darwin': '.2.4.dylib',
# 'linux2': '.so.2.4'}[PLATFORM]
target_cv_version = '3.0.0'
OPENCV_EXT = {
'win32': target_cv_version.replace('.', '') + '.dll',
'darwin': '.' + target_cv_version + '.dylib',
'linux2': '.so.' + target_cv_version
}[PLATFORM]
missing_cv_name_list = [
'libopencv_videostab',
'libopencv_superres',
'libopencv_stitching',
#'libopencv_gpu',
'libopencv_core',
'libopencv_highgui',
'libopencv_imgproc',
]
# Hack to find the appropriate opencv libs
for name in missing_cv_name_list:
fname = name + OPENCV_EXT
src = ''
dst = ''
if APPLE:
src = join('/opt/local/lib', fname)
elif LINUX:
#src = join('/usr/lib', fname)
src, tried = ut.search_in_dirs(fname,
linux_lib_dpaths,
strict=True,
return_tried=True)
elif WIN32:
if ut.get_computer_name() == 'Ooo':
src = join(r'C:/Program Files (x86)/OpenCV/x86/mingw/bin',
fname)
else:
src = join(root_dir, '../opencv/build/bin', fname)
dst = join(ibsbuild, fname)
# ut.assertpath(src)
DATATUP_LIST.append((dst, src))
##################################
# QT Gui dependencies
##################################
if APPLE:
walk_path = '/opt/local/Library/Frameworks/QtGui.framework/Versions/4/Resources/qt_menu.nib'
for root, dirs, files in os.walk(walk_path):
for lib_fname in files:
toc_src = join(walk_path, lib_fname)
toc_dst = join('qt_menu.nib', lib_fname)
DATATUP_LIST.append((toc_dst, toc_src))
##################################
# Documentation, Icons, and Web Assets
##################################
# Documentation
#userguide_dst = join('.', '_docs', 'IBEISUserGuide.pdf')
#userguide_src = join(root_dir, '_docs', 'IBEISUserGuide.pdf')
#DATATUP_LIST.append((userguide_dst, userguide_src))
# Icon File
ICON_EXT = {'darwin': '.icns', 'win32': '.ico', 'linux2': '.ico'}[PLATFORM]
iconfile = join('_installers', 'ibsicon' + ICON_EXT)
icon_src = join(root_dir, iconfile)
icon_dst = join(ibsbuild, iconfile)
DATATUP_LIST.append((icon_dst, icon_src))
print('[installer] Checking Data (preweb)')
try:
for (dst, src) in DATATUP_LIST:
assert ut.checkpath(
src, verbose=True), 'checkpath for src=%r failed' % (src, )
except Exception as ex:
ut.printex(
ex,
'Checking data failed DATATUP_LIST=' + ut.list_str(DATATUP_LIST))
raise
# Web Assets
INSTALL_WEB = True and not ut.get_argflag('--noweb')
if INSTALL_WEB:
web_root = join('ibeis', 'web/')
#walk_path = join(web_root, 'static')
#static_data = []
#for root, dirs, files in os.walk(walk_path):
# root2 = root.replace(web_root, '')
# for icon_fname in files:
# if '.DS_Store' not in icon_fname:
# toc_src = join(abspath(root), icon_fname)
# toc_dst = join(root2, icon_fname)
# static_data.append((toc_dst, toc_src))
#ut.get_list_column(static_data, 1) == ut.glob(walk_path, '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store'])
static_src_list = ut.glob(join(web_root, 'static'),
'*',
recursive=True,
with_dirs=False,
exclude_dirs=['.DS_Store'])
static_dst_list = [
relpath(src, join(root_dir, 'ibeis')) for src in static_src_list
]
static_data = zip(static_dst_list, static_src_list)
DATATUP_LIST.extend(static_data)
#walk_path = join(web_root, 'templates')
#template_data = []
#for root, dirs, files in os.walk(walk_path):
# root2 = root.replace(web_root, '')
# for icon_fname in files:
# if '.DS_Store' not in icon_fname:
# toc_src = join(abspath(root), icon_fname)
# toc_dst = join(root2, icon_fname)
# template_data.append((toc_dst, toc_src))
template_src_list = ut.glob(join(web_root, 'templates'),
'*',
recursive=True,
with_dirs=False,
exclude_dirs=['.DS_Store'])
template_dst_list = [
relpath(src, join(root_dir, 'ibeis')) for src in template_src_list
]
template_data = zip(template_dst_list, template_src_list)
DATATUP_LIST.extend(template_data)
print('[installer] Checking Data (postweb)')
try:
for (dst, src) in DATATUP_LIST:
assert ut.checkpath(
src, verbose=False), 'checkpath for src=%r failed' % (src, )
except Exception as ex:
ut.printex(
ex,
'Checking data failed DATATUP_LIST=' + ut.list_str(DATATUP_LIST))
raise
return DATATUP_LIST, BINARYTUP_LIST, iconfile
def ensure_flann(dstcnvs_normer, cachedir=None):
if not ut.checkpath(dstcnvs_normer.get_flann_fpath(cachedir)):
dstcnvs_normer.rebuild(cachedir)
dstcnvs_normer.save_flann(cachedir)
def autogenerate_nth_schema_version(schema_spec, n=-1):
r"""
dumps, prints, or diffs autogen schema based on command line
Args:
n (int):
CommandLine:
python -m ibeis.control._sql_helpers --test-autogenerate_nth_schema_version
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.control._sql_helpers import * # NOQA
>>> from ibeis.control import DB_SCHEMA
>>> # build test data
>>> schema_spec = DB_SCHEMA
>>> n = 1
>>> # execute function
>>> tablename = autogenerate_nth_schema_version(schema_spec, n)
>>> # verify results
>>> result = str(tablename)
>>> print(result)
"""
import utool as ut
print('[_SQL] AUTOGENERATING CURRENT SCHEMA')
db = get_nth_test_schema_version(schema_spec, n=n)
# Auto-generate the version skip schema file
schema_spec_dir, schema_spec_fname = split(schema_spec.__file__)
schema_spec_fname = splitext(schema_spec_fname)[0]
# HACK TO GET AUTOGEN COMMAND
# FIXME: Make this autogen command a bit more sane and not completely
# coupled with ibeis
autogen_cmd = ut.codeblock(
'''
python -m ibeis.control.{schema_spec_fname} --test-autogen_{funcname} --force-incremental-db-update --write
python -m ibeis.control.{schema_spec_fname} --test-autogen_{funcname} --force-incremental-db-update --diff=1
python -m ibeis.control.{schema_spec_fname} --test-autogen_{funcname} --force-incremental-db-update
'''
).format(schema_spec_fname=schema_spec_fname, funcname=schema_spec_fname.lower())
autogen_text = db.get_schema_current_autogeneration_str(autogen_cmd)
autogen_fname = '%s_CURRENT.py' % schema_spec_fname
autogen_fpath = join(schema_spec_dir, autogen_fname)
dowrite = ut.get_argflag(('-w', '--write', '--dump-autogen-schema'))
show_diff = ut.get_argflag('--diff')
num_context_lines = ut.get_argval('--diff', type_=int, default=None)
show_diff = show_diff or num_context_lines is not None
dowrite = dowrite and not show_diff
if dowrite:
ut.write_to(autogen_fpath, autogen_text)
else:
if show_diff:
if ut.checkpath(autogen_fpath, verbose=True):
prev_text = ut.read_from(autogen_fpath)
textdiff = ut.util_str.get_textdiff(prev_text, autogen_text, num_context_lines=num_context_lines)
ut.print_difftext(textdiff)
else:
ut.util_print.print_python_code(autogen_text)
print('\nL___\n...would write to: %s' % autogen_fpath)
print(' Run with -n=%r to get a specific schema version by index. -1 == latest')
print(' Run with --write to autogenerate latest schema version')
print(' Run with --diff or --diff=<numcontextlines> to see the difference between current and requested')
return db
def update_schema_version(ibs, db, schema_spec, version, version_target,
dobackup=True, clearbackup=False):
"""
version_target = version_expected
clearbackup = False
FIXME: AN SQL HELPER FUNCTION SHOULD BE AGNOSTIC TO CONTROLER OBJECTS
"""
def _check_superkeys():
all_tablename_list = db.get_table_names()
# always ignore the metadata table.
ignore_tables_ = ['metadata']
tablename_list = [tablename for tablename in all_tablename_list
if tablename not in ignore_tables_]
for tablename in tablename_list:
superkey_colnames_list = db.get_table_superkey_colnames(tablename)
# some tables seem to only have old constraints and aren't
# properly updated to superkeys... weird.
old_constraints = db.get_table_constraints(tablename)
assert len(superkey_colnames_list) > 0 or len(old_constraints) > 0, (
'ERROR UPDATING DATABASE, SUPERKEYS of %s DROPPED!' % (tablename,))
print('[_SQL] update_schema_version')
db_fpath = db.fpath
if dobackup:
db_dpath, db_fname = split(db_fpath)
db_fname_noext, ext = splitext(db_fname)
db_backup_fname = ''.join((db_fname_noext, '_backup', '_v', version, ext))
db_backup_fpath = join(db_dpath, db_backup_fname)
count = 0
# TODO MAKE UTOOL THAT DOES THIS (there might be one in util_logging)
while ut.checkpath(db_backup_fpath, verbose=True):
db_backup_fname = ''.join((db_fname_noext, '_backup', '_v',
version, '_copy', str(count), ext))
db_backup_fpath = join(db_dpath, db_backup_fname)
count += 1
ut.copy(db_fpath, db_backup_fpath)
legacy_update_funcs = schema_spec.LEGACY_UPDATE_FUNCTIONS
for legacy_version, func in legacy_update_funcs:
if compare_string_versions(version, legacy_version) == -1:
func(db)
db_versions = schema_spec.VALID_VERSIONS
valid_versions = sorted(db_versions.keys(), compare_string_versions)
try:
start_index = valid_versions.index(version) + 1
except IndexError:
raise AssertionError('[!update_schema_version]'
' The current database version is unknown')
try:
end_index = valid_versions.index(version_target) + 1
except IndexError:
raise AssertionError('[!update_schema_version]'
' The target database version is unknown')
try:
print('Update path: %r ' % (valid_versions[start_index:end_index]))
for index in range(start_index, end_index):
next_version = valid_versions[index]
print('Updating database to version: %r' % (next_version))
pre, update, post = db_versions[next_version]
if pre is not None:
pre(db, ibs=ibs)
if update is not None:
update(db, ibs=ibs)
if post is not None:
post(db, ibs=ibs)
_check_superkeys()
except Exception as ex:
if dobackup:
msg = 'The database update failed, rolled back to the original version.'
ut.printex(ex, msg, iswarning=True)
ut.remove_file(db_fpath)
ut.copy(db_backup_fpath, db_fpath)
if clearbackup:
ut.remove_file(db_backup_fpath)
raise
else:
ut.printex(ex, (
'The database update failed, and no backup was made.'),
iswarning=False)
raise
if dobackup and clearbackup:
ut.remove_file(db_backup_fpath)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
script to open directory in current window manager
"""
import utool as ut
if __name__ == '__main__':
import sys
if len(sys.argv) == 2:
path = sys.argv[1]
else:
path = None
ut.assertpath(path)
if ut.checkpath(path, verbose=True):
ut.view_directory(path)
#F:\\data\\work\\PZ_MTEST\\_ibsdb\\
def grab_liberty_siam_dataset(pairs=250000):
"""
References:
http://www.cs.ubc.ca/~mbrown/patchdata/patchdata.html
https://github.com/osdf/datasets/blob/master/patchdata/dataset.py
Notes:
"info.txt" contains the match information Each row of info.txt
corresponds corresponds to a separate patch, with the patches ordered
from left to right and top to bottom in each bitmap image.
3 types of metadata files
info.txt - contains patch ids that correspond with the order of patches
in the bmp images
In the format:
pointid, unused
interest.txt -
interest points corresponding to patches with patchids
has same number of rows as info.txt
In the format:
reference image id, x, y, orientation, scale (in log2 units)
m50_<d>_<d>_0.txt -
matches files
patchID1 3DpointID1 unused1 patchID2 3DpointID2 unused2
CommandLine:
python -m ibeis_cnn.ingest_data --test-grab_liberty_siam_dataset --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis_cnn.ingest_data import * # NOQA
>>> pairs = 500
>>> dataset = grab_liberty_siam_dataset(pairs)
>>> ut.quit_if_noshow()
>>> from ibeis_cnn import draw_results
>>> #ibsplugin.rrr()
>>> flat_metadata = {}
>>> data, labels = dataset.subset('full')
>>> ut.quit_if_noshow()
>>> warped_patch1_list = data[::2]
>>> warped_patch2_list = data[1::2]
>>> dataset.interact()
>>> ut.show_if_requested()
"""
datakw = {
'detector': 'dog',
'pairs': pairs,
}
assert datakw['detector'] in ['dog', 'harris']
assert pairs in [500, 50000, 100000, 250000]
liberty_urls = {
'dog': 'http://www.cs.ubc.ca/~mbrown/patchdata/liberty.zip',
'harris': 'http://www.cs.ubc.ca/~mbrown/patchdata/liberty_harris.zip',
}
url = liberty_urls[datakw['detector']]
ds_path = ut.grab_zipped_url(url)
ds_name = splitext(basename(ds_path))[0]
alias_key = 'liberty;' + ut.dict_str(datakw, nl=False, explicit=True)
cfgstr = ','.join([str(val) for key, val in ut.iteritems_sorted(datakw)])
# TODO: allow a move of the base data prefix
training_dpath = ut.ensure_app_resource_dir('ibeis_cnn', 'training',
ds_name)
if ut.get_argflag('--vtd'):
ut.vd(training_dpath)
ut.ensuredir(training_dpath)
data_fpath = join(training_dpath, 'liberty_data_' + cfgstr + '.pkl')
labels_fpath = join(training_dpath, 'liberty_labels_' + cfgstr + '.pkl')
if not ut.checkpath(data_fpath, verbose=True):
data, labels = ingest_helpers.extract_liberty_style_patches(
ds_path, pairs)
ut.save_data(data_fpath, data)
ut.save_data(labels_fpath, labels)
# hack for caching num_labels
labels = ut.load_data(labels_fpath)
num_labels = len(labels)
dataset = DataSet.new_training_set(
alias_key=alias_key,
data_fpath=data_fpath,
labels_fpath=labels_fpath,
metadata_fpath=None,
training_dpath=training_dpath,
data_shape=(64, 64, 1),
data_per_label=2,
output_dims=1,
num_labels=num_labels,
)
return dataset
def archive_files(archive_fpath, fpath_list, small=True, allowZip64=False,
overwrite=False, verbose=True, common_prefix=False):
r"""
Adds the files in `fpath_list` to an zip/tar archive.
Args:
archive_fpath (str): path to zipfile to create
fpath_list (list): path of files to add to the zipfile
small (bool): if True uses compression but the zipfile will take more
time to write
allowZip64 (bool): use if a file is over 2GB
overwrite (bool):
verbose (bool): verbosity flag(default = True)
common_prefix (bool): (default = False)
References:
https://docs.python.org/2/library/zipfile.html
CommandLine:
python -m utool.util_grabdata --test-archive_files
Example:
>>> # SLOW_DOCTEST
>>> from utool.util_grabdata import * # NOQA
>>> import utool as ut
>>> archive_fpath = ut.get_app_resource_dir('utool', 'testarchive.zip')
>>> # remove an existing test archive
>>> ut.delete(archive_fpath)
>>> assert not exists(archive_fpath), 'archive should not exist'
>>> fpath_list = [ut.grab_test_imgpath(key) for key in ut.TESTIMG_URL_DICT]
>>> small = True
>>> allowZip64 = False
>>> overwrite = True
>>> result = archive_files(archive_fpath, fpath_list, small, allowZip64, overwrite)
>>> # verify results
>>> print(result)
>>> assert exists(archive_fpath), 'archive should exist'
Ignore:
# http://superuser.com/questions/281573/best-options-compressing-files-7-zip
# Create a small 7zip archive
7z a -t7z -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on archive.7z dir1
7z a -t7z -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on ibeis-linux-binary.7z ibeis
# Create a small zip archive
7za a -mm=Deflate -mfb=258 -mpass=15 -r ibeis-linux-binary.zip ibeis
"""
import utool as ut
from os.path import relpath, dirname
if not overwrite and ut.checkpath(archive_fpath, verbose=True):
raise AssertionError('cannot overrwite archive_fpath=%r' % (archive_fpath,))
print('Archiving %d files' % len(fpath_list))
compression = zipfile.ZIP_DEFLATED if small else zipfile.ZIP_STORED
if common_prefix:
# Note: common prefix does not care about file structures
if isinstance(common_prefix, six.string_types):
# use given path as base path
rel_arcpath = common_prefix
else:
rel_arcpath = commonprefix(fpath_list)
rel_arcpath = ut.longest_existing_path(rel_arcpath)
else:
rel_arcpath = dirname(archive_fpath)
with zipfile.ZipFile(archive_fpath, 'w', compression, allowZip64) as myzip:
for fpath in ut.ProgressIter(fpath_list, lbl='archiving files',
enabled=verbose, adjust=True):
arcname = relpath(fpath, rel_arcpath)
myzip.write(fpath, arcname)
def ggr_random_name_splits():
"""
CommandLine:
python -m wbia.viz.viz_graph2 ggr_random_name_splits --show
Ignore:
sshfs -o idmap=user lev:/ ~/lev
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.viz.viz_graph2 import * # NOQA
>>> ggr_random_name_splits()
"""
import wbia.guitool as gt
gt.ensure_qtapp()
# nid_list = ibs.get_valid_nids(filter_empty=True)
import wbia
dbdir = '/media/danger/GGR/GGR-IBEIS'
dbdir = (dbdir if ut.checkpath(dbdir) else
ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS'))
ibs = wbia.opendb(dbdir=dbdir, allow_newdir=False)
import datetime
day1 = datetime.date(2016, 1, 30)
day2 = datetime.date(2016, 1, 31)
orig_filter_kw = {
'multiple': None,
# 'view': ['right'],
# 'minqual': 'good',
'is_known': True,
'min_pername': 2,
}
orig_aids = ibs.filter_annots_general(filter_kw=ut.dict_union(
orig_filter_kw,
{
'min_unixtime':
ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)),
'max_unixtime':
ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)),
},
))
orig_all_annots = ibs.annots(orig_aids)
orig_unique_nids, orig_grouped_annots_ = orig_all_annots.group(
orig_all_annots.nids)
# Ensure we get everything
orig_grouped_annots = [
ibs.annots(aids_) for aids_ in ibs.get_name_aids(orig_unique_nids)
]
# pip install quantumrandom
if False:
import quantumrandom
data = quantumrandom.uint16()
seed = data.sum()
print('seed = %r' % (seed, ))
# import Crypto.Random
# from Crypto import Random
# quantumrandom.get_data()
# StrongRandom = Crypto.Random.random.StrongRandom
# aes.reseed(3340258)
# chars = [str(chr(x)) for x in data.view(np.uint8)]
# aes_seed = str('').join(chars)
# aes = Crypto.Random.Fortuna.FortunaGenerator.AESGenerator()
# aes.reseed(aes_seed)
# aes.pseudo_random_data(10)
orig_rand_idxs = ut.random_indexes(len(orig_grouped_annots), seed=3340258)
orig_sample_size = 75
random_annot_groups = ut.take(orig_grouped_annots, orig_rand_idxs)
orig_annot_sample = random_annot_groups[:orig_sample_size]
# OOOPS MADE ERROR REDO ----
filter_kw = {
'multiple': None,
'view': ['right'],
'minqual': 'good',
'is_known': True,
'min_pername': 2,
}
filter_kw_ = ut.dict_union(
filter_kw,
{
'min_unixtime':
ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)),
'max_unixtime':
ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)),
},
)
refiltered_sample = [
ibs.filter_annots_general(annot.aids, filter_kw=filter_kw_)
for annot in orig_annot_sample
]
is_ok = np.array(ut.lmap(len, refiltered_sample)) >= 2
ok_part_orig_sample = ut.compress(orig_annot_sample, is_ok)
ok_part_orig_nids = [x.nids[0] for x in ok_part_orig_sample]
# Now compute real sample
aids = ibs.filter_annots_general(filter_kw=filter_kw_)
all_annots = ibs.annots(aids)
unique_nids, grouped_annots_ = all_annots.group(all_annots.nids)
grouped_annots = grouped_annots_
# Ensure we get everything
# grouped_annots = [ibs.annots(aids_) for aids_ in ibs.get_name_aids(unique_nids)]
pop = len(grouped_annots)
pername_list = ut.lmap(len, grouped_annots)
groups = wbia.annots.AnnotGroups(grouped_annots, ibs)
match_tags = [ut.unique(ut.flatten(t)) for t in groups.match_tags]
tag_case_hist = ut.dict_hist(ut.flatten(match_tags))
print('name_pop = %r' % (pop, ))
print('Annots per Multiton Name' +
ut.repr3(ut.get_stats(pername_list, use_median=True)))
print('Name Tag Hist ' + ut.repr3(tag_case_hist))
print('Percent Photobomb: %.2f%%' %
(tag_case_hist['photobomb'] / pop * 100))
print('Percent Split: %.2f%%' % (tag_case_hist['splitcase'] / pop * 100))
# Remove the ok part from this sample
remain_unique_nids = ut.setdiff(unique_nids, ok_part_orig_nids)
remain_grouped_annots = [
ibs.annots(aids_) for aids_ in ibs.get_name_aids(remain_unique_nids)
]
sample_size = 75
import vtool as vt
vt.calc_sample_from_error_bars(0.05, pop, conf_level=0.95, prior=0.05)
remain_rand_idxs = ut.random_indexes(len(remain_grouped_annots),
seed=3340258)
remain_sample_size = sample_size - len(ok_part_orig_nids)
remain_random_annot_groups = ut.take(remain_grouped_annots,
remain_rand_idxs)
remain_annot_sample = remain_random_annot_groups[:remain_sample_size]
annot_sample_nofilter = ok_part_orig_sample + remain_annot_sample
# Filter out all bad parts
annot_sample_filter = [
ibs.annots(ibs.filter_annots_general(annot.aids, filter_kw=filter_kw_))
for annot in annot_sample_nofilter
]
annot_sample = annot_sample_filter
win = None
from wbia.viz import viz_graph2
for annots in ut.InteractiveIter(annot_sample):
if win is not None:
win.close()
win = viz_graph2.make_qt_graph_interface(ibs,
aids=annots.aids,
init_mode='rereview')
print(win)
sample_groups = wbia.annots.AnnotGroups(annot_sample, ibs)
flat_tags = [ut.unique(ut.flatten(t)) for t in sample_groups.match_tags]
print('Using Split and Photobomb')
is_positive = ['photobomb' in t or 'splitcase' in t for t in flat_tags]
num_positive = sum(is_positive)
vt.calc_error_bars_from_sample(sample_size,
num_positive,
pop,
conf_level=0.95)
print('Only Photobomb')
is_positive = ['photobomb' in t for t in flat_tags]
num_positive = sum(is_positive)
vt.calc_error_bars_from_sample(sample_size,
num_positive,
pop,
conf_level=0.95)
print('Only SplitCase')
is_positive = ['splitcase' in t for t in flat_tags]
num_positive = sum(is_positive)
vt.calc_error_bars_from_sample(sample_size,
num_positive,
pop,
conf_level=0.95)
def ensure_pz_mtest_mergesplit_test():
r"""
Make a test database for MERGE and SPLIT cases
CommandLine:
python -m ibeis.init.sysres --test-ensure_pz_mtest_mergesplit_test
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest_mergesplit_test()
"""
import ibeis
ibeis.ensure_pz_mtest()
workdir = ibeis.sysres.get_workdir()
mtest_dbpath = join(workdir, 'PZ_MTEST')
source_dbdir = mtest_dbpath
dest_dbdir = join(workdir, 'PZ_MERGESPLIT_MTEST')
if ut.get_argflag('--reset'):
ut.delete(dest_dbdir)
if ut.checkpath(dest_dbdir):
return
copy_ibeisdb(source_dbdir, dest_dbdir)
ibs = ibeis.opendb('PZ_MERGESPLIT_MTEST')
assert len(ibs.get_valid_aids()) == 119
assert len(ibs.get_valid_nids()) == 41
aid_list = ibs.get_valid_aids()
aids_list, nid_list = ibs.group_annots_by_name(aid_list)
num_aids = list(map(len, aids_list))
# num cases wanted
num_merge = 3
num_split = 1
num_combo = 1
# num inputs needed
num_merge_names = num_merge
num_split_names = num_split * 2
num_combo_names = num_combo * 3
total_names = num_merge_names + num_split_names + num_combo_names
modify_aids = ut.take(
aids_list,
ut.list_argsort(num_aids, reverse=True)[0:total_names])
merge_nids1 = ibs.make_next_nids(num_merge, location_text='XMERGE')
merge_nids2 = ibs.make_next_nids(num_merge, location_text='XMERGE')
split_nid = ibs.make_next_nids(num_split, location_text='XSPLIT')[0]
combo_nids = ibs.make_next_nids(num_combo * 2, location_text='XCOMBO')
# the first 3 become merge cases
#left = 0
#right = left + num_merge
for aids, nid1, nid2 in zip(modify_aids[0:3], merge_nids1, merge_nids2):
#ibs.get_annot_nids(aids)
aids_ = aids[::2]
ibs.set_annot_name_rowids(aids_, [nid1] * len(aids_))
ibs.set_annot_name_rowids(aids_, [nid2] * len(aids_))
# the next 2 become split cases
#left = right
#right = left + num_split_names
for aids in modify_aids[3:5]:
ibs.set_annot_name_rowids(aids, [split_nid] * len(aids))
#left = right
#right = left + num_combo_names
# The final 3 are a combination case
for aids in modify_aids[5:8]:
aids_even = aids[::2]
aids_odd = aids[1::2]
ibs.set_annot_name_rowids(aids_even, [combo_nids[0]] * len(aids_even))
ibs.set_annot_name_rowids(aids_odd, [combo_nids[1]] * len(aids_odd))
final_result = ibs.unflat_map(ibs.get_annot_nids, modify_aids)
print('final_result = %s' % (ut.list_str(final_result), ))
