def _preload(mpl=True, par=True, logging=True):
""" Sets up python environment """
import utool as ut
# from wbia.init import main_helpers
# params.parse_args()
# from wbia.init import sysres
if multiprocessing.current_process().name != 'MainProcess':
return
if ut.VERBOSE:
logger.info('[wbia] _preload')
params.parse_args()
# mpl backends
# if logging and not params.args.nologging:
# if params.args.logdir is not None:
# sysres.set_logdir(params.args.logdir)
# else:
# # Log in the configured wbia log dir (which is maintained by utool)
# # fix this to be easier to figure out where the logs actually are
# ut.start_logging(appname='wbia')
if mpl:
_init_matplotlib()
# numpy print settings
_init_numpy()
# parallel servent processes
if par:
_init_parallel()
# ctrl+c
_init_signals()
# inject colored exceptions
ut.util_inject.inject_colored_exceptions()
def _init_wbia(dbdir=None, verbose=None, use_cache=True, web=None, **kwargs):
"""
Private function that calls code to create an wbia controller
"""
import utool as ut
params.parse_args()
from wbia.control import IBEISControl
# Set up logging
# TODO (30-Nov-12020) This is intended to be a temporary fix to logging.
# logger.setLevel(logging.DEBUG)
# logger.addHandler(logging.StreamHandler())
if verbose is None:
verbose = ut.VERBOSE
if verbose and NOT_QUIET:
logger.info('[main] _init_wbia()')
# Use command line dbdir unless user specifies it
if dbdir is None:
ibs = None
logger.info('[main!] WARNING: args.dbdir is None')
else:
kwargs = kwargs.copy()
request_dbversion = kwargs.pop('request_dbversion', None)
force_serial = kwargs.get('force_serial', None)
ibs = IBEISControl.request_IBEISController(
dbdir=dbdir,
use_cache=use_cache,
request_dbversion=request_dbversion,
force_serial=force_serial,
)
# BBB (12-Jan-12021) daily database backup for the sqlite database
if not ibs.is_using_postgres_db:
ibs.daily_backup_database()
if web is None:
web = ut.get_argflag(
('--webapp', '--webapi', '--web', '--browser'),
help_='automatically launch the web app / web api',
)
# web = params.args.webapp
if web:
from wbia.web import app
port = params.args.webport
app.start_from_wbia(ibs, port=port, **kwargs)
return ibs
def _init_parallel():
import utool as ut
if ut.VERBOSE:
logger.info('_init_parallel')
from utool import util_parallel
params.parse_args()
# Import any modules which parallel process will use here
# so they are accessable when the program forks
# from utool import util_sysreq
# util_sysreq.ensure_in_pythonpath('hesaff')
# util_sysreq.ensure_in_pythonpath('pyrf')
# util_sysreq.ensure_in_pythonpath('code')
# import pyhesaff # NOQA
# import pyrf # NOQA
from wbia import core_annots # NOQA
# .algo.preproc import preproc_chip # NOQA
util_parallel.set_num_procs(params.args.num_procs)
def _init_wbia(dbdir=None, verbose=None, use_cache=True, web=None, **kwargs):
"""
Private function that calls code to create an wbia controller
"""
import utool as ut
params.parse_args()
from wbia.control import IBEISControl
if verbose is None:
verbose = ut.VERBOSE
if verbose and NOT_QUIET:
logger.info('[main] _init_wbia()')
# Use command line dbdir unless user specifies it
if dbdir is None:
ibs = None
logger.info('[main!] WARNING: args.dbdir is None')
else:
kwargs = kwargs.copy()
request_dbversion = kwargs.pop('request_dbversion', None)
force_serial = kwargs.get('force_serial', None)
ibs = IBEISControl.request_IBEISController(
dbdir=dbdir,
use_cache=use_cache,
request_dbversion=request_dbversion,
force_serial=force_serial,
)
if web is None:
web = ut.get_argflag(
('--webapp', '--webapi', '--web', '--browser'),
help_='automatically launch the web app / web api',
)
# web = params.args.webapp
if web:
from wbia.web import app
port = params.args.webport
app.start_from_wbia(ibs, port=port, **kwargs)
return ibs
def main_loop(main_locals, rungui=True, ipy=False, persist=True):
"""
Runs the qt loop if the GUI was initialized and returns an executable string
for embedding an IPython terminal if requested.
If rungui is False the gui will not loop even if back has been created
the main locals dict must be callsed main_locals in the scope you call this
function in.
Args:
main_locals (dict_):
rungui (bool):
ipy (bool):
persist (bool):
Returns:
str: execstr
"""
logger.info('[main] wbia.entry_points.main_loop()')
params.parse_args()
import utool as ut
# logger.info('current process = %r' % (multiprocessing.current_process().name,))
# == 'MainProcess':
if rungui and not params.args.nogui:
try:
_guitool_loop(main_locals, ipy=ipy)
except Exception as ex:
ut.printex(ex, 'error in main_loop')
raise
# if not persist or params.args.cmd:
# main_close()
# Put locals in the exec namespace
ipycmd_execstr = ut.ipython_execstr()
locals_execstr = ut.execstr_dict(main_locals, 'main_locals')
execstr = locals_execstr + '\n' + ipycmd_execstr
return execstr
def preload_commands(dbdir, **kwargs):
""" Preload commands work with command line arguments and global caches """
# logger.info('[main_cmd] preload_commands')
params.parse_args()
if params.args.dump_argv:
logger.info(ut.repr2(vars(params.args), sorted_=False))
if params.args.dump_global_cache:
ut.global_cache_dump() # debug command, dumps to stdout
if params.args.set_workdir is not None:
sysres.set_workdir(params.args.set_workdir)
if params.args.get_workdir:
logger.info(' Current work dir = %s' % sysres.get_workdir())
# if params.args.logdir is not None:
# sysres.set_logdir(params.args.logdir)
if params.args.get_logdir:
logger.info(' Current local log dir = %s' %
(sysres.get_logdir_local(), ))
logger.info(' Current global log dir = %s' %
(sysres.get_logdir_global(), ))
if params.args.view_logdir:
ut.view_directory(sysres.get_logdir_local())
ut.view_directory(sysres.get_logdir_global())
if params.args.view_logdir_local:
ut.view_directory(sysres.get_logdir_local())
if params.args.view_logdir_global:
ut.view_directory(sysres.get_logdir_local())
if ut.get_argflag('--vwd'):
vwd()
if ut.get_argflag('--vdq'):
logger.info('got arg --vdq')
vdq(dbdir)
if kwargs.get('delete_ibsdir', False):
ibsfuncs.delete_wbia_database(dbdir)
if params.args.preload_exit:
logger.info('[main_cmd] preload exit')
sys.exit(0)
def ensure_correct_version(ibs,
db,
version_expected,
schema_spec,
dobackup=True,
verbose=ut.NOT_QUIET):
"""
FIXME: AN SQL HELPER FUNCTION SHOULD BE AGNOSTIC TO CONTROLER OBJECTS
ensure_correct_version
Args:
ibs (IBEISController):
db (SQLController):
version_expected (str): version you want to be at
schema_spec (module): schema module
dobackup (bool):
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.control._sql_helpers import * # NOQA
>>> ibs = '?'
>>> db = ibs.db
>>> version_expected = ibs.db_version_expected
>>> schema_spec = DB_SCHEMA
>>> dobackup = True
>>> result = ensure_correct_version(ibs, db, version_expected, schema_spec, dobackup)
>>> print(result)
Args:
schema_spec (module): module of schema specifications
"""
from wbia import constants as const
from wbia import params
params.parse_args()
# logger.info('[SQL_] ensure_correct_version')
force_incremental = params.args.force_incremental_db_update
want_base_version = version_expected == const.BASE_DATABASE_VERSION
if want_base_version:
logger.info('[SQL_] base version expected... returning')
# Nothing to do. autogenerated file is pointless
return
version = db.get_db_version()
# NEW DATABASE CONDITION
is_base_version = version == const.BASE_DATABASE_VERSION
# <DEBUG>
# if ut.get_flag('--verbsql') or ut.VERBOSE or True:
# key_list = locals().keys()
# keystr_list = sorted(ut.parse_locals_keylist(locals(), key_list))
# logger.info('KEYLIST:' + ut.indentjoin(keystr_list, '\n * '))
# </DEBUG>
# +-----------------------------------
# SKIP TO CURRENT VERSION IF POSSIBLE
# +-----------------------------------
can_skip = is_base_version and not force_incremental
if can_skip:
# logger.info('[SQL_] we can skip')
# Check to see if a prebuilt current schema_spec module exists
current_schema_exists = (schema_spec.UPDATE_CURRENT is not None
and schema_spec.VERSION_CURRENT is not None)
if current_schema_exists:
# check to see if more than the metadata table exists
is_newdb = db.get_table_names() == [const.METADATA_TABLE]
current_schema_compatible = (
is_newdb and schema_spec.VERSION_CURRENT <= version_expected)
if current_schema_compatible:
# Since this is a new database, we do not have to worry about backinng up the
# current database. The subsequent update functions (if needed) will handle
# this for us.
if verbose:
logger.info(
'[_SQL] New database and a current schema found')
schema_spec.UPDATE_CURRENT(db, ibs=ibs)
db.set_db_version(schema_spec.VERSION_CURRENT)
if verbose:
logger.info(
'[_SQL] Database version updated (skipped) to %r ' %
(schema_spec.VERSION_CURRENT))
else:
logger.info(
'[_SQL] Current database is not compatible, updating incrementally...'
)
else:
logger.info(
'[_SQL] New database but current version not exported, updating incrementally...'
)
# +--------------------------------------
# INCREMENTAL UPDATE TO EXPECTED VERSION
# +--------------------------------------
# Check version again for sanity's sake, update if exported current is behind expected
version = db.get_db_version()
if verbose:
logger.info('[_SQL.%s] Database version: %r | Expected version: %r ' %
(ut.get_caller_name(), version, version_expected))
if version < version_expected:
logger.info('[_SQL] Database version behind, updating...')
update_schema_version(ibs,
db,
schema_spec,
version,
version_expected,
dobackup=dobackup)
db.set_db_version(version_expected)
logger.info('[_SQL] Database version updated (incrementally) to %r' %
(version_expected))
elif version > version_expected:
msg = ('[_SQL] ERROR: '
'Expected database version behind. expected: %r. got: %r') % (
version_expected, version)
raise AssertionError(msg)
def postload_commands(ibs, back):
"""
Postload commands deal with a specific wbia database
wbia --db PZ_MTEST --occur "*All Images" --query 1
wbia --db PZ_MTEST --occur "*All Images" --query-intra
"""
params.parse_args()
if ut.NOT_QUIET:
logger.info('\n[main_cmd] postload_commands')
if params.args.view_database_directory:
logger.info('got arg --vdd')
vdd(ibs)
if params.args.set_default_dbdir:
sysres.set_default_dbdir(ibs.get_dbdir())
if params.args.update_query_cfg is not None:
# Set query parameters from command line using the --cfg flag
cfgdict = ut.parse_cfgstr_list(params.args.update_query_cfg)
logger.info('Custom cfgdict specified')
logger.info(ut.repr2(cfgdict))
ibs.update_query_cfg(**cfgdict)
if params.args.edit_notes:
ut.editfile(ibs.get_dbnotes_fpath(ensure=True))
if params.args.delete_cache:
ibs.delete_cache()
if params.args.delete_cache_complete:
ibs.delete_cache(delete_imagesets=True)
if params.args.delete_query_cache:
ibs.delete_qres_cache()
if params.args.set_all_species is not None:
ibs._overwrite_all_annot_species_to(params.args.set_all_species)
if params.args.dump_schema:
ibs.db.print_schema()
if ut.get_argflag('--ipynb'):
back.launch_ipy_notebook()
select_imgsetid = ut.get_argval(
('--select-imgsetid', '--imgsetid', '--occur', '--gsid'), None)
if select_imgsetid is not None:
logger.info('\n+ --- CMD SELECT IMGSETID=%r ---' % (select_imgsetid, ))
# Whoa: this doesnt work. weird.
# back.select_imgsetid(select_imgsetid)
# This might be the root of gui problems
# back.front._change_imageset(select_imgsetid)
back.front.select_imageset_tab(select_imgsetid)
logger.info('L ___ CMD SELECT IMGSETID=%r ___\n' % (select_imgsetid, ))
# Send commands to GUIBack
if params.args.select_aid is not None:
if back is not None:
try:
ibsfuncs.assert_valid_aids(ibs, (params.args.select_aid, ))
except AssertionError:
logger.info('Valid RIDs are: %r' % (ibs.get_valid_aids(), ))
raise
back.select_aid(params.args.select_aid)
if params.args.select_gid is not None:
back.select_gid(params.args.select_gid)
if params.args.select_nid is not None:
back.select_nid(params.args.select_nid)
select_name = ut.get_argval('--select-name')
if select_name is not None:
import wbia.gui.guiheaders as gh
back.ibswgt.select_table_indicies_from_text(gh.NAMES_TREE,
select_name,
allow_table_change=True)
if ut.get_argflag(
('--intra-occur-query', '--query-intra-occur', '--query-intra')):
back.special_query_funcs['intra_occurrence'](cfgdict={
'use_k_padding': False
})
qaid_list = ut.get_argval(('--query-aid', '--query'),
type_=list,
default=None)
if qaid_list is not None:
# qaid_list = params.args.query_aid
# fix stride case
if len(qaid_list) == 1 and isinstance(qaid_list[0], tuple):
qaid_list = list(qaid_list[0])
daids_mode = ut.get_argval('--daids-mode',
type_=str,
default=const.VS_EXEMPLARS_KEY)
back.compute_queries(qaid_list=qaid_list,
daids_mode=daids_mode,
ranks_top=10)
if ut.get_argflag('--inc-query'):
back.incremental_query()
if ut.get_argflag(('--dbinfo', '--display_dbinfo')):
back.display_dbinfo()
pass
aidcmd = ut.get_argval('--aidcmd', default=None)
aid = ut.get_argval('--aid', type_=int, default=1)
if aidcmd:
# aidcmd = 'Interact image'
metadata = ibs.get_annot_lazy_dict(aid)
annot_context_options = metadata['annot_context_options']
aidcmd_dict = dict(annot_context_options)
logger.info('aidcmd_dict = %s' % (ut.repr3(aidcmd_dict), ))
command = aidcmd_dict[aidcmd]
command()
# import utool
# utool.embed()
# back.start_web_server_parallel()
if ut.get_argflag('--start-web'):
back.start_web_server_parallel()
if ut.get_argflag('--name-tab'):
from wbia.gui.guiheaders import NAMES_TREE
back.front.set_table_tab(NAMES_TREE)
view = back.front.views[NAMES_TREE]
model = view.model()
view._set_sort(model.col_name_list.index('nAids'),
col_sort_reverse=True)
if ut.get_argflag('--graph'):
back.make_qt_graph_interface()
if params.args.postload_exit:
logger.info('[main_cmd] postload exit')
sys.exit(0)
def devmain():
"""
The Developer Script
A command line interface to almost everything
-w # wait / show the gui / figures are visible
--cmd # ipython shell to play with variables
-t # run list of tests
"""
params.parse_args()
print('DEVMAIN INIT - PRELOGGING')
if params.args.logdir is not None:
sysres.set_logdir(params.args.logdir)
else:
ut.start_logging(appname='wbia')
print('DEVMAIN INIT - POSTLOGGING')
helpstr = ut.codeblock("""
Dev is meant to be run as an interactive script.
The dev.py script runs any test you regiter with @devcmd in any combination
of configurations specified by a Config object.
Dev caches information in order to get quicker results. # FIXME: Provide quicker results # FIXME: len(line)
""")
INTRO_TITLE = 'The dev.py Script'
# INTRO_TEXT = ''.join((ut.bubbletext(INTRO_TITLE, font='cybermedium'), helpstr))
INTRO_TEXT = ut.bubbletext(INTRO_TITLE, font='cybermedium')
INTRO_STR = ut.msgblock('dev.py Intro', INTRO_TEXT)
EXAMPLE_STR = ut.msgblock('dev.py Examples', ut.codeblock(EXAMPLE_TEXT))
if ut.NOT_QUIET:
print(INTRO_STR)
if ut.get_argflag(('--help', '--verbose')):
print(EXAMPLE_STR)
CMD = ut.get_argflag('--cmd')
NOGUI = not ut.get_argflag('--gui')
if len(sys.argv) == 1:
print('Run dev.py with arguments!')
sys.exit(1)
# Run Precommands
run_devprecmds()
#
#
# Run IBEIS Main, create controller, and possibly gui
print('++dev')
main_locals = wbia.main(gui=ut.get_argflag('--gui'))
# utool.set_process_title('wbia-dev')
#
#
# Load snippet variables
SNIPPITS = True and CMD
if SNIPPITS:
snippet_locals = dev_snippets(main_locals)
snippet_execstr = utool.execstr_dict(snippet_locals, 'snippet_locals')
exec(snippet_execstr)
#
#
# Development code
RUN_DEV = True
if RUN_DEV:
dev_locals = run_dev(main_locals['ibs'])
dev_execstr = utool.execstr_dict(dev_locals, 'dev_locals')
exec(dev_execstr)
command = ut.get_argval('--eval', type_=str, default=None)
if command is not None:
result = eval(command, globals(), locals())
print('result = %r' % (result, ))
# ibs.search_annot_notes('360')
#
#
# Main Loop (IPython interaction, or some exec loop)
# if '--nopresent' not in sys.argv or '--noshow' in sys.argv:
ut.show_if_requested()
if ut.get_argflag(('--show', '--wshow')):
pt.present()
main_execstr = wbia.main_loop(main_locals,
rungui=not NOGUI,
ipy=(NOGUI or CMD))
exec(main_execstr)
#
#
# Memory profile
if ut.get_argflag('--memprof'):
utool.print_resource_usage()
utool.memory_profile()
print('exiting dev')
def run_devcmds(ibs, qaid_list, daid_list, acfg=None):
"""
This function runs tests passed in with the -t flag
"""
params.parse_args()
print('\n')
# print('[dev] run_devcmds')
print('==========================')
print('[DEV] RUN EXPERIMENTS %s' % ibs.get_dbname())
print('==========================')
input_test_list = params.args.tests[:]
print('input_test_list = %s' % (ut.repr2(input_test_list), ))
# fnum = 1
valid_test_list = [] # build list for printing in case of failure
valid_test_helpstr_list = [] # for printing
def mark_test_handled(testname):
input_test_list.remove(testname)
def intest(*args, **kwargs):
helpstr = kwargs.get('help', '')
valid_test_helpstr_list.append(' -t ' + ', '.join(args) + helpstr)
for testname in args:
valid_test_list.append(testname)
ret = testname in input_test_list
ret2 = testname in params.unknown # Let unparsed args count towards tests
if ret or ret2:
if ret:
mark_test_handled(testname)
else:
ret = ret2
print('\n+===================')
print(' [dev] running testname = %s' % (args, ))
print('+-------------------\n')
return ret
return False
valid_test_helpstr_list.append(' # --- Simple Tests ---')
# Explicit (simple) test functions
if intest('export'):
raise NotImplementedError('export')
if intest('dbinfo'):
dbinfo.get_dbinfo(ibs)
if intest('headers', 'schema'):
ibs.db.print_schema()
if intest('info'):
print(ibs.get_infostr())
if intest('printcfg'):
raise NotImplementedError('printcfg')
# printcfg(ibs)
if intest('tables'):
ibs.print_tables()
if intest('imgtbl'):
ibs.print_image_table()
valid_test_helpstr_list.append(' # --- Decor Tests ---')
locals_ = locals()
# Implicit (decorated) test functions
for (func_aliases, func) in DEVCMD_FUNCTIONS:
if intest(*func_aliases):
funcname = get_funcname(func)
# with utool.Indenter('[dev.' + funcname + ']'):
with utool.Timer(funcname):
# print('[dev] qid_list=%r' % (qaid_list,))
# FIXME: , daid_list
if len(ut.get_func_argspec(func).args) == 0:
ret = func()
else:
ret = func(ibs, qaid_list, daid_list)
# Add variables returned by the function to the
# "local scope" (the exec scop)
if hasattr(ret, 'items'):
for key, val in ret.items():
if utool.is_valid_varname(key):
locals_[key] = val
valid_test_helpstr_list.append(' # --- Config Tests ---')
# ------
# RUNS EXPERIMENT HARNESS OVER VALID TESTNAMES SPECIFIED WITH -t
# ------
# Config driven test functions
# Allow any testcfg to be in tests like: vsone_1 or vsmany_3
test_cfg_name_list = []
for test_cfg_name in experiment_configs.TEST_NAMES:
if intest(test_cfg_name):
test_cfg_name_list.append(test_cfg_name)
# Hack to allow for very customized harness tests
for testname in input_test_list[:]:
if testname.startswith('custom:'):
test_cfg_name_list.append(testname)
mark_test_handled(testname)
if len(test_cfg_name_list):
fnum = pt.next_fnum()
# Run Experiments
# backwards compatibility yo
acfgstr_name_list = {'OVERRIDE_HACK': (qaid_list, daid_list)}
assert (
False
), 'This way of running tests no longer works. It may be fixed in the future'
# acfg
harness.test_configurations(ibs, acfgstr_name_list, test_cfg_name_list)
valid_test_helpstr_list.append(' # --- Help ---')
if intest('help'):
print('valid tests are:')
print('\n'.join(valid_test_helpstr_list))
return locals_
if len(input_test_list) > 0:
print('valid tests are: \n')
print('\n'.join(valid_test_list))
raise Exception('Unknown tests: %r ' % input_test_list)
return locals_
