def deploy(self, task_key=None, publish=False):
"""
Trains and saves a classifier for deployment
Notes:
A deployment consists of the following information
* The classifier itself
* Information needed to construct the input to the classifier
- TODO: can this be encoded as an sklearn pipeline?
* Metadata concerning what data the classifier was trained with
* PUBLISH TO /media/hdd/PUBLIC/models/pairclf
Example:
>>> # xdoctest: +REQUIRES(module:wbia_cnn)
>>> from wbia.algo.verif.vsone import * # NOQA
>>> params = dict(sample_method='random')
>>> pblm = OneVsOneProblem.from_empty('PZ_MTEST', **params)
>>> pblm.setup(with_simple=False)
>>> task_key = pblm.primary_task_key
>>> self = Deployer(dpath='.', pblm=pblm)
>>> deploy_info = self.deploy()
Ignore:
pblm.evaluate_classifiers(with_simple=False)
res = pblm.task_combo_res[pblm.primary_task_key]['RF']['learn(sum,glob)']
"""
deploy_info = self._make_deploy_info(task_key=task_key)
deploy_fname = deploy_info['metadata']['fname']
meta_fname = deploy_fname + self.meta_suffix
deploy_fpath = join(self.dpath, deploy_fname)
meta_fpath = join(self.dpath, meta_fname)
ut.save_json(meta_fpath, deploy_info['metadata'])
ut.save_data(deploy_fpath, deploy_info)
if publish:
user = ut.get_user_name()
remote_uri = '{user}@{remote}:{path}'.format(user=user,
**self.publish_info)
ut.rsync(meta_fpath, remote_uri + '/' + meta_fname)
ut.rsync(deploy_fpath, remote_uri + '/' + deploy_fname)
return deploy_info
def sync_wbiadb(remote_uri,
dbname,
mode='pull',
workdir=None,
port=22,
dryrun=False):
"""
syncs an wbiadb without syncing the cache or the chip directory
(or the top level image directory because it shouldnt exist unlese
it is an old hots database)
"""
logger.info('[sync_wbiadb] Syncing')
logger.info(' * dbname=%r ' % (dbname, ))
logger.info(' * remote_uri=%r' % (remote_uri, ))
logger.info(' * mode=%r' % (mode))
import wbia
assert dbname is not None, 'must specify a database name'
# Excluded temporary and cached data
exclude_dirs = list(
map(ut.ensure_unixslash, wbia.const.EXCLUDE_COPY_REL_DIRS))
# Specify local workdir
if workdir is None:
workdir = wbia.sysres.get_workdir()
local_uri = ut.ensure_unixslash(workdir)
if ut.WIN32:
# fix for mingw rsync
local_uri = ut.ensure_mingw_drive(local_uri)
if mode == 'pull':
# pull remote to local
remote_src = ut.unixjoin(remote_uri, dbname)
ut.assert_exists(local_uri)
ut.rsync(remote_src, local_uri, exclude_dirs, port, dryrun=dryrun)
elif mode == 'push':
# push local to remote
local_src = ut.unixjoin(local_uri, dbname)
if not dryrun:
ut.assert_exists(local_src)
ut.rsync(local_src, remote_uri, exclude_dirs, port, dryrun=dryrun)
if dryrun:
ut.assert_exists(local_src)
else:
raise AssertionError('unknown mode=%r' % (mode, ))
def sync_ibeisdb(remote_uri, dbname, mode='pull', workdir=None, port=22, dryrun=False):
"""
syncs an ibeisdb without syncing the cache or the chip directory
(or the top level image directory because it shouldnt exist unless it is an
old hots database)
"""
print('[sync_ibeisdb] Syncing')
print(' * dbname=%r ' % (dbname,))
print(' * remote_uri=%r' % (remote_uri,))
print(' * mode=%r' % (mode))
import ibeis
assert dbname is not None, 'must specify a database name'
# Excluded temporary and cached data
exclude_dirs = list(map(ut.ensure_unixslash, ibeis.const.EXCLUDE_COPY_REL_DIRS))
# Specify local workdir
if workdir is None:
workdir = ibeis.sysres.get_workdir()
local_uri = ut.ensure_unixslash(workdir)
if ut.WIN32:
# fix for mingw rsync
local_uri = ut.ensure_mingw_drive(local_uri)
if mode == 'pull':
# pull remote to local
remote_src = ut.unixjoin(remote_uri, dbname)
ut.assert_exists(local_uri)
ut.rsync(remote_src, local_uri, exclude_dirs, port, dryrun=dryrun)
elif mode == 'push':
# push local to remote
local_src = ut.unixjoin(local_uri, dbname)
if not dryrun:
ut.assert_exists(local_src)
ut.rsync(local_src, remote_uri, exclude_dirs, port, dryrun=dryrun)
if dryrun:
ut.assert_exists(local_src)
else:
raise AssertionError('unknown mode=%r' % (mode,))
def _fix_ggr2018_directory_structure(ggr_path):
# Manual fixes for bad directories
src_uri = join(ggr_path, 'Clarine\\ Plane\\ Kurungu/')
dst_uri = join(ggr_path, '231/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '231B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(
ggr_path,
'Alex\\ Peltier\\ -\\ Plane\\ -\\ Ngurnit/giraffe\\ grevy\\ count\\ feb\\ 18/',
)
dst_uri = join(ggr_path, '232/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '232B/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, 'Mint\\ Media\\ Footage',
'Mpala\\ day\\ 1\\ spark', 'PANORAMA/')
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Mint\\ Media\\ Footage', 'Mpala\\ day\\ 1/')
dst_uri = join(ggr_path, '233/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '233B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Mint\\ Media\\ Footage',
'Mpala\\ day\\ 1\\ spark/')
dst_uri = join(ggr_path, '233', '233B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Mint\\ Media\\ Footage', 'Mpala\\ day2\\ /')
dst_uri = join(ggr_path, '233', '233B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Mint\\ Media\\ Footage',
'Mpala\\ day\\ 2\\ spark/')
dst_uri = join(ggr_path, '233', '233B/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '103\\ \\(1\\)/')
dst_uri = join(ggr_path, '103/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '103\\ \\(ccef473b\\)/')
dst_uri = join(ggr_path, '103/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '108\\ \\(1\\)/')
dst_uri = join(ggr_path, '108/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '226A\\ \\(Shaba\\ Funan\\ Camp\\)/')
dst_uri = join(ggr_path, '226/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '226A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '121/*.*')
dst_uri = join(ggr_path, '121', '121A/')
ut.rsync(src_uri, dst_uri)
for src_filepath in ut.glob(src_uri.replace('\\', '')):
ut.delete(src_filepath)
src_uri = join(ggr_path, '54', '54A\\(16\\)/')
dst_uri = join(ggr_path, '54', '54A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '54', '54B\\(16\\)/')
dst_uri = join(ggr_path, '54', '54B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '87', '87/')
dst_uri = join(ggr_path, '87', '87A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '223', 'A/')
dst_uri = join(ggr_path, '223', '223A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '223', 'B/')
dst_uri = join(ggr_path, '223', '223B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '14', '15A/')
dst_uri = join(ggr_path, '14', '14A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '73/')
dst_uri = join(ggr_path, '85/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '117', '115A/')
dst_uri = join(ggr_path, '117', '117A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200\\ A/')
dst_uri = join(ggr_path, '200', '200A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200\\ B/')
dst_uri = join(ggr_path, '200', '200B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200\\ F/')
dst_uri = join(ggr_path, '200', '200F/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200A/')
dst_uri = join(ggr_path, '201/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '201A/')
ut.rsync(src_uri, dst_uri)
# ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '201\\ E/')
dst_uri = join(ggr_path, '201', '201E/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '201\\ F/')
dst_uri = join(ggr_path, '201', '201F/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200A/')
dst_uri = join(ggr_path, '202/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '202A/')
ut.rsync(src_uri, dst_uri)
# ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '202\\ B/')
dst_uri = join(ggr_path, '202', '202B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '202\\ F/')
dst_uri = join(ggr_path, '202', '202F/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '230', '230A', 'El\\ Karama/*.*')
dst_uri = join(ggr_path, '230', '230A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-1])
ut.delete(src_uri)
src_uri = join(ggr_path, '136', '136B', '136B\\ Grevys\\ Rally/*.*')
dst_uri = join(ggr_path, '136', '136B/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-1])
ut.delete(src_uri)
src_uri = join(ggr_path, '160', '160E', '104DUSIT')
if exists(src_uri):
direct = Directory(src_uri, recursive=False)
filename_list = direct.files()
for filename in sorted(filename_list):
dst_uri = filename.replace('104DUSIT/',
'').replace('.JPG', '_.JPG')
assert not exists(dst_uri)
ut.rsync(filename, dst_uri)
ut.delete(src_uri)
src_uri = join(ggr_path, '222', '222B', '102DUSIT')
if exists(src_uri):
direct = Directory(src_uri, recursive=False)
filename_list = direct.files()
for filename in sorted(filename_list):
dst_uri = filename.replace('102DUSIT/',
'').replace('.JPG', '_.JPG')
assert not exists(dst_uri)
ut.rsync(filename, dst_uri)
ut.delete(src_uri)
# Errors found by QR codes
# No conflicts
src_uri = join(ggr_path, '5', '5A/')
dst_uri = join(ggr_path, '5', '5B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '14', '14A/')
dst_uri = join(ggr_path, '14', '14B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '118', '118A/')
dst_uri = join(ggr_path, '192')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '192A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '119', '119A/')
dst_uri = join(ggr_path, '189')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '189A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '120', '120A/')
dst_uri = join(ggr_path, '190')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '190A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '138', '138C/')
dst_uri = join(ggr_path, '169', '169C/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri)
# Conflicts - Move first
src_uri = join(ggr_path, '115', '115A/')
dst_uri = join(ggr_path, '191')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '191A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '148', '148A/')
dst_uri = join(ggr_path, '149', '149A-temp/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
# Conflicts - Move second
src_uri = join(ggr_path, '117', '117A/')
dst_uri = join(ggr_path, '115', '115A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '149', '149A/')
dst_uri = join(ggr_path, '148', '148A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
# Conflicts - Move third
src_uri = join(ggr_path, '149', '149A-temp/')
dst_uri = join(ggr_path, '149', '149A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
# Conflicts - Merge third
src_uri = join(ggr_path, '57', '57A/')
dst_uri = join(ggr_path, '25', '25A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)