def unload_cxdata(hs, cx):
'unloads features and chips. not tables'
print('[hs] unload_cxdata(cx=%r)' % cx)
# HACK This should not really be removed EVERY time you unload any cx
hs.qdat.unload_data()
hs.clear_lru_caches()
lists = []
if hs.cpaths is not None:
lists += [hs.cpaths.cx2_rchip_path, hs.cpaths.cx2_rchip_size]
if hs.feats is not None:
lists += [hs.feats.cx2_kpts, hs.feats.cx2_desc]
if cx == 'all':
hs.unload_all()
return
for list_ in lists:
helpers.ensure_list_size(list_, cx + 1)
list_[cx] = None
def unload_cxdata(hs, cx):
'unloads features and chips. not tables'
print('[hs] unload_cxdata(cx=%r)' % cx)
# HACK This should not really be removed EVERY time you unload any cx
hs.qdat.unload_data()
hs.clear_lru_caches()
lists = []
if hs.cpaths is not None:
lists += [hs.cpaths.cx2_rchip_path, hs.cpaths.cx2_rchip_size]
if hs.feats is not None:
lists += [hs.feats.cx2_kpts, hs.feats.cx2_desc]
if cx == 'all':
hs.unload_all()
return
for list_ in lists:
helpers.ensure_list_size(list_, cx + 1)
list_[cx] = None
def load_features(hs, cx_list=None, **kwargs):
print('\n=============================')
print('[fc2] Precomputing and loading features: %r' % hs.get_db_name())
print('=============================')
#----------------
# COMPUTE SETUP
#----------------
use_cache = not hs.args.nocache_feats
use_big_cache = use_cache and cx_list is None
feat_cfg = hs.prefs.feat_cfg
feat_uid = feat_cfg.get_uid()
if hs.feats.feat_uid != '' and hs.feats.feat_uid != feat_uid:
print('[fc2] Disagreement: OLD_feat_uid = %r' % hs.feats.feat_uid)
print('[fc2] Disagreement: NEW_feat_uid = %r' % feat_uid)
print('[fc2] Unloading all chip information')
hs.unload_all()
hs.load_chips(cx_list=cx_list)
print('[fc2] feat_uid = %r' % feat_uid)
# Get the list of chip features to load
cx_list = hs.get_valid_cxs() if cx_list is None else cx_list
if not np.iterable(cx_list):
cx_list = [cx_list]
if len(cx_list) == 0:
return # HACK
cx_list = np.array(cx_list) # HACK
if use_big_cache: # use only if all descriptors requested
kpts_list, desc_list = _load_features_bigcache(hs, cx_list)
else:
kpts_list, desc_list = _load_features_individualy(hs, cx_list)
# Extend the datastructure if needed
list_size = max(cx_list) + 1
helpers.ensure_list_size(hs.feats.cx2_kpts, list_size)
helpers.ensure_list_size(hs.feats.cx2_desc, list_size)
# Copy the values into the ChipPaths object
for lx, cx in enumerate(cx_list):
hs.feats.cx2_kpts[cx] = kpts_list[lx]
for lx, cx in enumerate(cx_list):
hs.feats.cx2_desc[cx] = desc_list[lx]
hs.feats.feat_uid = feat_uid
print('[fc2]=============================')
def load_chips(hs, cx_list=None, **kwargs):
print('\n=============================')
print('[cc2] Precomputing chips and loading chip paths: %r' % hs.get_db_name())
print('=============================')
#----------------
# COMPUTE SETUP
#----------------
chip_cfg = hs.prefs.chip_cfg
chip_uid = chip_cfg.get_uid()
if hs.cpaths.chip_uid != '' and hs.cpaths.chip_uid != chip_uid:
print('[cc2] Disagreement: OLD_chip_uid = %r' % hs.cpaths.chip_uid)
print('[cc2] Disagreement: NEW_chip_uid = %r' % chip_uid)
print('[cc2] Unloading all chip information')
hs.unload_all()
print('[cc2] chip_uid = %r' % chip_uid)
# Get the list of chips paths to load
cx_list = hs.get_valid_cxs() if cx_list is None else cx_list
if not np.iterable(cx_list):
cx_list = [cx_list]
if len(cx_list) == 0:
return # HACK
cx_list = np.array(cx_list) # HACK
hs.cpaths.chip_uid = chip_uid
#print('[cc2] Requested %d chips' % (len(cx_list)))
#print('[cc2] cx_list = %r' % (cx_list,))
# Get table information
try:
gx_list = hs.tables.cx2_gx[cx_list]
cid_list = hs.tables.cx2_cid[cx_list]
theta_list = hs.tables.cx2_theta[cx_list]
roi_list = hs.tables.cx2_roi[cx_list]
#gname_list = hs.tables.gx2_gname[gx_list]
except IndexError as ex:
print(repr(ex))
print(hs.tables)
print('cx_list=%r' % (cx_list,))
raise
# Get ChipConfig Parameters
sqrt_area = chip_cfg['chip_sqrt_area']
filter_list = []
if chip_cfg['adapteq']:
filter_list.append(adapteq_fn)
if chip_cfg['histeq']:
filter_list.append(histeq_fn)
if chip_cfg['region_norm']:
filter_list.append(region_norm_fn)
#if chip_cfg['maxcontrast']:
#filter_list.append(maxcontr_fn)
#if chip_cfg['rank_eq']:
#filter_list.append(rankeq_fn)
#if chip_cfg['local_eq']:
#filter_list.append(localeq_fn)
if chip_cfg['grabcut']:
filter_list.append(grabcut_fn)
#---------------------------
# ___Normalized Chip Args___
#---------------------------
# Full Image Paths: where to extract the chips from
gfpath_list = hs.gx2_gname(gx_list, full=True)
#img_dir = hs.dirs.img_dir
#gfpath_list = [join(img_dir, gname) for gname in iter(gname_list)]
# Chip Paths: where to write extracted chips to
_cfname_fmt = 'cid%d' + chip_uid + '.png'
_cfpath_fmt = join(hs.dirs.chip_dir, _cfname_fmt)
cfpath_list = [_cfpath_fmt % cid for cid in iter(cid_list)]
# Normalized Chip Sizes: ensure chips have about sqrt_area squared pixels
chipsz_list = compute_uniform_area_chip_sizes(roi_list, sqrt_area)
#--------------------------
# EXTRACT AND RESIZE CHIPS
#--------------------------
pcc_kwargs = {
'arg_list': [gfpath_list, cfpath_list, roi_list, theta_list, chipsz_list],
'lazy': not hs.args.nocache_chips,
'num_procs': hs.args.num_procs,
'common_args': [filter_list]
}
# Compute all chips with paramatarized filters
parallel_compute(compute_chip, **pcc_kwargs)
# Read sizes
try:
rsize_list = [(None, None) if path is None else Image.open(path).size
for path in iter(cfpath_list)]
except IOError as ex:
import gc
gc.collect()
print('[cc] ex=%r' % ex)
print('path=%r' % path)
if helpers.checkpath(path, verbose=True):
import time
time.sleep(1) # delays for 1 seconds
print('[cc] file exists but cause IOError?')
print('[cc] probably corrupted. Removing it')
try:
helpers.remove_file(path)
except OSError:
print('Something bad happened')
raise
raise
#----------------------
# UPDATE API VARIABLES
#----------------------
print('[cc2] Done Precomputing chips and loading chip paths')
# Extend the datastructure if needed
list_size = max(cx_list) + 1
#helpers.ensure_list_size(hs.cpaths.cx2_chip_path, list_size)
helpers.ensure_list_size(hs.cpaths.cx2_rchip_path, list_size)
helpers.ensure_list_size(hs.cpaths.cx2_rchip_size, list_size)
# Copy the values into the ChipPaths object
#for lx, cx in enumerate(cx_list):
#hs.cpaths.cx2_chip_path[cx] = cfpath_list[lx]
for lx, cx in enumerate(cx_list):
hs.cpaths.cx2_rchip_path[cx] = cfpath_list[lx]
for lx, cx in enumerate(cx_list):
hs.cpaths.cx2_rchip_size[cx] = rsize_list[lx]
#hs.load_cx2_rchip_size() # TODO: Loading rchip size should be handled more robustly
print('[cc2]=============================')
def load_chips(hs, cx_list=None, **kwargs):
print('\n=============================')
print('[cc2] Precomputing chips and loading chip paths: %r' %
hs.get_db_name())
print('=============================')
#----------------
# COMPUTE SETUP
#----------------
chip_cfg = hs.prefs.chip_cfg
chip_uid = chip_cfg.get_uid()
if hs.cpaths.chip_uid != '' and hs.cpaths.chip_uid != chip_uid:
print('[cc2] Disagreement: OLD_chip_uid = %r' % hs.cpaths.chip_uid)
print('[cc2] Disagreement: NEW_chip_uid = %r' % chip_uid)
print('[cc2] Unloading all chip information')
hs.unload_all()
print('[cc2] chip_uid = %r' % chip_uid)
# Get the list of chips paths to load
cx_list = hs.get_valid_cxs() if cx_list is None else cx_list
if not np.iterable(cx_list):
cx_list = [cx_list]
if len(cx_list) == 0:
return # HACK
cx_list = np.array(cx_list) # HACK
hs.cpaths.chip_uid = chip_uid
#print('[cc2] Requested %d chips' % (len(cx_list)))
#print('[cc2] cx_list = %r' % (cx_list,))
# Get table information
try:
gx_list = hs.tables.cx2_gx[cx_list]
cid_list = hs.tables.cx2_cid[cx_list]
theta_list = hs.tables.cx2_theta[cx_list]
roi_list = hs.tables.cx2_roi[cx_list]
#gname_list = hs.tables.gx2_gname[gx_list]
except IndexError as ex:
print(repr(ex))
print(hs.tables)
print('cx_list=%r' % (cx_list, ))
raise
# Get ChipConfig Parameters
sqrt_area = chip_cfg['chip_sqrt_area']
filter_list = []
if chip_cfg['adapteq']:
filter_list.append(adapteq_fn)
if chip_cfg['histeq']:
filter_list.append(histeq_fn)
if chip_cfg['region_norm']:
filter_list.append(region_norm_fn)
#if chip_cfg['maxcontrast']:
#filter_list.append(maxcontr_fn)
#if chip_cfg['rank_eq']:
#filter_list.append(rankeq_fn)
#if chip_cfg['local_eq']:
#filter_list.append(localeq_fn)
if chip_cfg['grabcut']:
filter_list.append(grabcut_fn)
#---------------------------
# ___Normalized Chip Args___
#---------------------------
# Full Image Paths: where to extract the chips from
gfpath_list = hs.gx2_gname(gx_list, full=True)
#img_dir = hs.dirs.img_dir
#gfpath_list = [join(img_dir, gname) for gname in iter(gname_list)]
# Chip Paths: where to write extracted chips to
_cfname_fmt = 'cid%d' + chip_uid + '.png'
_cfpath_fmt = join(hs.dirs.chip_dir, _cfname_fmt)
cfpath_list = [_cfpath_fmt % cid for cid in iter(cid_list)]
# Normalized Chip Sizes: ensure chips have about sqrt_area squared pixels
chipsz_list = compute_uniform_area_chip_sizes(roi_list, sqrt_area)
#--------------------------
# EXTRACT AND RESIZE CHIPS
#--------------------------
pcc_kwargs = {
'arg_list':
[gfpath_list, cfpath_list, roi_list, theta_list, chipsz_list],
'lazy': not hs.args.nocache_chips,
'num_procs': hs.args.num_procs,
'common_args': [filter_list]
}
# Compute all chips with paramatarized filters
parallel_compute(compute_chip, **pcc_kwargs)
# Read sizes
try:
rsize_list = [(None, None) if path is None else Image.open(path).size
for path in iter(cfpath_list)]
except IOError as ex:
import gc
gc.collect()
print('[cc] ex=%r' % ex)
print('path=%r' % path)
if helpers.checkpath(path, verbose=True):
import time
time.sleep(1) # delays for 1 seconds
print('[cc] file exists but cause IOError?')
print('[cc] probably corrupted. Removing it')
try:
helpers.remove_file(path)
except OSError:
print('Something bad happened')
raise
raise
#----------------------
# UPDATE API VARIABLES
#----------------------
print('[cc2] Done Precomputing chips and loading chip paths')
# Extend the datastructure if needed
list_size = max(cx_list) + 1
#helpers.ensure_list_size(hs.cpaths.cx2_chip_path, list_size)
helpers.ensure_list_size(hs.cpaths.cx2_rchip_path, list_size)
helpers.ensure_list_size(hs.cpaths.cx2_rchip_size, list_size)
# Copy the values into the ChipPaths object
#for lx, cx in enumerate(cx_list):
#hs.cpaths.cx2_chip_path[cx] = cfpath_list[lx]
for lx, cx in enumerate(cx_list):
hs.cpaths.cx2_rchip_path[cx] = cfpath_list[lx]
for lx, cx in enumerate(cx_list):
hs.cpaths.cx2_rchip_size[cx] = rsize_list[lx]
#hs.load_cx2_rchip_size() # TODO: Loading rchip size should be handled more robustly
print('[cc2]=============================')