def get_text_between_lines(lnum1, lnum2, col1=0, col2=sys.maxsize - 1):
import vim
lines = vim.eval('getline({}, {})'.format(lnum1, lnum2))
import utool as ut
lines = ut.ensure_unicode_strlist(lines)
try:
if len(lines) == 0:
pass
elif len(lines) == 1:
lines[0] = lines[0][col1:col2 + 1]
else:
lines[0] = lines[0][col1:]
lines[-1] = lines[-1][:col2 + 1]
text = '\n'.join(lines)
except Exception:
import utool as ut
ut.ENABLE_COLORS = False
ut.util_str.ENABLE_COLORS = False
ut.util_dbg.COLORED_EXCEPTIONS = False
print(ut.repr2(lines))
raise
return text
def get_text_between_lines(lnum1, lnum2, col1=0, col2=sys.maxint - 1):
import vim
lines = vim.eval('getline({}, {})'.format(lnum1, lnum2))
import utool as ut
lines = ut.ensure_unicode_strlist(lines)
try:
if len(lines) == 0:
pass
elif len(lines) == 1:
lines[0] = lines[0][col1:col2 + 1]
else:
lines[0] = lines[0][col1:]
lines[-1] = lines[-1][:col2 + 1]
text = '\n'.join(lines)
except Exception:
import utool as ut
ut.ENABLE_COLORS = False
ut.util_str.ENABLE_COLORS = False
ut.util_dbg.COLORED_EXCEPTIONS = False
print(ut.list_str(lines))
raise
return text
def ingest_rawdata(ibs, ingestable, localize=False):
"""
Ingests rawdata into an ibeis database.
if ingest_type == 'named_folders':
Converts folder structure where folders = name, to ibsdb
if ingest_type == 'named_images':
Converts imgname structure where imgnames = name_id.ext, to ibsdb
CommandLine:
python ibeis/dbio/ingest_database.py --db seals_drop2
Args:
ibs (IBEISController): ibeis controller object
ingestable (?):
localize (bool): (default = False)
Returns:
list: aid_list - list of annotation rowids
CommandLine:
python -m ibeis.dbio.ingest_database --exec-ingest_rawdata
python -m ibeis.dbio.ingest_database --exec-ingest_rawdata --db snow-leopards --imgdir /raid/raw_rsync/snow-leopards
Example:
>>> # SCRIPT
>>> # General ingest script
>>> from ibeis.dbio.ingest_database import * # NOQA
>>> import ibeis
>>> dbname = ut.get_argval('--db', str, None) # 'snow-leopards')
>>> force_delete = ut.get_argflag(('--force_delete', '--force-delete'))
>>> img_dir = ut.get_argval('--imgdir', type_=str, default=None)
>>> assert img_dir is not None, 'specify img dir'
>>> assert dbname is not None, 'specify dbname'
>>> ingestable = Ingestable(dbname,
>>> img_dir=img_dir,
>>> ingest_type='unknown',
>>> fmtkey=None,
>>> species=None,
>>> images_as_annots=False,
>>> adjust_percent=0.00)
>>> from ibeis.control import IBEISControl
>>> dbdir = ibeis.sysres.db_to_dbdir(dbname, allow_newdir=True, use_sync=False)
>>> ut.ensuredir(dbdir, verbose=True)
>>> ibs = IBEISControl.request_IBEISController(dbdir)
>>> localize = False
>>> aid_list = ingest_rawdata(ibs, ingestable, localize)
>>> result = ('aid_list = %s' % (str(aid_list),))
>>> print(result)
"""
print('[ingest_rawdata] Ingestable' + str(ingestable))
if ingestable.zipfile is not None:
zipfile_fpath = ut.truepath(join(ibeis.sysres.get_workdir(), ingestable.zipfile))
ingestable.img_dir = ut.unarchive_file(zipfile_fpath)
img_dir = realpath(ingestable.img_dir)
ingest_type = ingestable.ingest_type
fmtkey = ingestable.fmtkey
adjust_percent = ingestable.adjust_percent
species_text = ingestable.species
postingest_func = ingestable.postingest_func
print('[ingest] ingesting rawdata: img_dir=%r, injest_type=%r' % (img_dir, ingest_type))
# Get images in the image directory
unzipped_file_base_dir = join(ibs.get_dbdir(), 'unzipped_files')
def extract_zipfile_images(ibs, ingestable):
import utool as ut # NOQA
zipfile_list = ut.glob(ingestable.img_dir, '*.zip', recursive=True)
if len(zipfile_list) > 0:
print('Found zipfile_list = %r' % (zipfile_list,))
ut.ensuredir(unzipped_file_base_dir)
for zipfile in zipfile_list:
unziped_file_relpath = dirname(relpath(relpath(realpath(zipfile), realpath(ingestable.img_dir))))
unzipped_file_dir = join(unzipped_file_base_dir, unziped_file_relpath)
ut.ensuredir(unzipped_file_dir)
ut.unzip_file(zipfile, output_dir=unzipped_file_dir, overwrite=False)
gpath_list = ut.list_images(unzipped_file_dir, fullpath=True, recursive=True)
else:
gpath_list = []
return gpath_list
def list_images(img_dir):
""" lists images that are not in an internal cache """
import utool as ut # NOQA
ignore_list = ['_hsdb', '.hs_internals', '_ibeis_cache', '_ibsdb']
gpath_list = ut.list_images(img_dir,
fullpath=True,
recursive=True,
ignore_list=ignore_list)
return gpath_list
# FIXME ensure python3 works with this
gpath_list1 = ut.ensure_unicode_strlist(list_images(img_dir))
gpath_list2 = ut.ensure_unicode_strlist(extract_zipfile_images(ibs, ingestable))
gpath_list = gpath_list1 + gpath_list2
#__STR__ = const.__STR__
#map(__STR__, gpath_list)
#ut.embed()
# Parse structure for image names
if ingest_type == 'named_folders':
name_list1 = get_name_texts_from_parent_folder(gpath_list1, img_dir, fmtkey)
name_list2 = get_name_texts_from_parent_folder(gpath_list2, unzipped_file_base_dir, fmtkey)
name_list = name_list1 + name_list2
pass
elif ingest_type == 'named_images':
name_list = get_name_texts_from_gnames(gpath_list, img_dir, fmtkey)
elif ingest_type == 'unknown':
name_list = [const.UNKNOWN for _ in range(len(gpath_list))]
else:
raise NotImplementedError('unknwon ingest_type=%r' % (ingest_type,))
# Find names likely to be the same?
RECTIFY_NAMES_HUERISTIC = True
if RECTIFY_NAMES_HUERISTIC:
names = sorted(list(set(name_list)))
splitchars = [' ', '/']
def multisplit(str_, splitchars):
import utool as ut
n = [str_]
for char in splitchars:
n = ut.flatten([_.split(char) for _ in n])
return n
groupids = [multisplit(n1, splitchars)[0] for n1 in names]
grouped_names = ut.group_items(names, groupids)
fixed_names = {newkey: key for key, val in grouped_names.items() if len(val) > 1 for newkey in val}
name_list = [fixed_names.get(name, name) for name in name_list]
# Add Images
gpath_list = [gpath.replace('\\', '/') for gpath in gpath_list]
gid_list_ = ibs.add_images(gpath_list)
# <DEBUG>
#print('added: ' + ut.indentjoin(map(str, zip(gid_list_, gpath_list))))
unique_gids = list(set(gid_list_))
print("[ingest] Length gid list: %d" % len(gid_list_))
print("[ingest] Length unique gid list: %d" % len(unique_gids))
assert len(gid_list_) == len(gpath_list)
for gid in gid_list_:
if gid is None:
print('[ingest] big fat warning')
# </DEBUG>
gid_list = ut.filter_Nones(gid_list_)
unique_gids, unique_names, unique_notes = resolve_name_conflicts(
gid_list, name_list)
# Add ANNOTATIONs with names and notes
if ingestable.images_as_annots:
aid_list = ibs.use_images_as_annotations(unique_gids,
adjust_percent=adjust_percent)
ibs.set_annot_names(aid_list, unique_names)
ibs.set_annot_notes(aid_list, unique_notes)
if species_text is not None:
ibs.set_annot_species(aid_list, [species_text] * len(aid_list))
if localize:
ibs.localize_images()
if postingest_func is not None:
postingest_func(ibs)
