def ensure_db_from_url(zipped_db_url):
""" SeeAlso ibeis.init.sysres """
from ibeis import sysres
workdir = sysres.get_workdir()
dbdir = ut.grab_zipped_url(zipped_url=zipped_db_url, ensure=True, download_dir=workdir)
print('have %s=%r' % (zipped_db_url, dbdir,))
return dbdir
def ensure_db_from_url(zipped_db_url):
""" SeeAlso ibeis.init.sysres """
from ibeis import sysres
dbname = 'testdb2'
workdir = sysres.get_workdir()
zipped_db_url = 'https://dl.dropboxusercontent.com/s/or2ngpaodrb42gd/testdb2.tar.gz'
dbdir = ut.grab_zipped_url(zipped_db_url, ensure=True, download_dir=workdir)
print('have %s=%r' % (dbname, dbdir,))
def ensure_pz_mtest():
"""
Ensures that you have the PZ_MTEST dataset
CommandLine:
python -m ibeis.init.sysres --exec-ensure_pz_mtest
python -m ibeis --tf ensure_pz_mtest
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest()
"""
print('ensure_pz_mtest')
from ibeis import sysres
workdir = sysres.get_workdir()
mtest_zipped_url = const.ZIPPED_URLS.PZ_MTEST
mtest_dir = ut.grab_zipped_url(mtest_zipped_url, ensure=True, download_dir=workdir)
print('have mtest_dir=%r' % (mtest_dir,))
# update the the newest database version
import ibeis
ibs = ibeis.opendb('PZ_MTEST')
print('cleaning up old database and ensureing everything is properly computed')
ibs.db.vacuum()
valid_aids = ibs.get_valid_aids()
assert len(valid_aids) == 119
ibs.update_annot_semantic_uuids(valid_aids)
if ut.VERYVERBOSE:
ibs.print_annotation_table()
nid = ibs.get_name_rowids_from_text('', ensure=False)
if nid is not None:
ibs.set_name_texts([nid], ['lostname'])
# Remove old imagesets and update to new special ones
all_imgset_ids = ibs.get_valid_imgsetids()
special_imgset_ids = ibs.get_special_imgsetids()
other_imgset_ids = ut.setdiff(all_imgset_ids, special_imgset_ids)
ibs.delete_imagesets(other_imgset_ids)
ibs.set_exemplars_from_quality_and_viewpoint()
ibs.update_all_image_special_imageset()
occurrence_gids = [2, 9, 12, 16, 25, 26, 29, 30, 32, 33, 35, 46, 47, 52,
57, 61, 66, 70, 71, 73, 74, 76, 77, 78, 79, 87, 88, 90,
96, 97, 103, 106, 108, 110, 112, 113]
other_gids = ut.setdiff(ibs.get_valid_gids(), occurrence_gids)
other_gids1 = other_gids[0::2]
other_gids2 = other_gids[1::2]
ibs.set_image_imagesettext(occurrence_gids, ['Occurrence 1'] * len(occurrence_gids))
ibs.set_image_imagesettext(other_gids1, ['Occurrence 2'] * len(other_gids1))
ibs.set_image_imagesettext(other_gids2, ['Occurrence 3'] * len(other_gids2))
# hack in some tags
print('Hacking in some tags')
foal_aids = [4, 8, 15, 21, 28, 34, 38, 41, 45, 49, 51, 56, 60, 66, 69, 74, 80, 83, 91, 97, 103, 107, 109, 119]
mother_aids = [9, 16, 35, 42, 52, 57, 61, 67, 75, 84, 98, 104, 108, 114]
ibs.append_annot_case_tags(foal_aids, ['foal'] * len(foal_aids))
ibs.append_annot_case_tags(mother_aids, ['mother'] * len(mother_aids))
def ingest_unconverted_hsdbs_in_workdir():
workdir = sysres.get_workdir()
needs_convert_hsdbs = get_unconverted_hsdbs(workdir)
for hsdb in needs_convert_hsdbs:
try:
convert_hsdb_to_ibeis(hsdb, force_delete=FORCE_DELETE)
except Exception as ex:
utool.printex(ex)
raise
def test_openworkdirs():
"""
problems:
PZ_DanExt_All
PZ_DanExt_Test
GZ_March2012
Wildebeest_ONLY_MATCHES
python dev.py --convert --dbdir /raid/work/PZ_Marianne --force-delete
python dev.py --convert --dbdir /raid/work/SL_Siva --force-delete
python dev.py --convert --dbdir /raid/work/PZ_SweatwaterSmall --force-delete
"""
canskip = [
'/raid/work/NAUT_test2', '/raid/work/WD_Siva',
'/raid/work/PZ_FlankHack', '/raid/work/PZ_Mothers',
'/raid/work/GZ_Foals', '/raid/work/PZ_MTEST', '/raid/work/GIR_Tanya',
'/raid/work/GZ_Siva', '/raid/work/Wildebeest', '/raid/work/sonograms',
'/raid/work/MISC_Jan12', '/raid/work/GZ_Master0',
'/raid/work/LF_OPTIMIZADAS_NI_V_E', '/raid/work/LF_Bajo_bonito',
'/raid/work/Frogs', '/raid/work/GZ_ALL', '/raid/work/JAG_Kelly',
'/raid/work/NAUT_test (copy)', '/raid/work/WS_hard',
'/raid/work/WY_Toads', '/raid/work/NAUT_Dan',
'/raid/work/LF_WEST_POINT_OPTIMIZADAS', '/raid/work/Seals',
'/raid/work/Rhinos_Stewart', '/raid/work/Elephants_Stewart',
'/raid/work/NAUT_test',
]
import ibeis
from ibeis.init import sysres
import os
import utool as ut # NOQA
from os.path import join
from ibeis.dbio import ingest_hsdb
import ibeis.other.dbinfo
ibeis.other.dbinfo.rrr()
workdir = sysres.get_workdir()
dbname_list = os.listdir(workdir)
dbpath_list = [join(workdir, name) for name in dbname_list]
is_hsdb_list = list(map(ingest_hsdb.is_hsdb, dbpath_list))
hsdb_list = ut.compress(dbpath_list, is_hsdb_list)
#is_ibs_cvt_list = np.array(list(map(is_succesful_convert, dbpath_list)))
regen_cmds = []
for hsdb_dpath in hsdb_list:
if hsdb_dpath in canskip:
continue
try:
ibs = ibeis.opendb(hsdb_dpath) # NOQA
print('Succesfully opened hsdb: ' + hsdb_dpath)
print(ibs.get_dbinfo_str())
except Exception as ex:
ut.printex(ex, 'Failed to convert hsdb: ' + hsdb_dpath)
regen_cmd = 'python dev.py --convert --dbdir ' + hsdb_dpath
regen_cmds.append(regen_cmd)
print('\n'.join(regen_cmds))
def test_openworkdirs():
"""
problems:
PZ_DanExt_All
PZ_DanExt_Test
GZ_March2012
Wildebeest_ONLY_MATCHES
python dev.py --convert --dbdir /raid/work/PZ_Marianne --force-delete
python dev.py --convert --dbdir /raid/work/SL_Siva --force-delete
python dev.py --convert --dbdir /raid/work/PZ_SweatwaterSmall --force-delete
"""
canskip = [
'/raid/work/NAUT_test2', '/raid/work/WD_Siva',
'/raid/work/PZ_FlankHack', '/raid/work/PZ_Mothers',
'/raid/work/GZ_Foals', '/raid/work/PZ_MTEST', '/raid/work/GIR_Tanya',
'/raid/work/GZ_Siva', '/raid/work/Wildebeest', '/raid/work/sonograms',
'/raid/work/MISC_Jan12', '/raid/work/GZ_Master0',
'/raid/work/LF_OPTIMIZADAS_NI_V_E', '/raid/work/LF_Bajo_bonito',
'/raid/work/Frogs', '/raid/work/GZ_ALL', '/raid/work/JAG_Kelly',
'/raid/work/NAUT_test (copy)', '/raid/work/WS_hard',
'/raid/work/WY_Toads', '/raid/work/NAUT_Dan',
'/raid/work/LF_WEST_POINT_OPTIMIZADAS', '/raid/work/Seals',
'/raid/work/Rhinos_Stewart', '/raid/work/Elephants_Stewart',
'/raid/work/NAUT_test',
]
import ibeis
from ibeis.init import sysres
import os
import utool as ut # NOQA
from os.path import join
from ibeis.dbio import ingest_hsdb
import ibeis.other.dbinfo
ibeis.other.dbinfo.rrr()
workdir = sysres.get_workdir()
dbname_list = os.listdir(workdir)
dbpath_list = [join(workdir, name) for name in dbname_list]
is_hsdb_list = list(map(ingest_hsdb.is_hsdb, dbpath_list))
hsdb_list = ut.compress(dbpath_list, is_hsdb_list)
#is_ibs_cvt_list = np.array(list(map(is_succesful_convert, dbpath_list)))
regen_cmds = []
for hsdb_dpath in hsdb_list:
if hsdb_dpath in canskip:
continue
try:
ibs = ibeis.opendb(hsdb_dpath) # NOQA
print('Succesfully opened hsdb: ' + hsdb_dpath)
print(ibs.get_dbinfo_str())
except Exception as ex:
ut.printex(ex, 'Failed to convert hsdb: ' + hsdb_dpath)
regen_cmd = 'python dev.py --convert --dbdir ' + hsdb_dpath
regen_cmds.append(regen_cmd)
print('\n'.join(regen_cmds))
def get_unconverted_hsdbs(workdir=None):
import os
import numpy as np
from vtool import linalg as ltool
if workdir is None:
workdir = sysres.get_workdir()
dbname_list = os.listdir(workdir)
dbpath_list = np.array([join(workdir, name) for name in dbname_list])
is_hsdb_list = np.array(map(sysres.is_hsdb, dbpath_list))
is_ibs_cvt_list = np.array(map(is_succesful_convert, dbpath_list))
if FORCE_DELETE:
needs_convert = is_hsdb_list
else:
needs_convert = ltool.and_lists(is_hsdb_list, True - is_ibs_cvt_list)
needs_convert_hsdbs = dbpath_list[needs_convert].tolist()
return needs_convert_hsdbs
def ensure_pz_mtest():
"""
Ensures that you have the PZ_MTEST dataset
CommandLine:
python -m ibeis.init.sysres --exec-ensure_pz_mtest
python -m ibeis --tf ensure_pz_mtest
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest()
"""
print('ensure_pz_mtest')
from ibeis import sysres
workdir = sysres.get_workdir()
mtest_zipped_url = const.ZIPPED_URLS.PZ_MTEST
mtest_dir = ut.grab_zipped_url(mtest_zipped_url, ensure=True, download_dir=workdir)
print('have mtest_dir=%r' % (mtest_dir,))
# update the the newest database version
import ibeis
ibs = ibeis.opendb('PZ_MTEST')
print('cleaning up old database and ensureing everything is properly computed')
ibs.db.vacuum()
valid_aids = ibs.get_valid_aids()
assert len(valid_aids) == 119
ibs.update_annot_semantic_uuids(valid_aids)
if ut.VERYVERBOSE:
ibs.print_annotation_table()
nid = ibs.get_name_rowids_from_text('', ensure=False)
if nid is not None:
ibs.set_name_texts([nid], ['lostname'])
# hack in some tags
print('Hacking in some tags')
foal_aids = [4, 8, 15, 21, 28, 34, 38, 41, 45, 49, 51, 56, 60, 66, 69, 74, 80, 83, 91, 97, 103, 107, 109, 119]
mother_aids = [9, 16, 35, 42, 52, 57, 61, 67, 75, 84, 98, 104, 108, 114]
ibs.append_annot_case_tags(foal_aids, ['foal'] * len(foal_aids))
ibs.append_annot_case_tags(mother_aids, ['mother'] * len(mother_aids))
def ensure_pz_mtest():
"""
Ensures that you have the PZ_MTEST dataset
CommandLine:
python -m ibeis.init.sysres --exec-ensure_pz_mtest
python -m ibeis --tf ensure_pz_mtest
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest()
"""
print('ensure_pz_mtest')
from ibeis import sysres
workdir = sysres.get_workdir()
mtest_zipped_url = const.ZIPPED_URLS.PZ_MTEST
mtest_dir = ut.grab_zipped_url(mtest_zipped_url,
ensure=True,
download_dir=workdir)
print('have mtest_dir=%r' % (mtest_dir, ))
# update the the newest database version
import ibeis
ibs = ibeis.opendb('PZ_MTEST')
print(
'cleaning up old database and ensureing everything is properly computed'
)
ibs.db.vacuum()
valid_aids = ibs.get_valid_aids()
assert len(valid_aids) == 119
ibs.update_annot_semantic_uuids(valid_aids)
if ut.VERYVERBOSE:
ibs.print_annotation_table()
nid = ibs.get_name_rowids_from_text('', ensure=False)
if nid is not None:
ibs.set_name_texts([nid], ['lostname'])
# Remove old imagesets and update to new special ones
all_imgset_ids = ibs.get_valid_imgsetids()
special_imgset_ids = ibs.get_special_imgsetids()
other_imgset_ids = ut.setdiff(all_imgset_ids, special_imgset_ids)
ibs.delete_imagesets(other_imgset_ids)
ibs.set_exemplars_from_quality_and_viewpoint()
ibs.update_all_image_special_imageset()
occurrence_gids = [
2, 9, 12, 16, 25, 26, 29, 30, 32, 33, 35, 46, 47, 52, 57, 61, 66, 70,
71, 73, 74, 76, 77, 78, 79, 87, 88, 90, 96, 97, 103, 106, 108, 110,
112, 113
]
other_gids = ut.setdiff(ibs.get_valid_gids(), occurrence_gids)
other_gids1 = other_gids[0::2]
other_gids2 = other_gids[1::2]
ibs.set_image_imagesettext(occurrence_gids,
['Occurrence 1'] * len(occurrence_gids))
ibs.set_image_imagesettext(other_gids1,
['Occurrence 2'] * len(other_gids1))
ibs.set_image_imagesettext(other_gids2,
['Occurrence 3'] * len(other_gids2))
# hack in some tags
print('Hacking in some tags')
foal_aids = [
4, 8, 15, 21, 28, 34, 38, 41, 45, 49, 51, 56, 60, 66, 69, 74, 80, 83,
91, 97, 103, 107, 109, 119
]
mother_aids = [9, 16, 35, 42, 52, 57, 61, 67, 75, 84, 98, 104, 108, 114]
ibs.append_annot_case_tags(foal_aids, ['foal'] * len(foal_aids))
ibs.append_annot_case_tags(mother_aids, ['mother'] * len(mother_aids))
def ensure_db_from_url(zipped_db_url):
""" SeeAlso ibeis.init.sysres """
from ibeis import sysres
workdir = sysres.get_workdir()
dbdir = ut.grab_zipped_url(zipped_url=zipped_db_url, ensure=True, download_dir=workdir)
print('have %s=%r' % (zipped_db_url, dbdir,))
