def ensure_db_from_url(zipped_db_url): """ SeeAlso ibeis.init.sysres """ from ibeis import sysres workdir = sysres.get_workdir() dbdir = ut.grab_zipped_url(zipped_url=zipped_db_url, ensure=True, download_dir=workdir) print('have %s=%r' % (zipped_db_url, dbdir,)) return dbdir
def ensure_db_from_url(zipped_db_url): """ SeeAlso ibeis.init.sysres """ from ibeis import sysres dbname = 'testdb2' workdir = sysres.get_workdir() zipped_db_url = 'https://dl.dropboxusercontent.com/s/or2ngpaodrb42gd/testdb2.tar.gz' dbdir = ut.grab_zipped_url(zipped_db_url, ensure=True, download_dir=workdir) print('have %s=%r' % (dbname, dbdir,))
def ensure_pz_mtest(): """ Ensures that you have the PZ_MTEST dataset CommandLine: python -m ibeis.init.sysres --exec-ensure_pz_mtest python -m ibeis --tf ensure_pz_mtest Example: >>> # SCRIPT >>> from ibeis.init.sysres import * # NOQA >>> ensure_pz_mtest() """ print('ensure_pz_mtest') from ibeis import sysres workdir = sysres.get_workdir() mtest_zipped_url = const.ZIPPED_URLS.PZ_MTEST mtest_dir = ut.grab_zipped_url(mtest_zipped_url, ensure=True, download_dir=workdir) print('have mtest_dir=%r' % (mtest_dir,)) # update the the newest database version import ibeis ibs = ibeis.opendb('PZ_MTEST') print('cleaning up old database and ensureing everything is properly computed') ibs.db.vacuum() valid_aids = ibs.get_valid_aids() assert len(valid_aids) == 119 ibs.update_annot_semantic_uuids(valid_aids) if ut.VERYVERBOSE: ibs.print_annotation_table() nid = ibs.get_name_rowids_from_text('', ensure=False) if nid is not None: ibs.set_name_texts([nid], ['lostname']) # Remove old imagesets and update to new special ones all_imgset_ids = ibs.get_valid_imgsetids() special_imgset_ids = ibs.get_special_imgsetids() other_imgset_ids = ut.setdiff(all_imgset_ids, special_imgset_ids) ibs.delete_imagesets(other_imgset_ids) ibs.set_exemplars_from_quality_and_viewpoint() ibs.update_all_image_special_imageset() occurrence_gids = [2, 9, 12, 16, 25, 26, 29, 30, 32, 33, 35, 46, 47, 52, 57, 61, 66, 70, 71, 73, 74, 76, 77, 78, 79, 87, 88, 90, 96, 97, 103, 106, 108, 110, 112, 113] other_gids = ut.setdiff(ibs.get_valid_gids(), occurrence_gids) other_gids1 = other_gids[0::2] other_gids2 = other_gids[1::2] ibs.set_image_imagesettext(occurrence_gids, ['Occurrence 1'] * len(occurrence_gids)) ibs.set_image_imagesettext(other_gids1, ['Occurrence 2'] * len(other_gids1)) ibs.set_image_imagesettext(other_gids2, ['Occurrence 3'] * len(other_gids2)) # hack in some tags print('Hacking in some tags') foal_aids = [4, 8, 15, 21, 28, 34, 38, 41, 45, 49, 51, 56, 60, 66, 69, 74, 80, 83, 91, 97, 103, 107, 109, 119] mother_aids = [9, 16, 35, 42, 52, 57, 61, 67, 75, 84, 98, 104, 108, 114] ibs.append_annot_case_tags(foal_aids, ['foal'] * len(foal_aids)) ibs.append_annot_case_tags(mother_aids, ['mother'] * len(mother_aids))
def ingest_unconverted_hsdbs_in_workdir(): workdir = sysres.get_workdir() needs_convert_hsdbs = get_unconverted_hsdbs(workdir) for hsdb in needs_convert_hsdbs: try: convert_hsdb_to_ibeis(hsdb, force_delete=FORCE_DELETE) except Exception as ex: utool.printex(ex) raise
def test_openworkdirs(): """ problems: PZ_DanExt_All PZ_DanExt_Test GZ_March2012 Wildebeest_ONLY_MATCHES python dev.py --convert --dbdir /raid/work/PZ_Marianne --force-delete python dev.py --convert --dbdir /raid/work/SL_Siva --force-delete python dev.py --convert --dbdir /raid/work/PZ_SweatwaterSmall --force-delete """ canskip = [ '/raid/work/NAUT_test2', '/raid/work/WD_Siva', '/raid/work/PZ_FlankHack', '/raid/work/PZ_Mothers', '/raid/work/GZ_Foals', '/raid/work/PZ_MTEST', '/raid/work/GIR_Tanya', '/raid/work/GZ_Siva', '/raid/work/Wildebeest', '/raid/work/sonograms', '/raid/work/MISC_Jan12', '/raid/work/GZ_Master0', '/raid/work/LF_OPTIMIZADAS_NI_V_E', '/raid/work/LF_Bajo_bonito', '/raid/work/Frogs', '/raid/work/GZ_ALL', '/raid/work/JAG_Kelly', '/raid/work/NAUT_test (copy)', '/raid/work/WS_hard', '/raid/work/WY_Toads', '/raid/work/NAUT_Dan', '/raid/work/LF_WEST_POINT_OPTIMIZADAS', '/raid/work/Seals', '/raid/work/Rhinos_Stewart', '/raid/work/Elephants_Stewart', '/raid/work/NAUT_test', ] import ibeis from ibeis.init import sysres import os import utool as ut # NOQA from os.path import join from ibeis.dbio import ingest_hsdb import ibeis.other.dbinfo ibeis.other.dbinfo.rrr() workdir = sysres.get_workdir() dbname_list = os.listdir(workdir) dbpath_list = [join(workdir, name) for name in dbname_list] is_hsdb_list = list(map(ingest_hsdb.is_hsdb, dbpath_list)) hsdb_list = ut.compress(dbpath_list, is_hsdb_list) #is_ibs_cvt_list = np.array(list(map(is_succesful_convert, dbpath_list))) regen_cmds = [] for hsdb_dpath in hsdb_list: if hsdb_dpath in canskip: continue try: ibs = ibeis.opendb(hsdb_dpath) # NOQA print('Succesfully opened hsdb: ' + hsdb_dpath) print(ibs.get_dbinfo_str()) except Exception as ex: ut.printex(ex, 'Failed to convert hsdb: ' + hsdb_dpath) regen_cmd = 'python dev.py --convert --dbdir ' + hsdb_dpath regen_cmds.append(regen_cmd) print('\n'.join(regen_cmds))
def get_unconverted_hsdbs(workdir=None): import os import numpy as np from vtool import linalg as ltool if workdir is None: workdir = sysres.get_workdir() dbname_list = os.listdir(workdir) dbpath_list = np.array([join(workdir, name) for name in dbname_list]) is_hsdb_list = np.array(map(sysres.is_hsdb, dbpath_list)) is_ibs_cvt_list = np.array(map(is_succesful_convert, dbpath_list)) if FORCE_DELETE: needs_convert = is_hsdb_list else: needs_convert = ltool.and_lists(is_hsdb_list, True - is_ibs_cvt_list) needs_convert_hsdbs = dbpath_list[needs_convert].tolist() return needs_convert_hsdbs
def ensure_pz_mtest(): """ Ensures that you have the PZ_MTEST dataset CommandLine: python -m ibeis.init.sysres --exec-ensure_pz_mtest python -m ibeis --tf ensure_pz_mtest Example: >>> # SCRIPT >>> from ibeis.init.sysres import * # NOQA >>> ensure_pz_mtest() """ print('ensure_pz_mtest') from ibeis import sysres workdir = sysres.get_workdir() mtest_zipped_url = const.ZIPPED_URLS.PZ_MTEST mtest_dir = ut.grab_zipped_url(mtest_zipped_url, ensure=True, download_dir=workdir) print('have mtest_dir=%r' % (mtest_dir,)) # update the the newest database version import ibeis ibs = ibeis.opendb('PZ_MTEST') print('cleaning up old database and ensureing everything is properly computed') ibs.db.vacuum() valid_aids = ibs.get_valid_aids() assert len(valid_aids) == 119 ibs.update_annot_semantic_uuids(valid_aids) if ut.VERYVERBOSE: ibs.print_annotation_table() nid = ibs.get_name_rowids_from_text('', ensure=False) if nid is not None: ibs.set_name_texts([nid], ['lostname']) # hack in some tags print('Hacking in some tags') foal_aids = [4, 8, 15, 21, 28, 34, 38, 41, 45, 49, 51, 56, 60, 66, 69, 74, 80, 83, 91, 97, 103, 107, 109, 119] mother_aids = [9, 16, 35, 42, 52, 57, 61, 67, 75, 84, 98, 104, 108, 114] ibs.append_annot_case_tags(foal_aids, ['foal'] * len(foal_aids)) ibs.append_annot_case_tags(mother_aids, ['mother'] * len(mother_aids))
def ensure_pz_mtest(): """ Ensures that you have the PZ_MTEST dataset CommandLine: python -m ibeis.init.sysres --exec-ensure_pz_mtest python -m ibeis --tf ensure_pz_mtest Example: >>> # SCRIPT >>> from ibeis.init.sysres import * # NOQA >>> ensure_pz_mtest() """ print('ensure_pz_mtest') from ibeis import sysres workdir = sysres.get_workdir() mtest_zipped_url = const.ZIPPED_URLS.PZ_MTEST mtest_dir = ut.grab_zipped_url(mtest_zipped_url, ensure=True, download_dir=workdir) print('have mtest_dir=%r' % (mtest_dir, )) # update the the newest database version import ibeis ibs = ibeis.opendb('PZ_MTEST') print( 'cleaning up old database and ensureing everything is properly computed' ) ibs.db.vacuum() valid_aids = ibs.get_valid_aids() assert len(valid_aids) == 119 ibs.update_annot_semantic_uuids(valid_aids) if ut.VERYVERBOSE: ibs.print_annotation_table() nid = ibs.get_name_rowids_from_text('', ensure=False) if nid is not None: ibs.set_name_texts([nid], ['lostname']) # Remove old imagesets and update to new special ones all_imgset_ids = ibs.get_valid_imgsetids() special_imgset_ids = ibs.get_special_imgsetids() other_imgset_ids = ut.setdiff(all_imgset_ids, special_imgset_ids) ibs.delete_imagesets(other_imgset_ids) ibs.set_exemplars_from_quality_and_viewpoint() ibs.update_all_image_special_imageset() occurrence_gids = [ 2, 9, 12, 16, 25, 26, 29, 30, 32, 33, 35, 46, 47, 52, 57, 61, 66, 70, 71, 73, 74, 76, 77, 78, 79, 87, 88, 90, 96, 97, 103, 106, 108, 110, 112, 113 ] other_gids = ut.setdiff(ibs.get_valid_gids(), occurrence_gids) other_gids1 = other_gids[0::2] other_gids2 = other_gids[1::2] ibs.set_image_imagesettext(occurrence_gids, ['Occurrence 1'] * len(occurrence_gids)) ibs.set_image_imagesettext(other_gids1, ['Occurrence 2'] * len(other_gids1)) ibs.set_image_imagesettext(other_gids2, ['Occurrence 3'] * len(other_gids2)) # hack in some tags print('Hacking in some tags') foal_aids = [ 4, 8, 15, 21, 28, 34, 38, 41, 45, 49, 51, 56, 60, 66, 69, 74, 80, 83, 91, 97, 103, 107, 109, 119 ] mother_aids = [9, 16, 35, 42, 52, 57, 61, 67, 75, 84, 98, 104, 108, 114] ibs.append_annot_case_tags(foal_aids, ['foal'] * len(foal_aids)) ibs.append_annot_case_tags(mother_aids, ['mother'] * len(mother_aids))
def ensure_db_from_url(zipped_db_url): """ SeeAlso ibeis.init.sysres """ from ibeis import sysres workdir = sysres.get_workdir() dbdir = ut.grab_zipped_url(zipped_url=zipped_db_url, ensure=True, download_dir=workdir) print('have %s=%r' % (zipped_db_url, dbdir,))