def cx2_rr(qm, qcx, qhs=None): hs = qm.hs qhs = hs if qhs is None else qhs rr = RawResults(qcx, hs, qhs) qm.rr = rr # Save in query database. rr_dpath = qhs.iom.ensure_computed_directory('query_results') # Filename points to result database depends = ['chiprep', 'preproc', 'model', 'query'] algo_suffix = hs.am.get_algo_suffix(depends) samp_suffix = hs.vm.get_samp_suffix() rr_suffix = algo_suffix + samp_suffix # Populate raw results force_recompute = False #TODO PREFERENCE if rr.has_result(rr_dpath, rr_suffix) and not force_recompute: logdbg('Loading query.') rr.load_result(rr_dpath, rr_suffix) else: logmsg('Recomputing query.') (qfpts, qfdsc) = qhs.cm.get_feats(qcx) rr.qfpts = qfpts rr.qfdsc = qfdsc rr.qchip_size = qhs.cm.cx2_chip_size(qcx) qm.repopulate_raw_results(rr, hs, qhs) rr.save_result(rr_dpath, rr_suffix) return rr
def load_image_table(iom): logmsg("Loading Image Table") img_table_fpath = iom.get_image_table_fpath() if not exists(img_table_fpath): img_table_fpath = iom._check_altfname(alt_names=["image_table.csv"]) image_csv_func = lambda f, d: iom.__image_csv_func(f, d) return iom._load_table(img_table_fpath, "Image", iom.hs.gm.img_alloc, image_csv_func)
def select_images_on_disk(uim, start_path=None): dlg = QFileDialog() logmsg('Select one or more images to add.') image_list = dlg.getOpenFileNames(caption='Select one or more images to add.',\ directory=uim.hs.db_dpath) image_list = [str(fpath) for fpath in image_list] return image_list
def delete_computed_cid(cm, cid): iom = cm.hs.iom if np.iterable(cid): logerr('this function only works for a single cid') logmsg('Removing CID=%d\'s computed files' % cid) cid_fname_pattern = iom.get_chip_prefix(cid, []) + '*' iom.remove_computed_files_with_pattern(cid_fname_pattern)
def load_chip_table(iom): logmsg("Loading Chip Table") chip_table_fpath = iom.get_chip_table_fpath() if not exists(chip_table_fpath): alt_names = ["chip_table.csv", "instance_table.csv", "animal_info_table.csv", "SightingData.csv"] chip_table_fpath = iom._check_altfname(alt_names=alt_names) return iom._load_table(chip_table_fpath, "Chip", iom.hs.cm.chip_alloc, iom.hs.cm.load_csv_line)
def add_roi_to_all_images(hs): cm, gm, nm = hs.get_managers('cm','gm','nm') gx_list = gm.get_empty_gxs() logmsg('Adding '+str(len(gx_list))+' rois to empty images') for gx in gx_list: (gw, gh) = gm.gx2_img_size(gx) cm.add_chip(-1, nm.UNIDEN_NX(), gx, [0, 0, gw, gh], 0)
def load_chip_table(iom): logmsg('Loading Chip Table') chip_table_fpath = iom.get_chip_table_fpath() if not exists(chip_table_fpath): alt_names=['chip_table.csv','instance_table.csv','animal_info_table.csv','SightingData.csv'] chip_table_fpath = iom._check_altfname(alt_names=alt_names) return iom._load_table(chip_table_fpath, 'Chip', iom.hs.cm.chip_alloc, iom.hs.cm.load_csv_line)
def load_name_table(iom): logmsg('Loading Name Table') name_table_fpath = iom.get_name_table_fpath() if not exists(name_table_fpath): name_table_fpath = iom._check_altfname(alt_names=['name_table.csv']) name_csv_func = lambda f,d: iom.__name_csv_func(f,d) return iom._load_table(name_table_fpath, 'Name', iom.hs.nm.name_alloc, name_csv_func)
def load_image_table(iom): logmsg('Loading Image Table') img_table_fpath = iom.get_image_table_fpath() if not exists(img_table_fpath): img_table_fpath = iom._check_altfname(alt_names=['image_table.csv']) image_csv_func = lambda f,d: iom.__image_csv_func(f,d) return iom._load_table(img_table_fpath, 'Image', iom.hs.gm.img_alloc, image_csv_func)
def rename_chip(cm, cx, new_name): nm = cm.hs.nm cid = cm.cid(cx) old_nx = cm.cx2_nx[cx] old_name = nm.nx2_name[old_nx] if old_name == new_name: logdbg('new_name == old_name') return logmsg('Renaming cid=' + str(cid) + ' from ' + str(old_name) + ' to ' + new_name) if not new_name in nm.name2_nx.keys(): nm.add_name(-1, new_name) old_nx = cm.cx2_nx[cx] new_nx = nm.name2_nx[new_name] #Debug old_nid = nm.nx2_nid[old_nx] new_nid = nm.nx2_nid[new_nx] logdbg('Old Name Info: cid=%d cx=%d, nid=%d, nx=%d, name=%s' % (cid, cx, old_nid, old_nx, old_name)) logdbg('New Name Info: cid=%d cx=%d, nid=%d, nx=%d, name=%s' % (cid, cx, new_nid, new_nx, new_name)) #EndDebug nm.nx2_cx_list[old_nx].remove(cx) nm.nx2_cx_list[new_nx].append(cx) cm.cx2_nx[cx] = new_nx
def load_name_table(iom): logmsg("Loading Name Table") name_table_fpath = iom.get_name_table_fpath() if not exists(name_table_fpath): name_table_fpath = iom._check_altfname(alt_names=["name_table.csv"]) name_csv_func = lambda f, d: iom.__name_csv_func(f, d) return iom._load_table(name_table_fpath, "Name", iom.hs.nm.name_alloc, name_csv_func)
def change_roi(cm, cx, new_roi): cid = cm.cx2_cid[cx] logmsg('Giving cid=%d new roi: %r' % (cid, new_roi)) assert not new_roi is None if new_roi is None: logerr('The ROI is np.empty') cm.hs.on_cx_modified(cx) cm.cx2_roi[cx] = new_roi
def reset(vm): logmsg('Reseting the Visual Model') vm.isDirty = True vm.wx2_fdsc = np.array([], dtype=np.uint8) vm.wx2_axs = [] vm.wx2_idf = np.array([], dtype=np.float32) vm.wx2_maxtf = np.array([], dtype=np.float32) vm.ax2_cid = np.array([], dtype=np.uint32) vm.ax2_fx = np.array([], dtype=np.uint32)
def write(iom, fpath, to_write): logmsg('Writing to: %s' % fpath) print 'Writing String:\n%s' % to_write try: with open(fpath, 'w') as f: f.write(to_write) print 'Wrote to %s' % fpath except Exception as ex: print 'Error: '+str(ex) print 'Failed to write to %s ' % fpath
def write(iom, fpath, to_write): logmsg("Writing to: %s" % fpath) print "Writing String:\n%s" % to_write try: with open(fpath, "w") as f: f.write(to_write) print "Wrote to %s" % fpath except Exception as ex: print "Error: " + str(ex) print "Failed to write to %s " % fpath
def unload_features(cm, cxs): if not np.iterable(cxs): cxs = [cxs] nRequest = len(cxs) nUnload = nRequest - np.sum(cm.cx2_dirty_bit[cxs]) # Print unloaded cxs unless there are more than 3 logdbg('Unloading features: %r' % cxs) logmsg('Unloading %d/%d features: ' % (nUnload, nRequest)) cm.cx2_fpts[cxs] = np.empty(nUnload,dtype=object) cm.cx2_fdsc[cxs] = np.empty(nUnload,dtype=object) cm.cx2_fdsc[cxs] = np.empty(nUnload,dtype=object) cm.cx2_dirty_bit[cxs] = True
def unload_features(cm, cxs): if not np.iterable(cxs): cxs = [cxs] nRequest = len(cxs) nUnload = nRequest - np.sum(cm.cx2_dirty_bit[cxs]) # Print unloaded cxs unless there are more than 3 logdbg('Unloading features: %r' % cxs) logmsg('Unloading %d/%d features: ' % (nUnload, nRequest)) cm.cx2_fpts[cxs] = np.empty(nUnload, dtype=object) cm.cx2_fdsc[cxs] = np.empty(nUnload, dtype=object) cm.cx2_fdsc[cxs] = np.empty(nUnload, dtype=object) cm.cx2_dirty_bit[cxs] = True
def remove_files_with_pattern(iom, dpath, fname_pattern, recursive_bit=True): logdbg('Removing files in directory %r %s' % (dpath, ['', ', Recursively'][recursive_bit])) logdbg('Removing files with pattern: %r' % fname_pattern) num_removed = 0 num_matched = 0 for root, dname_list, fname_list in os.walk(dpath): for fname in fnmatch.filter(fname_list, fname_pattern): num_matched += 1 num_removed += iom.remove_file(join(root, fname)) if not recursive_bit: break logmsg('Removed %d/%d files' % (num_removed, num_matched)) return True
def gui_log_wrapper(hsgui, *args, **kwargs): try: function_name = fn.func_name into_str = 'In hsgui.'+function_name outo_str = 'Out hsgui.'+function_name+'\n' hsgui.logdbgSignal.emit(into_str) ret = fn(hsgui, *args, **kwargs) hsgui.logdbgSignal.emit(outo_str) return ret except Exception as ex: import traceback logmsg('\n\n *!!* HotSpotter GUI Raised Exception: '+str(ex)) logmsg('\n\n *!!* HotSpotter GUI Exception Traceback: \n\n'+traceback.format_exc())
def remove_files_with_pattern(iom, dpath, fname_pattern, recursive_bit=True): logdbg("Removing files in directory %r %s" % (dpath, ["", ", Recursively"][recursive_bit])) logdbg("Removing files with pattern: %r" % fname_pattern) num_removed = 0 num_matched = 0 for root, dname_list, fname_list in os.walk(dpath): for fname in fnmatch.filter(fname_list, fname_pattern): num_matched += 1 num_removed += iom.remove_file(join(root, fname)) if not recursive_bit: break logmsg("Removed %d/%d files" % (num_removed, num_matched)) return True
def batch_rename(hs, name1, name2): logmsg('Batch Renaming %s to %s' % (name1, name2)) cm, nm = hs.get_managers('cm','nm') if name1 == nm.UNIDEN_NAME(): logerr('Cannot batch rename '+str(name1)+'. It is UNIDENTIFIED and has special meaning') if name1 not in nm.name2_nx.keys(): logerr('Cannot batch rename. '+str(name1)+' does not exist') cx_list = nm.name2_cx_list(name1)[:] # COPY BEFORE YOU CHANGE. Man, sneaky errors num_chips = len(cx_list) if num_chips == 0: logerr('Cannot batch rename. '+str(name1)+' has no chips') logmsg('Renaming '+str(num_chips)+' chips: '+str(cx_list)) for cx in cx_list: logdbg('Batch Rename '+str(cx)) cm.rename_chip(cx, name2) return True
def smartget_db_dpath(hs, db_dpath): ''' Performs a smart update of the db_dpath Trys a number of various options to get it right None = Read from preferences '' = Prompt the User For database ''' if db_dpath is None: # If requested to read prefs db_dpath = str(hs.core_prefs.database_dpath) if db_dpath in [None, 'None'] or\ not os.path.exists(db_dpath): # Check validity logwarn('db_dpath='+repr(db_dpath)+' is invalid') db_dpath = '' if db_dpath == '': # Prompt The User. TODO Move this to Facade/UIManager logmsg('what database should I open?') try: db_dpath = hs.uim.select_database() except: logerr(' Was unable to prompt user with QT') return db_dpath
def load_model(vm): # See if the model is loadable if not vm.model_prefs.save_load_model: logdbg('Can NOT load the visual model') return False if not vm.flann is None: raise Exception('Cannot load a model when FLANN already exists') logdbg('Trying to load visual model') # Check to see if new model on disk model_fpath = vm.hs.iom.get_model_fpath() if not os.path.exists(model_fpath): logdbg(' * A saved model data file was missing: '+ model_fpath); return False flann_index_fpath = vm.hs.iom.get_flann_index_fpath() if not os.path.exists(flann_index_fpath): logdbg(' * A saved flann index file was missing: '+ flann_index_fpath); return False # Model and Flann Exist on disk # Load the model data first # Read model into dictionary logmsg('Loading visual model data: ' + model_fpath) npz = np.load(model_fpath) for _key in npz.files: vm.__dict__[_key] = npz[_key] npz.close() # Read FLANN index logmsg('Loading FLANN index: '+ flann_index_fpath) vm.flann = FLANN() vm.flann.load_index(flann_index_fpath, vm.wx2_fdsc) vm.isDirty = False logmsg('The model was sucessfully loaded') return True
def remove_chip(cm, cx): cx_list = [cx] if type(cx) == types.ListType: cx_list = cx logdbg('Removing CXs '+str(cx_list)) for cx in cx_list: # Remove data saved on disk and memory cm.hs.on_cx_modified(cx) cid = cm.cx2_cid[cx] logmsg('Removing cid=%d' % cid) # Remove cx from other.data managers gx = cm.cx2_gx[cx] nx = cm.cx2_nx[cx] cm.hs.gm.gx2_cx_list[gx].remove(cx) cm.hs.nm.nx2_cx_list[nx].remove(cx) # Remove data saved in memory cm.cx2_cid[cx] = 0 cm.cx2_nx[cx] = 0 cm.cx2_gx[cx] = 0 cm.cx2_roi[cx] = np.array([0,0,0,0],dtype=np.uint32) cm.cx2_theta[cx] = 0 cm.cid2_cx[cid] = 0
def rename_chip(cm, cx, new_name): nm = cm.hs.nm cid = cm.cid(cx) old_nx = cm.cx2_nx[cx] old_name = nm.nx2_name[old_nx] if old_name == new_name: logdbg('new_name == old_name') return logmsg('Renaming cid='+str(cid)+' from '+str(old_name)+' to '+new_name) if not new_name in nm.name2_nx.keys(): nm.add_name(-1,new_name) old_nx = cm.cx2_nx[cx] new_nx = nm.name2_nx[new_name] #Debug old_nid = nm.nx2_nid[old_nx] new_nid = nm.nx2_nid[new_nx] logdbg('Old Name Info: cid=%d cx=%d, nid=%d, nx=%d, name=%s' % (cid, cx, old_nid, old_nx, old_name)) logdbg('New Name Info: cid=%d cx=%d, nid=%d, nx=%d, name=%s' % (cid, cx, new_nid, new_nx, new_name)) #EndDebug nm.nx2_cx_list[old_nx].remove(cx) nm.nx2_cx_list[new_nx].append(cx) cm.cx2_nx[cx] = new_nx
def remove_chip(cm, cx): cx_list = [cx] if type(cx) == types.ListType: cx_list = cx logdbg('Removing CXs ' + str(cx_list)) for cx in cx_list: # Remove data saved on disk and memory cm.hs.on_cx_modified(cx) cid = cm.cx2_cid[cx] logmsg('Removing cid=%d' % cid) # Remove cx from other.data managers gx = cm.cx2_gx[cx] nx = cm.cx2_nx[cx] cm.hs.gm.gx2_cx_list[gx].remove(cx) cm.hs.nm.nx2_cx_list[nx].remove(cx) # Remove data saved in memory cm.cx2_cid[cx] = 0 cm.cx2_nx[cx] = 0 cm.cx2_gx[cx] = 0 cm.cx2_roi[cx] = np.array([0, 0, 0, 0], dtype=np.uint32) cm.cx2_theta[cx] = 0 cm.cid2_cx[cid] = 0
def batch_query(em, force_recomp=False, test_cxs=None): '''Runs each test_cxs as a query. If test_cxs is None, then all queries are run''' 'TODO: Fix up the VM dependencies' vm, iom, am, cm = em.hs.get_managers('vm', 'iom', 'am', 'cm') # Compute the matches qm = vm.hs.qm vm.sample_train_set() vm.build_model(force_recomp=force_recomp) if test_cxs == None: test_cxs = vm.get_train_cx() logmsg('Building matching graph. This may take awhile') depends = ['chiprep', 'preproc', 'model', 'query'] algo_suffix = am.get_algo_suffix(depends) samp_suffix = vm.get_samp_suffix() result_dpath = iom.ensure_directory(iom.get_temp_fpath('raw_results')) rr_fmtstr_cid = os.path.join( result_dpath, 'rr_cid%07d' + samp_suffix + algo_suffix + '.pkl') # Find the Queries which need to be run unsaved_cxs = [] for cx in iter(test_cxs): cid = cm.cx2_cid[cx] rr_fpath = rr_fmtstr_cid % cid if not os.path.exists(rr_fpath): unsaved_cxs.append(cx) # Run Unsaved Query total = len(unsaved_cxs) for count, cx in enumerate(unsaved_cxs): logmsg('Query %d/%d' % (count, total)) em.run_and_save_query(cx, rr_fmtstr_cid) # Read Each Query cx2_rr = alloc_lists(test_cxs.max() + 1) total = len(test_cxs) for count, cx in enumerate(test_cxs): logmsg('Loading Result %d/%d' % (count, total)) cid = cm.cx2_cid[cx] rr_fpath = rr_fmtstr_cid % cid if not os.path.exists(rr_fpath): logwarn('Result does not exist for CID=%d' % cid) rr_file = open(rr_fpath, 'rb') try: rr = cPickle.load(rr_file) except EOFError: rr_file.close() os.remove(rr_fpath) logwarn('Result was corrupted for CID=%d' % cid) rr_file.close() rr.cx2_cscore_ = [] rr.cx2_fs_ = [] rr.qfdsc = [] rr.qfpts = [] cx2_rr[cx] = rr return cx2_rr
def batch_query(em, force_recomp=False, test_cxs=None): '''Runs each test_cxs as a query. If test_cxs is None, then all queries are run''' 'TODO: Fix up the VM dependencies' vm, iom, am, cm = em.hs.get_managers('vm','iom','am', 'cm') # Compute the matches qm = vm.hs.qm vm.sample_train_set() vm.build_model(force_recomp=force_recomp) if test_cxs == None: test_cxs = vm.get_train_cx() logmsg('Building matching graph. This may take awhile') depends = ['chiprep','preproc','model','query'] algo_suffix = am.get_algo_suffix(depends) samp_suffix = vm.get_samp_suffix() result_dpath = iom.ensure_directory(iom.get_temp_fpath('raw_results')) rr_fmtstr_cid = os.path.join(result_dpath, 'rr_cid%07d'+samp_suffix+algo_suffix+'.pkl') # Find the Queries which need to be run unsaved_cxs = [] for cx in iter(test_cxs): cid = cm.cx2_cid[cx] rr_fpath = rr_fmtstr_cid % cid if not os.path.exists(rr_fpath): unsaved_cxs.append(cx) # Run Unsaved Query total = len(unsaved_cxs) for count, cx in enumerate(unsaved_cxs): logmsg('Query %d/%d' % (count, total)) em.run_and_save_query(cx, rr_fmtstr_cid) # Read Each Query cx2_rr = alloc_lists(test_cxs.max()+1) total = len(test_cxs) for count, cx in enumerate(test_cxs): logmsg('Loading Result %d/%d' % (count, total)) cid = cm.cx2_cid[cx] rr_fpath = rr_fmtstr_cid % cid if not os.path.exists(rr_fpath): logwarn('Result does not exist for CID=%d' % cid) rr_file = open(rr_fpath,'rb') try: rr = cPickle.load(rr_file) except EOFError: rr_file.close() os.remove(rr_fpath) logwarn('Result was corrupted for CID=%d' % cid) rr_file.close() rr.cx2_cscore_ = [] rr.cx2_fs_ = [] rr.qfdsc = [] rr.qfpts = [] cx2_rr[cx] = rr return cx2_rr
def populate_result_table(uim): col_headers = ['Rank', 'Chip ID', 'Chip Name', 'score'] col_editable = [False , False , True , False ] # Check to see if results exist res = uim.sel_res if res is None: logdbg('Not populating. selected results are None.') return None logmsg(res) gm, cm, am = uim.hs.get_managers('gm','cm','am') dynargs =\ ('cid', 'name' ) (qcid , qname ) = res.qcid2_(*dynargs) (tcid , tname , tscore ) = res.tcid2_(*dynargs+('score',)) num_results = len(tcid) data_list = [None]*(num_results+1) row_list = range(num_results+1) data_list[0] = [0, qcid, qname, 'Queried Chip'] for (ix, (cid, name, score)) in enumerate(zip(tcid, tname, tscore)): rank = ix+1 data_list[ix+1] = (rank, cid, name, score) uim.populateResultTblSignal.emit(col_headers, col_editable, row_list, data_list)
def populate_result_table(uim): col_headers = ['Rank', 'Chip ID', 'Chip Name', 'score'] col_editable = [False, False, True, False] # Check to see if results exist res = uim.sel_res if res is None: logdbg('Not populating. selected results are None.') return None logmsg(res) gm, cm, am = uim.hs.get_managers('gm', 'cm', 'am') dynargs =\ ('cid', 'name' ) (qcid, qname) = res.qcid2_(*dynargs) (tcid, tname, tscore) = res.tcid2_(*dynargs + ('score', )) num_results = len(tcid) data_list = [None] * (num_results + 1) row_list = range(num_results + 1) data_list[0] = [0, qcid, qname, 'Queried Chip'] for (ix, (cid, name, score)) in enumerate(zip(tcid, tname, tscore)): rank = ix + 1 data_list[ix + 1] = (rank, cid, name, score) uim.populateResultTblSignal.emit(col_headers, col_editable, row_list, data_list)
def load_tables(iom): logmsg('Loading data tables in '+iom.hs.db_dpath) if not (exists(iom.get_image_table_fpath()) and\ exists(iom.get_name_table_fpath()) and\ exists(iom.get_image_table_fpath())): if exists(iom.get_oxford_gt_dpath()): logmsg('You have selected an Oxford style groundtruth') iom.load_oxford_gt() logmsg('Succesfully Loaded Oxford style groundtruth') sys.stdout.flush() return logwarn('Trying to load a Legacy Database') iom.load_image_table() iom.load_name_table() iom.load_chip_table() logmsg('Done loading data tables') sys.stdout.flush()
def load_tables(iom): logmsg("Loading data tables in " + iom.hs.db_dpath) if not ( exists(iom.get_image_table_fpath()) and exists(iom.get_name_table_fpath()) and exists(iom.get_image_table_fpath()) ): if exists(iom.get_oxford_gt_dpath()): logmsg("You have selected an Oxford style groundtruth") iom.load_oxford_gt() logmsg("Succesfully Loaded Oxford style groundtruth") sys.stdout.flush() return logwarn("Trying to load a Legacy Database") iom.load_image_table() iom.load_name_table() iom.load_chip_table() logmsg("Done loading data tables") sys.stdout.flush()
def add_image_list(hs, image_list): num_add = len(image_list) logmsg('Selected '+str(num_add)+' images to import') prev_g = hs.gm.num_g logdbg('Prev #g=%d' % prev_g) for src_img in image_list: hs.gm.add_img(gid=None, gname=None, aif=False, src_img=src_img) post_g = hs.gm.num_g num_new = (post_g - prev_g) num_old = num_add - num_new logdbg('Post #g=%d' % post_g) logmsg('Imported '+str(num_new)+' new images') if num_old != 0: logmsg('%d Images had already been copied into the image directory' % num_old)
def annotate_orientation(dm): logmsg('Please select an orientation of the torso (Click Two Points on the Image)') try: # Compute an angle from user interaction sys.stdout.flush() fig = dm.get_figure() pts = np.array(fig.ginput(2)) logdbg('GInput Points are: '+str(pts)) # Get reference point to origin refpt = pts[0] - pts[1] #theta = np.math.atan2(refpt[1], refpt[0]) theta = np.math.atan(refpt[1]/refpt[0]) logmsg('The angle in radians is: '+str(theta)) return theta except Exception as ex: logmsg('Annotate Orientation Failed'+str(ex)) return None
def annotate_orientation(dm): logmsg("Please select an orientation of the torso (Click Two Points on the Image)") try: # Compute an angle from user interaction sys.stdout.flush() fig = dm.get_figure() pts = np.array(fig.ginput(2)) logdbg("GInput Points are: " + str(pts)) # Get reference point to origin refpt = pts[0] - pts[1] # theta = np.math.atan2(refpt[1], refpt[0]) theta = np.math.atan(refpt[1] / refpt[0]) logmsg("The angle in radians is: " + str(theta)) return theta except Exception as ex: logmsg("Annotate Orientation Failed" + str(ex)) return None
def is_valid_db_dpath(hs, db_dpath): 'Checks to see if database conforms to expected conventions' if not os.path.exists(db_dpath): logwarn('db_dpath \"'+str(db_dpath)+'\" doesnt exist') return False db_dpath_files = os.listdir(db_dpath) if hs.iom.internal_dname in db_dpath_files: logmsg('Opening a HotSpotter database: '+db_dpath) elif 'images' in db_dpath_files or\ 'data' in db_dpath_files: logmsg('Opening a StripSpotter database: '+db_dpath) elif len(db_dpath_files) == 0: logmsg('Creating a new database: '+db_dpath) else: logwarn('Unknown database type: '+db_dpath) logdbg('Files in dir: '+str(db_dpath_files)) return False return True
def annotate_roi(dm): logmsg("Please Select a Rectangular Region of Interest (Click Two Points on the Image)") try: sys.stdout.flush() fig = dm.get_figure() pts = fig.ginput(2) logdbg("GInput Points are: " + str(pts)) (x1, y1, x2, y2) = (pts[0][0], pts[0][1], pts[1][0], pts[1][1]) xm = min(x1, x2) xM = max(x1, x2) ym = min(y1, y2) yM = max(y1, y2) (x, y, w, h) = (xm, ym, xM - xm, yM - ym) roi = array(round([x, y, w, h]), dtype=uint32) logmsg("The new ROI is: " + str(roi)) return roi except Exception as ex: logmsg("Annotate ROI Failed" + str(ex)) return None
def annotate_roi(dm): logmsg('Please Select a Rectangular Region of Interest (Click Two Points on the Image)') try: sys.stdout.flush() fig = dm.get_figure() pts = fig.ginput(2) logdbg('GInput Points are: '+str(pts)) (x1, y1, x2, y2) = (pts[0][0], pts[0][1], pts[1][0], pts[1][1]) xm = min(x1,x2) xM = max(x1,x2) ym = min(y1,y2) yM = max(y1,y2) (x, y, w, h) = (xm, ym, xM-xm, yM-ym) roi = array(round([x,y,w,h]),dtype=uint32) logmsg('The new ROI is: '+str(roi)) return roi except Exception as ex: logmsg('Annotate ROI Failed'+str(ex)) return None
def build_model(vm, force_recomp=False): ''' Builds the model, if needed. Tries to reload if it can ''' logmsg('\n\nRequested: Build Model') if not force_recomp and not vm.isDirty: logmsg('The model is clean and is not forced to recompute') return True cm = vm.hs.cm # Delete old index and resample chips to index vm.delete_model() vm.sample_train_set() # Try to load the correct model if not force_recomp and vm.load_model(): logmsg('Loaded saved model from disk') return logmsg('Building the model. This may take some time.') # Could not load old model. Do full rebuild # ----- # STEP 1 - Loading logdbg('Step 1: Aggregate the model support (Load feature vectors) ---') tx2_cx = vm.get_train_cx() tx2_cid = vm.get_train_cid() assert len(tx2_cx) > 0, 'Training set cannot be np.empty' logdbg('Building model with %d sample chips' % (vm.num_train())) cm.load_features(tx2_cx) tx2_nfpts = cm.cx2_nfpts(tx2_cx) num_train_keypoints = sum(tx2_nfpts) # ----- # STEP 2 - Aggregating logdbg('Step 2: Build the model Words') isTFIDF = False if vm.hs.am.algo_prefs.model.quantizer == 'naive_bayes': logdbg('No Quantization. Aggregating all fdscriptors for nearest neighbor search.') vm.wx2_fdsc = np.empty((num_train_keypoints,128),dtype=np.uint8) _p = 0 for cx in tx2_cx: nfdsc = cm.cx2_nfpts(cx) vm.wx2_fdsc[_p:_p+nfdsc,:] = cm.cx2_fdsc[cx] _p += nfdsc ax2_wx = np.array(range(0,num_train_keypoints),dtype=np.uint32) if vm.hs.am.algo_prefs.model.quantizer == 'akmeans': raise NotImplementedError(':)') # ----- # STEP 3 - Inverted Indexing logdbg('Step 3: Point the parts of the model back to their source') vm.wx2_axs = np.empty(vm.wx2_fdsc.shape[0], dtype=object) for ax in xrange(0,num_train_keypoints): if vm.wx2_axs[ax] is None: vm.wx2_axs[ax] = [] wx = ax2_wx[ax] vm.wx2_axs[wx].append(ax) vm.ax2_cid = -np.ones(num_train_keypoints,dtype=np.int32) vm.ax2_fx = -np.ones(num_train_keypoints,dtype=np.int32) ax2_tx = -np.ones(num_train_keypoints,dtype=np.int32) curr_fx = 0; next_fx = 0 for tx in xrange(vm.num_train()): nfpts = tx2_nfpts[tx] next_fx = next_fx + nfpts ax_range = range(curr_fx,next_fx) ax2_tx[ax_range] = tx vm.ax2_cid[ax_range] = tx2_cid[tx] # Point to Inst vm.ax2_fx[ax_range] = range(nfpts) # Point to Kpts curr_fx = curr_fx + nfpts if isTFIDF: # Compute info for TF-IDF logdbg('Computing TF-IDF metadata') max_tx = len(tx2_cx) tx2_wtf_denom = np.float32(cm.cx2_nfpts(tx2_cx)) vm.wx2_maxtf = map(lambda ax_of_wx:\ max( np.float32(bincount(ax2_tx[ax_of_wx], minlength=max_tx)) / tx2_wtf_denom ), vm.wx2_axs) vm.wx2_idf = np.log2(map(lambda ax_of_wx:\ vm.num_train()/len(pylab.unique(ax2_tx[ax_of_wx])),\ vm.wx2_axs)+eps(1)) logdbg('Built Model using %d feature vectors. Preparing to index.' % len(vm.ax2_cid)) # ----- # STEP 4 - Indexing logdbg('Step 4: Building FLANN Index: over '+str(len(vm.wx2_fdsc))+' words') assert vm.flann is None, 'Flann already exists' vm.flann = FLANN() flann_param_dict = vm.hs.am.algo_prefs.model.indexer.to_dict() flann_params = vm.flann.build_index(vm.wx2_fdsc, **flann_param_dict) vm.isDirty = False vm.save_model() logmsg('The model was built.')
def load_oxford_gt(iom): 'loads oxford style groundtruth' gm,cm,nm = iom.hs.get_managers('gm','cm','nm') # Check for corrupted files (Looking at your Paris Buildings Dataset) oxford_gt_dpath = iom.get_oxford_gt_dpath() corrupted_gname_list = [] corrupted_file_fname = 'corrupted_files.txt' corrupted_file_fpath = join(oxford_gt_dpath,corrupted_file_fname) if exists(corrupted_file_fpath): with open(corrupted_file_fpath) as f: corrupted_gname_list = f.read().splitlines() logmsg('Loading Oxford Style Images') #Recursively get relative path of all files in img_dpath img_dpath = iom.get_img_dpath() #with a sexy list comprehension gname_list = [join(relpath(root, img_dpath), fname).replace('\\','/').replace('./','')\ for (root,dlist,flist) in os.walk(img_dpath)\ for fname in flist] #Roughly Prealloc gm.img_alloc( len(gname_list)) cm.chip_alloc(len(gname_list)) #Add all images in images directory (allow nested directories (...paris)) for gname in gname_list: if gname in corrupted_gname_list: continue gm.add_img(-1, gname, True) logmsg('Loading Oxford Style Names and Chips') # Add names and chips from ground truth gt_fname_list = os.listdir(oxford_gt_dpath) iom.hs.nm.name_alloc(len(gt_fname_list)/4) for gt_fname in gt_fname_list: if gt_fname == corrupted_file_fname: continue #Get gt_name, quality, and num from fname gt_name = gt_fname.replace('.txt','') _pos1 = gt_name.rfind('_') quality = gt_name[_pos1+1:] gt_name = gt_name[:_pos1] _pos2 = gt_name.rfind('_') num = gt_name[_pos2+1:] gt_name = gt_name[:_pos2] # Add Name (-2 suppresses warnings) nid = nm.add_name(-2, gt_name) nx = nm.nid2_nx[nid] gt_fpath = join(oxford_gt_dpath, gt_fname) with open(gt_fpath,'r') as f: line_list = f.read().splitlines() for line in line_list: if line == '': continue fields = line.split(' ') gname = fields[0].replace('oxc1_','')+'.jpg' if gname.find('paris_') >= 0: # PARIS HACK >:( #Because they just cant keep their paths consistent paris_hack = gname[6:gname.rfind('_')] gname = paris_hack+'/'+gname if gname in corrupted_gname_list: continue gid = gm.gname2_gid[gname] gx = gm.gid2_gx[gid] if len(fields) > 1: #quality == query roi = map(lambda x: int(round(float(x))),fields[1:]) else: # quality in ['good','ok','junk'] (w,h) = gm.gx2_img_size(gx) roi = [0,0,w,h] cm.add_chip(-1, nx, gx, roi) # HACKISH Duplicate detection. Eventually this should actually be in the codebase logmsg('Detecting and Removing Duplicate Ground Truth') dup_cx_list = [] for nx in nm.get_valid_nxs(): cx_list = array(nm.nx2_cx_list[nx]) gx_list = cm.cx2_gx[cx_list] (unique_gx, unique_x) = np.unique(gx_list, return_index=True) name = nm.nx2_name[nx] for gx in gx_list[unique_x]: bit = False gname = gm.gx2_gname[gx] x_list = pylab.find(gx_list == gx) cx_list2 = cx_list[x_list] roi_list2 = cm.cx2_roi[cx_list2] roi_hash = lambda roi: roi[0]+roi[1]*10000+roi[2]*100000000+roi[3]*1000000000000 (_, unique_x2) = np.unique(map(roi_hash, roi_list2), return_index=True) non_unique_x2 = np.setdiff1d(np.arange(0,len(cx_list2)), unique_x2) for nux2 in non_unique_x2: cx_ = cx_list2[nux2] dup_cx_list += [cx_] roi_ = roi_list2[nux2] logmsg('Duplicate: cx=%4d, gx=%4d, nx=%4d roi=%r' % (cx_, gx, nx, roi_) ) logmsg(' Name:%s, Image:%s' % (name, gname) ) bit = True if bit: logmsg('-----------------') for cx in dup_cx_list: cm.remove_chip(cx)
def delete_preferences(hs): 'Deletes the preference files in the ~/.hotspotter directory' logmsg('Deleting the ~/.hotspotter preference directory') hs.iom.remove_settings_files_with_pattern('*')
def delete_computed_directory(hs): 'Unloads all features and models and deletes the computed directory' logmsg('Deleting the computed directory') hs.iom.remove_computed_files_with_pattern('*') hs.unload_all_features()
def delete_precomputed_results(hs): logmsg('Deleting precomputed results') hs.iom.remove_computed_files_with_pattern('rr_*')
def change_orientation(cm, cx, new_theta): cid = cm.cx2_cid[cx] logmsg('Giving cid=%d new theta: %r' % (cid, new_theta)) assert not new_theta is None cm.hs.on_cx_modified(cx) cm.cx2_theta[cx] = new_theta
def run_matching_experiment(em, expt_name='MatchingExperiment', with_images=True): '''Quick experiment: Query each chip with a duplicate against whole database Do not remove anyone from ANN matching''' import os logmsg('Running Quick Experiment: ' + expt_name) hs = em.hs am, cm, vm, qm, iom, dm = hs.get_managers('am', 'cm', 'vm', 'qm', 'iom', 'dm') with_ellipses = True with_points = False with_full_results = False with_ellipse_and_points = False # Create an Experiment Directory in .hs_internals/computed timestamp = iom.get_timestamp() expt_dpath = iom.ensure_directory( iom.get_user_fpath('_'.join(['expt', expt_name, timestamp]))) expt_img_dpath = iom.ensure_directory( os.path.join(expt_dpath, 'result_images')) expt_rr_dpath = iom.ensure_directory( os.path.join(expt_dpath, 'raw_results')) # Write Algorithm Settings to the file algo_prefs_text = am.get_algo_name('all') algo_prefs_fpath = os.path.join(expt_dpath, 'algo_prefs.txt') iom.write(algo_prefs_fpath, algo_prefs_text) vm.build_model() # Defaults to building model of all prev_ell = dm.draw_prefs.ellipse_bit prev_pts = dm.draw_prefs.points_bit dm.fignum = 1 # Create Matches File To Append to with open(os.path.join(expt_dpath, 'matches.txt'), 'a') as file: for cx in iter(cm.get_valid_cxs()): # Preform Query res = QueryResult(qm.hs, qm.cx2_rr(cx)) # Get Query Info qcid, gname = cm.cx2_(res.rr.qcx, 'cid', 'gname') # Get Result Info (tcid, tgname, tscore) = res.tcid2_('cid', 'gname', 'score') # Print Query Info logmsg('---QUERY---') outstr = 'QUERY, gname=%s, cid=%4d' % (gname, qcid) print outstr file.write(outstr + '\n') # Print Result Info if len(tscore) == 0: maxsim = 0 # Best Score if len(tscore) > 0: maxsim = tscore[0] # Best Score for (rank, tup) in enumerate( zip(*[x.tolist() for x in (tgname, tcid, tscore)])): outstr = ' rank=%d, gname=%s, cid=%4d, score=%7.2f' % tuple( [rank + 1] + list(tup)) print outstr file.write(outstr + '\n') print '' file.write('\n\n') # Output Images query_name = 'sim=%07.2f-qcid=%d.png' % (maxsim, qcid) if with_full_results: #import shelve #shelve.open(os.path.join(expt_rr_dpath, 'rr_'+query_name+'.shelf') #import cPickle #rr = res.rr #cPickle.dump(, rr) pass if with_images: dm.draw_prefs.ellipse_bit = False dm.draw_prefs.points_bit = False dm.draw_prefs.bbox_bit = False dm.show_query(res) dm.save_fig( os.path.join(expt_img_dpath, 'img_' + query_name)) if with_ellipses: dm.draw_prefs.ellipse_bit = True dm.draw_prefs.points_bit = False dm.show_query(res) dm.save_fig( os.path.join(expt_img_dpath, 'ellipse_' + query_name)) if with_points: dm.draw_prefs.ellipse_bit = False dm.draw_prefs.points_bit = True dm.show_query(res) dm.save_fig( os.path.join(expt_img_dpath, 'point_' + query_name)) if with_ellipse_and_points: dm.draw_prefs.ellipse_bit = True dm.draw_prefs.points_bit = True dm.show_query(res) dm.save_fig( os.path.join(expt_img_dpath, 'both_' + query_name)) logmsg('Finished Matching Experiment: ' + expt_name) timestamp = iom.get_timestamp() iom.write(os.path.join(expt_dpath, 'Finished_' + str(timestamp)), timestamp + '\n' + em.hs.get_database_stat_str()) prev_ell = dm.draw_prefs.ellipse_bit prev_pts = dm.draw_prefs.points_bit
def _load_table(iom, csv_fpath, table_name, alloc_func, csv_func): """ Reads csv files. Must pass in a table name a memory allocation function and a csv_func: function which parses the fields read by _load_table """ logio("Loading " + table_name + " Table: " + csv_fpath) if not exists(csv_fpath): logio('"' + csv_fpath + '" Does Not Exist') return False fid = file(csv_fpath, "r") csv_headers = None line = fid.readline() num_line_prefix = "# NumData" # Foreach line in the CSV file while line != "": line = line.strip() # NEW LINE: Skip if line == "": continue # COMMENT LINE: Check for metadata elif line[0] == "#": # CHECK Preallocation if line.find(num_line_prefix) > -1: # Parse out the number of lines to allocate # and use the given allocation function num_lines = int(line.replace(num_line_prefix, "").replace(" ", "")) alloc_func(num_lines) # CHECK Data Headers: StripeSpotter elif line.find("#imgindex") > -1: logmsg("Loading a Legacy StripeSpotter File") csv_headers = line[1:].split(",") # CHECK Data Headers: Legacy HotSpotter elif line.find("#01)") > -1: logmsg("Loading a Legacy HotSpotter File") csv_headers = [] while line != "": line = line[:-1] if len(line) < 4 or line[3] != ")": break parnstr = "#\\d\\d\\) " head_field = re.sub(parnstr, "", line) head_field = re.sub(" - .*", "", head_field) csv_headers += [head_field] line = fid.readline() # CHECK Data Headers: Hotspotter elif any([line.find(field) >= 0 for field in ["ChipID", "NameID", "ImageID"]]): csv_headers = [field.strip() for field in line[1:].split(",")] # HACK: Change the fields to the ones it actually expects import hotspotter.other.AbstractPrintable _lbl2_header = hotspotter.other.AbstractPrintable._lbl2_header _header2_lbl = {v: k for k, v in _lbl2_header.iteritems()} csv_headers = [ _header2_lbl[field] if field in _header2_lbl.keys() else field for field in csv_headers ] # DATA LINE: Read it else: csv_data = [data_field.strip() for data_field in line.split(",")] csv_func(csv_data, csv_headers) # Next Line line = fid.readline() # Finsh reading table fid.close() logio("Loaded " + table_name + " Table") return True
def save_tables(iom): hs = iom.hs gm = hs.gm cm = hs.cm nm = hs.nm logmsg("Saving the Database. Give it a sec.") chip_table_fpath = iom.get_chip_table_fpath() name_table_fpath = iom.get_name_table_fpath() img_table_fpath = iom.get_image_table_fpath() flat_table_fpath = iom.get_flat_table_fpath() logmsg("Saving Image Table") img_file = open(img_table_fpath, "w") img_file.write(gm.gx2_info(lbls=["gid", "gname", "aif"])) img_file.close() logmsg("Saving Name Table") name_file = open(name_table_fpath, "w") name_file.write(nm.nx2_info(lbls=["nid", "name"])) name_file.close() logmsg("Saving Chip Table") chip_file = open(chip_table_fpath, "w") chip_file.write(cm.cx2_info(lbls="all")) chip_file.close() logmsg("Saving Flat Table") flat_file = open(flat_table_fpath, "w") flat_lbls = ["cid", "gname", "name", "roi", "theta"] + cm.user_props.keys() flat_file.write(cm.cx2_info(lbls=flat_lbls)) flat_file.close() logmsg("The Database was Saved")
def load_oxford_gt(iom): "loads oxford style groundtruth" gm, cm, nm = iom.hs.get_managers("gm", "cm", "nm") # Check for corrupted files (Looking at your Paris Buildings Dataset) oxford_gt_dpath = iom.get_oxford_gt_dpath() corrupted_gname_list = [] corrupted_file_fname = "corrupted_files.txt" corrupted_file_fpath = join(oxford_gt_dpath, corrupted_file_fname) if exists(corrupted_file_fpath): with open(corrupted_file_fpath) as f: corrupted_gname_list = f.read().splitlines() logmsg("Loading Oxford Style Images") # Recursively get relative path of all files in img_dpath img_dpath = iom.get_img_dpath() # with a sexy list comprehension gname_list = [ join(relpath(root, img_dpath), fname).replace("\\", "/").replace("./", "") for (root, dlist, flist) in os.walk(img_dpath) for fname in flist ] # Roughly Prealloc gm.img_alloc(len(gname_list)) cm.chip_alloc(len(gname_list)) # Add all images in images directory (allow nested directories (...paris)) for gname in gname_list: if gname in corrupted_gname_list: continue gm.add_img(-1, gname, True) logmsg("Loading Oxford Style Names and Chips") # Add names and chips from ground truth gt_fname_list = os.listdir(oxford_gt_dpath) iom.hs.nm.name_alloc(len(gt_fname_list) / 4) for gt_fname in gt_fname_list: if gt_fname == corrupted_file_fname: continue # Get gt_name, quality, and num from fname gt_name = gt_fname.replace(".txt", "") _pos1 = gt_name.rfind("_") quality = gt_name[_pos1 + 1 :] gt_name = gt_name[:_pos1] _pos2 = gt_name.rfind("_") num = gt_name[_pos2 + 1 :] gt_name = gt_name[:_pos2] # Add Name (-2 suppresses warnings) nid = nm.add_name(-2, gt_name) nx = nm.nid2_nx[nid] gt_fpath = join(oxford_gt_dpath, gt_fname) with open(gt_fpath, "r") as f: line_list = f.read().splitlines() for line in line_list: if line == "": continue fields = line.split(" ") gname = fields[0].replace("oxc1_", "") + ".jpg" if gname.find("paris_") >= 0: # PARIS HACK >:( # Because they just cant keep their paths consistent paris_hack = gname[6 : gname.rfind("_")] gname = paris_hack + "/" + gname if gname in corrupted_gname_list: continue gid = gm.gname2_gid[gname] gx = gm.gid2_gx[gid] if len(fields) > 1: # quality == query roi = map(lambda x: int(round(float(x))), fields[1:]) else: # quality in ['good','ok','junk'] (w, h) = gm.gx2_img_size(gx) roi = [0, 0, w, h] cm.add_chip(-1, nx, gx, roi) # HACKISH Duplicate detection. Eventually this should actually be in the codebase logmsg("Detecting and Removing Duplicate Ground Truth") dup_cx_list = [] for nx in nm.get_valid_nxs(): cx_list = array(nm.nx2_cx_list[nx]) gx_list = cm.cx2_gx[cx_list] (unique_gx, unique_x) = np.unique(gx_list, return_index=True) name = nm.nx2_name[nx] for gx in gx_list[unique_x]: bit = False gname = gm.gx2_gname[gx] x_list = pylab.find(gx_list == gx) cx_list2 = cx_list[x_list] roi_list2 = cm.cx2_roi[cx_list2] roi_hash = lambda roi: roi[0] + roi[1] * 10000 + roi[2] * 100000000 + roi[3] * 1000000000000 (_, unique_x2) = np.unique(map(roi_hash, roi_list2), return_index=True) non_unique_x2 = np.setdiff1d(np.arange(0, len(cx_list2)), unique_x2) for nux2 in non_unique_x2: cx_ = cx_list2[nux2] dup_cx_list += [cx_] roi_ = roi_list2[nux2] logmsg("Duplicate: cx=%4d, gx=%4d, nx=%4d roi=%r" % (cx_, gx, nx, roi_)) logmsg(" Name:%s, Image:%s" % (name, gname)) bit = True if bit: logmsg("-----------------") for cx in dup_cx_list: cm.remove_chip(cx)
def delete_computed_cid(cm, cid): iom = cm.hs.iom if np.iterable(cid): logerr('this function only works for a single cid') logmsg('Removing CID=%d\'s computed files' % cid) cid_fname_pattern = iom.get_chip_prefix(cid, [])+'*' iom.remove_computed_files_with_pattern(cid_fname_pattern)
def print(*args): if len(args) == 1: logmsg(str(args[0])) else: logmsg(' '.join(args))