def _apply_global_rank_adjustment(base_result, indexed_dirs, query): all_open_filenames = [] if query.current_filename: all_open_filenames.append(query.current_filename) all_open_filenames.extend(query.open_filenames) # The active_dir_orders goes from a directory prefix to an order # value. We use this to break ties when the same basename crops up in two # places: the basename that is in the most recently active directory wins. # # This is built as a dict, but then converted to a flat list because we iterate it # during execution. inactive_dirs = set(indexed_dirs) active_dir_orders = {} for open_filename in all_open_filenames: for d in indexed_dirs: if open_filename.startswith(d): if d not in active_dir_orders: active_dir_orders[d] = len(active_dir_orders) inactive_dirs.remove(d) inactive_dirs = list(inactive_dirs) inactive_dirs.sort() for i in inactive_dirs: active_dir_orders[i] = len(active_dir_orders) active_dir_orders = [(x,y) for x,y in active_dir_orders.items()] def get_order(f): for d,order in active_dir_orders: if f.startswith(d): return order return sys.maxint def hit_cmp(x,y): # directory order trumps everything h = get_order(x[0]) - get_order(y[0]) if h != 0: return h # compare on the rank j = -cmp(x[1],y[1]) if j != 0: return j # if the ranks agree, compare on the filename, # first by basename, then by fullname x_base = os.path.basename(x[0]) y_base = os.path.basename(y[0]) j = cmp(x_base, y_base) if j != 0: return j # last resort is to compare full names return cmp(x[0], y[0]) hits = list(base_result.hits) hits.sort(hit_cmp) new_hits = _rerank(hits) res = QueryResult(new_hits, base_result.truncated) res.debug_info = copy.deepcopy(base_result.debug_info) return res
def _filter_for_base_path(base_result, query): res = QueryResult() res.debug_info = copy.deepcopy(base_result.debug_info) res.truncated = base_result.truncated for hit,rank in base_result.hits: if _is_in_base_path(hit, query.base_path): res.filenames.append(hit) res.ranks.append(rank) return res
def _filter_result_for_exact_matches(query_text, base_result): """ Returns a new QueryResult object containing only filenames that exactly match the provided query. """ res = QueryResult() res.debug_info = copy.deepcopy(base_result.debug_info) res.truncated = base_result.truncated for hit,rank in base_result.hits: if _is_exact_match(query_text, hit): res.filenames.append(hit) res.ranks.append(rank) return res
def execute_statement(self, p_sql_statement: str): """ Der Auftrag schickt den im Parameter p_sql_statement enthaltenen SQL-Befehl an die Datenbank ab. Handelt es sich bei p_sql_statement um einen SQL-Befehl, der eine Ergebnismenge liefert, so kann dieses Ergebnis anschließend mit der Methode get_current_query_result abgerufen werden. """ self.__current_query_result = None self.__message = None if self.__connection is None: self.__message = "No connection" return try: cursor = self.__connection.cursor() if cursor.execute(p_sql_statement): result_column_names = [] result_colum_types = [] result_data = cursor.fetchall() for column in cursor.description: result_column_names.append(column[0]) result_colum_types.append(column[1]) self.__current_query_result = QueryResult( result_data, result_column_names, result_colum_types) cursor.close() except Exception as e: self.__message = str(e)
def execute(self, shard_manager, query_cache): """ Searches the index given the provided query. args should be either a Query object, or arguments to the Query-object constructor. """ if self.text == '': return QueryResult() assert self.max_hits >= 0 res = query_cache.try_get(self) if not res: res_was_cache_hit = False ranked_results = self.execute_nocache(shard_manager, query_cache) ranked_and_truncated_results = ranked_results.get_copy_with_max_hits(self.max_hits) res = ranked_and_truncated_results query_cache.put(self, res) else: res_was_cache_hit = True if self.exact_match: final_res = _filter_result_for_exact_matches(self.text, res) else: final_res = _apply_global_rank_adjustment(res, shard_manager.dirs, self) if self.debug: final_res.debug_info.append({"res_was_cache_hit": res_was_cache_hit}) final_res.debug_info.append({"initial_res": res.as_dict(), "dirs": shard_manager.dirs }) return final_res
def test_adjustment_creates_decreasing_hit_order(self): initial_result = QueryResult.from_dict({'hits': [('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/js/script-tests/char-at.js', 12.0), ('/Users/nduca/home/quickopen/test_data/cr_files_basenames.json', 10.800000000000001), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options2/instant_confirm_overlay.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options/instant_confirm_overlay.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dynamic-updates/script-tests/SVGCircleElement-dom-requiredFeatures.js', 8.5999999999999996), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p9pass.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level3/core/domconfigurationcansetparameter03.js', 7.0), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options2/chromeos/cellular_plan_element.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options/chromeos/cellular_plan_element.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/svg/level3/xpath/Conformance_Expressions.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/Source/WebCore/inspector/front-end/ResourceResponseView.js', 9.3000000000000007), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/cr/ui/focus_outline_manager.js', 7.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr/ui/focus_outline_manager.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p2fail.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/Source/WebCore/inspector/front-end/InspectorView.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_characterdatadeletedatamiddle.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_characterdatadeletedatamiddle.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/http/tests/security/xssAuditor/resources/base-href/really-safe-script.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_attrreplacechild1.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_attrreplacechild1.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/dom/Geolocation/script-tests/timeout-clear-watch.js', 7.0), ('/Users/nduca/Local/chrome/src/chrome/test/data/extensions/api_test/history/search_after_add.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p5fail.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dynamic-updates/script-tests/SVGRectElement-dom-y-attr.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level3/core/documentrenamenode03.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_textsplittextfour.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_textsplittextfour.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/dom/Orientation/script-tests/create-event-orientationchange.js', 13.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/connection_manager.js', 10.5), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/chrome/common/extensions/docs/examples/extensions/plugin_settings/domui/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/js/mozilla/eval/script-tests/exhaustive-global-strictcaller-indirect-strictcode.js', 7.0), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/event_tracker.js', 7.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/event_tracker.js', 7.5)], 'debug_info': [], 'truncated': True}) dirs = ['/Users/nduca/Local/ndbg', '/Users/nduca/Local/quickopen', '/Users/nduca/home', '/Users/nduca/Local/chrome'] q = Query("cr.js") res = query._apply_global_rank_adjustment(initial_result, dirs, q) for i in range(1, len(res.ranks)): self.assertTrue(res.ranks[i-1] > res.ranks[i]) self.assertEquals('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr.js', res.filenames[0])
def test_global_rank_adjustment_puts_directories_into_predictable_order( self): # and if d if the ranks mismatch, dont reorder in_res = QueryResult(hits=[("/b/render_widget.cpp", 10), ("/a/render_widget.cpp", 10)]) res = query._apply_global_rank_adjustment(in_res, ["/"], Query("rw")) self.assertEquals(["/a/render_widget.cpp", "/b/render_widget.cpp"], res.filenames)
def search(self, *args, **kwargs): """ Searches the database. args should be either a Query object, or arguments to the Query-object constructor. """ query = Query.from_kargs(args, kwargs) d = self._req('POST', '/search', query.as_dict()) return QueryResult.from_dict(d)
def test_adjustment_puts_current_project_above_noncurrent(self): in_res = QueryResult(hits=[("/b/render_widget.cpp", 10), ("/a/render_widget.cpp", 10)]) res = query._apply_global_rank_adjustment( in_res, ["/a/", "/b/"], Query("rw", current_filename="/b/k/foobar.cpp")) rA = res.rank_of("/a/render_widget.cpp") rB = res.rank_of("/b/render_widget.cpp") self.assertTrue(rB > rA, "Expected %s > %s" % (rB, rA))
def search(self, *args, **kwargs): """ Searches the database. args/kwargs should be either a Query object, or arguments to the Query-object constructor. """ if self._pending_indexer: self.step_indexer() # step sync might change the db sync status if not self._cur_shard_manager: return QueryResult() query = Query.from_kargs(args, kwargs) return query.execute(self._cur_shard_manager, self._cur_query_cache)
def test_global_rank_adjustment_puts_suffixes_into_predictable_order(self): # render_widget.cpp should be get re-ranked higher than render_widget.h in_res = QueryResult(hits=[("/render_widget.h", 10), ("/render_widget.cpp", 10)]) res = query._apply_global_rank_adjustment(in_res, ["/"], Query("rw")) self.assertEquals([ "/render_widget.cpp", "/render_widget.h", ], res.filenames) # render_widget.cpp should stay ranked higher than render_widget.h in_res = QueryResult(hits=[("/render_widget.cpp", 10), ("/render_widget.h", 10)]) res = query._apply_global_rank_adjustment(in_res, ["/"], Query("rw")) self.assertEquals(["/render_widget.cpp", "/render_widget.h"], res.filenames) # but if the ranks mismatch, dont reorder in_res = QueryResult(hits=[("/render_widget.cpp", 10), ("/render_widget.h", 12)]) res = query._apply_global_rank_adjustment(in_res, ["/"], Query("rw")) self.assertEquals(["/render_widget.h", "/render_widget.cpp"], res.filenames)
def result(self): if not self.async_conn: raise AsyncSearchError, 'connection died during search' if not self._result: res = self.async_conn.get_response() if res.status != 200: self.async_conn.close() self.async_conn = None raise AsyncSearchError, 'got status %i' % res.status else: data = res.read() res = json.loads(data.encode('utf8')) self._result = QueryResult.from_dict(res) return self._result
def test_adjustment_puts_current_project_then_inactives_in_alphabetical( self): in_res = QueryResult(hits=[("/c/render_widget.cpp", 10), ("/b/render_widget.cpp", 10), ("/a/render_widget.cpp", 10)]) # A is current, so it should go top, then b c res = query._apply_global_rank_adjustment( in_res, ["/a/", "/b/", "/c/"], Query("rw", current_filename="/a/foobar.cpp")) rA = res.rank_of("/a/render_widget.cpp") rB = res.rank_of("/b/render_widget.cpp") rC = res.rank_of("/c/render_widget.cpp") self.assertTrue(rA > rB, "Expected %s > %s" % (rA, rB)) self.assertTrue(rB > rC, "Expected %s > %s" % (rB, rC)) self.assertTrue(rA > rC, "Expected %s > %s" % (rA, rC))
def _processing(self, text: str): try: text_input = dialogflow.types.TextInput( text=text, language_code=LANG_CODE.get('IETF', 'en-US')) query_input = dialogflow.types.QueryInput(text=text_input) response = self._client.detect_intent(self._session, query_input) except Exception as e: self._err_count += 1 self.log('Processing error: {}'.format(e), logger.ERROR) if self._err_count >= self.MAX_ERRORS: self.log('Detected many errors [{}]. Bye!', logger.CRIT) return else: self._err_count = 0 self.entrypoint(QueryResult(response.query_result))
def execute_nocache(self, shard_manager, query_cache): # What we'll actually return truncated = False slashIdx = self.text.rfind('/') if slashIdx != -1: dirpart_query = self.text[:slashIdx] basename_query = self.text[slashIdx + 1:] else: dirpart_query = '' basename_query = self.text lower_dirpart_query = dirpart_query.lower() # Get the files files = [] if len(basename_query): basename_hits, truncated = shard_manager.search_basenames( basename_query) for hit in basename_hits: hit_files = shard_manager.files_by_lower_basename[hit] for f in hit_files: if _is_dirmatch(lower_dirpart_query, f): files.append(f) else: i = 0 start = time.time() timeout = start + self._dir_search_timeout for f in shard_manager.files: if _is_dirmatch(lower_dirpart_query, f): files.append(f) i += 1 if i % 1000 == 0: if time.time() >= timeout: truncated = True break # Rank the results trace_begin("rank_results") hits = [] basename_ranker = BasenameRanker() for f in files: basename = os.path.basename(f) rank = basename_ranker.rank_query(basename_query, basename) hits.append((f, rank)) trace_end("rank_results") return QueryResult(hits=hits, truncated=truncated)
def test_adjustment_puts_different_files_in_different_base_directories_together( self): in_res = QueryResult(hits=[("/a/render_widget.cpp", 10), ("/b/render_widget.cpp", 10), ("/a/render_view.cpp", 9), ("/b/render_view.cpp", 9)]) # Cwd is not an active dir. res = query._apply_global_rank_adjustment( in_res, ["/a", "/b"], Query("render", current_filename="/x")) self.assertEquals([ "/a/render_widget.cpp", "/a/render_view.cpp", "/b/render_widget.cpp", "/b/render_view.cpp" ], res.filenames) # b is cwd. res = query._apply_global_rank_adjustment( in_res, ["/a", "/b"], Query("render", current_filename="/b")) self.assertEquals([ "/b/render_widget.cpp", "/b/render_view.cpp", "/a/render_widget.cpp", "/a/render_view.cpp" ], res.filenames)
def make_result(hits): res = QueryResult() for h in hits: res.filenames.append(h) res.ranks.append(10) return res
def test_adjustment_creates_decreasing_hit_order(self): initial_result = QueryResult.from_dict({ 'hits': [('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/js/script-tests/char-at.js', 12.0), ('/Users/nduca/home/quickopen/test_data/cr_files_basenames.json', 10.800000000000001), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options2/instant_confirm_overlay.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options/instant_confirm_overlay.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dynamic-updates/script-tests/SVGCircleElement-dom-requiredFeatures.js', 8.5999999999999996), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p9pass.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level3/core/domconfigurationcansetparameter03.js', 7.0), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options2/chromeos/cellular_plan_element.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options/chromeos/cellular_plan_element.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/svg/level3/xpath/Conformance_Expressions.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/Source/WebCore/inspector/front-end/ResourceResponseView.js', 9.3000000000000007), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/cr/ui/focus_outline_manager.js', 7.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr/ui/focus_outline_manager.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p2fail.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/Source/WebCore/inspector/front-end/InspectorView.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_characterdatadeletedatamiddle.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_characterdatadeletedatamiddle.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/http/tests/security/xssAuditor/resources/base-href/really-safe-script.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_attrreplacechild1.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_attrreplacechild1.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/dom/Geolocation/script-tests/timeout-clear-watch.js', 7.0), ('/Users/nduca/Local/chrome/src/chrome/test/data/extensions/api_test/history/search_after_add.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p5fail.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dynamic-updates/script-tests/SVGRectElement-dom-y-attr.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level3/core/documentrenamenode03.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_textsplittextfour.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_textsplittextfour.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/dom/Orientation/script-tests/create-event-orientationchange.js', 13.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/connection_manager.js', 10.5), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/chrome/common/extensions/docs/examples/extensions/plugin_settings/domui/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/js/mozilla/eval/script-tests/exhaustive-global-strictcaller-indirect-strictcode.js', 7.0), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/event_tracker.js', 7.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/event_tracker.js', 7.5)], 'debug_info': [], 'truncated': True }) dirs = [ '/Users/nduca/Local/ndbg', '/Users/nduca/Local/quickopen', '/Users/nduca/home', '/Users/nduca/Local/chrome' ] q = Query("cr.js") res = query._apply_global_rank_adjustment(initial_result, dirs, q) for i in range(1, len(res.ranks)): self.assertTrue(res.ranks[i - 1] > res.ranks[i]) self.assertEquals( '/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr.js', res.filenames[0])