def _apply_global_rank_adjustment(base_result, indexed_dirs, query):
all_open_filenames = []
if query.current_filename:
all_open_filenames.append(query.current_filename)
all_open_filenames.extend(query.open_filenames)
# The active_dir_orders goes from a directory prefix to an order
# value. We use this to break ties when the same basename crops up in two
# places: the basename that is in the most recently active directory wins.
#
# This is built as a dict, but then converted to a flat list because we iterate it
# during execution.
inactive_dirs = set(indexed_dirs)
active_dir_orders = {}
for open_filename in all_open_filenames:
for d in indexed_dirs:
if open_filename.startswith(d):
if d not in active_dir_orders:
active_dir_orders[d] = len(active_dir_orders)
inactive_dirs.remove(d)
inactive_dirs = list(inactive_dirs)
inactive_dirs.sort()
for i in inactive_dirs:
active_dir_orders[i] = len(active_dir_orders)
active_dir_orders = [(x,y) for x,y in active_dir_orders.items()]
def get_order(f):
for d,order in active_dir_orders:
if f.startswith(d):
return order
return sys.maxint
def hit_cmp(x,y):
# directory order trumps everything
h = get_order(x[0]) - get_order(y[0])
if h != 0:
return h
# compare on the rank
j = -cmp(x[1],y[1])
if j != 0:
return j
# if the ranks agree, compare on the filename,
# first by basename, then by fullname
x_base = os.path.basename(x[0])
y_base = os.path.basename(y[0])
j = cmp(x_base, y_base)
if j != 0:
return j
# last resort is to compare full names
return cmp(x[0], y[0])
hits = list(base_result.hits)
hits.sort(hit_cmp)
new_hits = _rerank(hits)
res = QueryResult(new_hits, base_result.truncated)
res.debug_info = copy.deepcopy(base_result.debug_info)
return res
def _filter_for_base_path(base_result, query):
res = QueryResult()
res.debug_info = copy.deepcopy(base_result.debug_info)
res.truncated = base_result.truncated
for hit,rank in base_result.hits:
if _is_in_base_path(hit, query.base_path):
res.filenames.append(hit)
res.ranks.append(rank)
return res
def _filter_result_for_exact_matches(query_text, base_result):
"""
Returns a new QueryResult object containing only filenames that exactly
match the provided query.
"""
res = QueryResult()
res.debug_info = copy.deepcopy(base_result.debug_info)
res.truncated = base_result.truncated
for hit,rank in base_result.hits:
if _is_exact_match(query_text, hit):
res.filenames.append(hit)
res.ranks.append(rank)
return res
def execute_statement(self, p_sql_statement: str):
"""
Der Auftrag schickt den im Parameter p_sql_statement enthaltenen SQL-Befehl an die
Datenbank ab.
Handelt es sich bei p_sql_statement um einen SQL-Befehl, der eine Ergebnismenge
liefert, so kann dieses Ergebnis anschließend mit der Methode get_current_query_result
abgerufen werden.
"""
self.__current_query_result = None
self.__message = None
if self.__connection is None:
self.__message = "No connection"
return
try:
cursor = self.__connection.cursor()
if cursor.execute(p_sql_statement):
result_column_names = []
result_colum_types = []
result_data = cursor.fetchall()
for column in cursor.description:
result_column_names.append(column[0])
result_colum_types.append(column[1])
self.__current_query_result = QueryResult(
result_data, result_column_names, result_colum_types)
cursor.close()
except Exception as e:
self.__message = str(e)
def execute(self, shard_manager, query_cache):
"""
Searches the index given the provided query.
args should be either a Query object, or arguments to the Query-object constructor.
"""
if self.text == '':
return QueryResult()
assert self.max_hits >= 0
res = query_cache.try_get(self)
if not res:
res_was_cache_hit = False
ranked_results = self.execute_nocache(shard_manager, query_cache)
ranked_and_truncated_results = ranked_results.get_copy_with_max_hits(self.max_hits)
res = ranked_and_truncated_results
query_cache.put(self, res)
else:
res_was_cache_hit = True
if self.exact_match:
final_res = _filter_result_for_exact_matches(self.text, res)
else:
final_res = _apply_global_rank_adjustment(res, shard_manager.dirs, self)
if self.debug:
final_res.debug_info.append({"res_was_cache_hit": res_was_cache_hit})
final_res.debug_info.append({"initial_res": res.as_dict(),
"dirs": shard_manager.dirs
})
return final_res
def test_adjustment_creates_decreasing_hit_order(self):
initial_result = QueryResult.from_dict({'hits': [('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/js/script-tests/char-at.js', 12.0), ('/Users/nduca/home/quickopen/test_data/cr_files_basenames.json', 10.800000000000001), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options2/instant_confirm_overlay.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options/instant_confirm_overlay.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dynamic-updates/script-tests/SVGCircleElement-dom-requiredFeatures.js', 8.5999999999999996), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p9pass.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level3/core/domconfigurationcansetparameter03.js', 7.0), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options2/chromeos/cellular_plan_element.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/options/chromeos/cellular_plan_element.js', 9.3000000000000007), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/svg/level3/xpath/Conformance_Expressions.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/Source/WebCore/inspector/front-end/ResourceResponseView.js', 9.3000000000000007), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/cr/ui/focus_outline_manager.js', 7.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr/ui/focus_outline_manager.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p2fail.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/Source/WebCore/inspector/front-end/InspectorView.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_characterdatadeletedatamiddle.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_characterdatadeletedatamiddle.js', 10.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/http/tests/security/xssAuditor/resources/base-href/really-safe-script.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_attrreplacechild1.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_attrreplacechild1.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/dom/Geolocation/script-tests/timeout-clear-watch.js', 7.0), ('/Users/nduca/Local/chrome/src/chrome/test/data/extensions/api_test/history/search_after_add.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p5fail.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dynamic-updates/script-tests/SVGRectElement-dom-y-attr.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level3/core/documentrenamenode03.js', 7.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_textsplittextfour.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_textsplittextfour.js', 7.5), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/dom/Orientation/script-tests/create-event-orientationchange.js', 13.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/connection_manager.js', 10.5), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/chrome/common/extensions/docs/examples/extensions/plugin_settings/domui/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr.js', 14.0), ('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/js/mozilla/eval/script-tests/exhaustive-global-strictcaller-indirect-strictcode.js', 7.0), ('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/event_tracker.js', 7.5), ('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/event_tracker.js', 7.5)], 'debug_info': [], 'truncated': True})
dirs = ['/Users/nduca/Local/ndbg', '/Users/nduca/Local/quickopen', '/Users/nduca/home', '/Users/nduca/Local/chrome']
q = Query("cr.js")
res = query._apply_global_rank_adjustment(initial_result, dirs, q)
for i in range(1, len(res.ranks)):
self.assertTrue(res.ranks[i-1] > res.ranks[i])
self.assertEquals('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr.js', res.filenames[0])
def test_global_rank_adjustment_puts_directories_into_predictable_order(
self):
# and if d if the ranks mismatch, dont reorder
in_res = QueryResult(hits=[("/b/render_widget.cpp",
10), ("/a/render_widget.cpp", 10)])
res = query._apply_global_rank_adjustment(in_res, ["/"], Query("rw"))
self.assertEquals(["/a/render_widget.cpp", "/b/render_widget.cpp"],
res.filenames)
def search(self, *args, **kwargs):
"""
Searches the database.
args should be either a Query object, or arguments to the Query-object constructor.
"""
query = Query.from_kargs(args, kwargs)
d = self._req('POST', '/search', query.as_dict())
return QueryResult.from_dict(d)
def test_adjustment_puts_current_project_above_noncurrent(self):
in_res = QueryResult(hits=[("/b/render_widget.cpp",
10), ("/a/render_widget.cpp", 10)])
res = query._apply_global_rank_adjustment(
in_res, ["/a/", "/b/"],
Query("rw", current_filename="/b/k/foobar.cpp"))
rA = res.rank_of("/a/render_widget.cpp")
rB = res.rank_of("/b/render_widget.cpp")
self.assertTrue(rB > rA, "Expected %s > %s" % (rB, rA))
def search(self, *args, **kwargs):
"""
Searches the database.
args/kwargs should be either a Query object, or arguments to the Query-object constructor.
"""
if self._pending_indexer:
self.step_indexer()
# step sync might change the db sync status
if not self._cur_shard_manager:
return QueryResult()
query = Query.from_kargs(args, kwargs)
return query.execute(self._cur_shard_manager, self._cur_query_cache)
def test_global_rank_adjustment_puts_suffixes_into_predictable_order(self):
# render_widget.cpp should be get re-ranked higher than render_widget.h
in_res = QueryResult(hits=[("/render_widget.h",
10), ("/render_widget.cpp", 10)])
res = query._apply_global_rank_adjustment(in_res, ["/"], Query("rw"))
self.assertEquals([
"/render_widget.cpp",
"/render_widget.h",
], res.filenames)
# render_widget.cpp should stay ranked higher than render_widget.h
in_res = QueryResult(hits=[("/render_widget.cpp",
10), ("/render_widget.h", 10)])
res = query._apply_global_rank_adjustment(in_res, ["/"], Query("rw"))
self.assertEquals(["/render_widget.cpp", "/render_widget.h"],
res.filenames)
# but if the ranks mismatch, dont reorder
in_res = QueryResult(hits=[("/render_widget.cpp",
10), ("/render_widget.h", 12)])
res = query._apply_global_rank_adjustment(in_res, ["/"], Query("rw"))
self.assertEquals(["/render_widget.h", "/render_widget.cpp"],
res.filenames)
def result(self):
if not self.async_conn:
raise AsyncSearchError, 'connection died during search'
if not self._result:
res = self.async_conn.get_response()
if res.status != 200:
self.async_conn.close()
self.async_conn = None
raise AsyncSearchError, 'got status %i' % res.status
else:
data = res.read()
res = json.loads(data.encode('utf8'))
self._result = QueryResult.from_dict(res)
return self._result
def test_adjustment_puts_current_project_then_inactives_in_alphabetical(
self):
in_res = QueryResult(hits=[("/c/render_widget.cpp",
10), ("/b/render_widget.cpp",
10), ("/a/render_widget.cpp", 10)])
# A is current, so it should go top, then b c
res = query._apply_global_rank_adjustment(
in_res, ["/a/", "/b/", "/c/"],
Query("rw", current_filename="/a/foobar.cpp"))
rA = res.rank_of("/a/render_widget.cpp")
rB = res.rank_of("/b/render_widget.cpp")
rC = res.rank_of("/c/render_widget.cpp")
self.assertTrue(rA > rB, "Expected %s > %s" % (rA, rB))
self.assertTrue(rB > rC, "Expected %s > %s" % (rB, rC))
self.assertTrue(rA > rC, "Expected %s > %s" % (rA, rC))
def _processing(self, text: str):
try:
text_input = dialogflow.types.TextInput(
text=text, language_code=LANG_CODE.get('IETF', 'en-US'))
query_input = dialogflow.types.QueryInput(text=text_input)
response = self._client.detect_intent(self._session, query_input)
except Exception as e:
self._err_count += 1
self.log('Processing error: {}'.format(e), logger.ERROR)
if self._err_count >= self.MAX_ERRORS:
self.log('Detected many errors [{}]. Bye!', logger.CRIT)
return
else:
self._err_count = 0
self.entrypoint(QueryResult(response.query_result))
def execute_nocache(self, shard_manager, query_cache):
# What we'll actually return
truncated = False
slashIdx = self.text.rfind('/')
if slashIdx != -1:
dirpart_query = self.text[:slashIdx]
basename_query = self.text[slashIdx + 1:]
else:
dirpart_query = ''
basename_query = self.text
lower_dirpart_query = dirpart_query.lower()
# Get the files
files = []
if len(basename_query):
basename_hits, truncated = shard_manager.search_basenames(
basename_query)
for hit in basename_hits:
hit_files = shard_manager.files_by_lower_basename[hit]
for f in hit_files:
if _is_dirmatch(lower_dirpart_query, f):
files.append(f)
else:
i = 0
start = time.time()
timeout = start + self._dir_search_timeout
for f in shard_manager.files:
if _is_dirmatch(lower_dirpart_query, f):
files.append(f)
i += 1
if i % 1000 == 0:
if time.time() >= timeout:
truncated = True
break
# Rank the results
trace_begin("rank_results")
hits = []
basename_ranker = BasenameRanker()
for f in files:
basename = os.path.basename(f)
rank = basename_ranker.rank_query(basename_query, basename)
hits.append((f, rank))
trace_end("rank_results")
return QueryResult(hits=hits, truncated=truncated)
def test_adjustment_puts_different_files_in_different_base_directories_together(
self):
in_res = QueryResult(hits=[("/a/render_widget.cpp",
10), ("/b/render_widget.cpp",
10), ("/a/render_view.cpp",
9), ("/b/render_view.cpp", 9)])
# Cwd is not an active dir.
res = query._apply_global_rank_adjustment(
in_res, ["/a", "/b"], Query("render", current_filename="/x"))
self.assertEquals([
"/a/render_widget.cpp", "/a/render_view.cpp",
"/b/render_widget.cpp", "/b/render_view.cpp"
], res.filenames)
# b is cwd.
res = query._apply_global_rank_adjustment(
in_res, ["/a", "/b"], Query("render", current_filename="/b"))
self.assertEquals([
"/b/render_widget.cpp", "/b/render_view.cpp",
"/a/render_widget.cpp", "/a/render_view.cpp"
], res.filenames)
def make_result(hits):
res = QueryResult()
for h in hits:
res.filenames.append(h)
res.ranks.append(10)
return res
def test_adjustment_creates_decreasing_hit_order(self):
initial_result = QueryResult.from_dict({
'hits':
[('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/js/script-tests/char-at.js',
12.0),
('/Users/nduca/home/quickopen/test_data/cr_files_basenames.json',
10.800000000000001),
('/Users/nduca/Local/chrome/src/chrome/browser/resources/options2/instant_confirm_overlay.js',
9.3000000000000007),
('/Users/nduca/Local/chrome/src/chrome/browser/resources/options/instant_confirm_overlay.js',
9.3000000000000007),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dynamic-updates/script-tests/SVGCircleElement-dom-requiredFeatures.js',
8.5999999999999996),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p9pass.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level3/core/domconfigurationcansetparameter03.js',
7.0),
('/Users/nduca/Local/chrome/src/chrome/browser/resources/options2/chromeos/cellular_plan_element.js',
9.3000000000000007),
('/Users/nduca/Local/chrome/src/chrome/browser/resources/options/chromeos/cellular_plan_element.js',
9.3000000000000007),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/svg/level3/xpath/Conformance_Expressions.js',
10.0),
('/Users/nduca/Local/chrome/src/third_party/WebKit/Source/WebCore/inspector/front-end/ResourceResponseView.js',
9.3000000000000007),
('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/cr/ui/focus_outline_manager.js',
7.5),
('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr/ui/focus_outline_manager.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p2fail.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/Source/WebCore/inspector/front-end/InspectorView.js',
7.0),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_characterdatadeletedatamiddle.js',
10.0),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_characterdatadeletedatamiddle.js',
10.0),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/http/tests/security/xssAuditor/resources/base-href/really-safe-script.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_attrreplacechild1.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_attrreplacechild1.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/dom/Geolocation/script-tests/timeout-clear-watch.js',
7.0),
('/Users/nduca/Local/chrome/src/chrome/test/data/extensions/api_test/history/search_after_add.js',
7.0),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dom/SVGScriptElement/resources/script-set-href-p5fail.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/svg/dynamic-updates/script-tests/SVGRectElement-dom-y-attr.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level3/core/documentrenamenode03.js',
7.0),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/xhtml/level1/core/hc_textsplittextfour.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/dom/html/level1/core/hc_textsplittextfour.js',
7.5),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/dom/Orientation/script-tests/create-event-orientationchange.js',
13.5),
('/Users/nduca/Local/chrome/src/chrome/browser/resources/connection_manager.js',
10.5),
('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/cr.js',
14.0),
('/Users/nduca/Local/chrome/src/chrome/common/extensions/docs/examples/extensions/plugin_settings/domui/js/cr.js',
14.0),
('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr.js',
14.0),
('/Users/nduca/Local/chrome/src/third_party/WebKit/LayoutTests/fast/js/mozilla/eval/script-tests/exhaustive-global-strictcaller-indirect-strictcode.js',
7.0),
('/Users/nduca/home/trace_event_viewer/third_party/chrome/shared/js/event_tracker.js',
7.5),
('/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/event_tracker.js',
7.5)],
'debug_info': [],
'truncated':
True
})
dirs = [
'/Users/nduca/Local/ndbg', '/Users/nduca/Local/quickopen',
'/Users/nduca/home', '/Users/nduca/Local/chrome'
]
q = Query("cr.js")
res = query._apply_global_rank_adjustment(initial_result, dirs, q)
for i in range(1, len(res.ranks)):
self.assertTrue(res.ranks[i - 1] > res.ranks[i])
self.assertEquals(
'/Users/nduca/Local/chrome/src/chrome/browser/resources/shared/js/cr.js',
res.filenames[0])
