def set_from_matches(self, matches):
""" set the content of the liststore based on a list of
xapian.MSetItems
"""
LOG.debug("set_from_matches len(matches)='%s'" % len(matches))
self.current_matches = matches
n_matches = len(matches)
if n_matches == 0:
return
extent = min(self.LOAD_INITIAL, n_matches)
with ExecutionTime("store.append_initial"):
for doc in [m.document for m in matches][:extent]:
doc.available = doc.installed = doc.purchasable = None
self.append((doc,))
if n_matches == extent:
return
with ExecutionTime("store.append_placeholders"):
for i in range(n_matches - extent):
self.append()
self.emit('appcount-changed', len(matches))
self.buffer_icons()
def __init__(self, db, doc=None, application=None):
super(AppDetailsDebFile, self).__init__(db, doc, application)
if doc:
raise DebFileOpenError("AppDetailsDebFile: doc must be None.")
self._error = None
self._deb = None
# check errors before creating the DebPackage
if not os.path.exists(self._app.request):
self._error = _("Not found")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d "
"does not exist.")) % utf8(self._app.request)
elif not is_deb_file(self._app.request):
self._error = _("Not found")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d "
"is not a software package.")) % utf8(self._app.request)
if self._error is not None:
return
try:
with ExecutionTime("create DebPackage"):
# Cache() used to be faster here than self._cache._cache
# but that is no longer the case with the latest apt
self._deb = DebPackage(self._app.request, self._cache._cache)
except:
# deb files which are corrupt
self._error = _("Internal Error")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d "
"could not be opened.")) % utf8(self._app.request)
return
if self.pkgname and self.pkgname != self._app.pkgname:
# this happens when the deb file has a quirky file name
self._app.pkgname = self.pkgname
# load pkg cache
self._pkg = None
if (self._app.pkgname in self._cache
and self._cache[self._app.pkgname].candidate):
self._pkg = self._cache[self._app.pkgname]
# load xapian document
self._doc = None
try:
self._doc = self._db.get_xapian_document(
self._app.appname, self._app.pkgname)
except:
pass
# check deb and set failure state on error
with ExecutionTime("AppDetailsDebFile._deb.check()"):
if not self._deb.check():
self._error = self._deb._failure_string.strip()
def __init__(self, db, doc=None, application=None):
super(AppDetailsDebFile, self).__init__(db, doc, application)
if doc:
raise ValueError("doc must be None for deb files")
try:
# for some reason Cache() is much faster than "self._cache._cache"
# on startup
with ExecutionTime("create DebPackage"):
self._deb = DebPackage(self._app.request, Cache())
except:
self._deb = None
self._pkg = None
if not os.path.exists(self._app.request):
self._error = _("Not found")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d does not exist.")) % utf8(
self._app.request)
else:
mimetype = guess_type(self._app.request)
if mimetype[0] != "application/x-debian-package":
self._error = _("Not found")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d is not a software package."
)) % utf8(self._app.request)
else:
# deb files which are corrupt
self._error = _("Internal Error")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d could not be opened.")
) % utf8(self._app.request)
return
if self.pkgname and self.pkgname != self._app.pkgname:
# this happens when the deb file has a quirky file name
self._app.pkgname = self.pkgname
# load pkg cache
self._pkg = None
if (self._app.pkgname in self._cache
and self._cache[self._app.pkgname].candidate):
self._pkg = self._cache[self._app.pkgname]
# load xapian document
self._doc = None
try:
self._doc = self._db.get_xapian_document(
self._app.appname, self._app.pkgname)
except:
pass
# check deb and set failure state on error
with ExecutionTime("AppDetailsDebFile._deb.check()"):
if not self._deb.check():
self._error = self._deb._failure_string
def init_view(self):
"""
Initialize those UI components that are common to all subclasses of
SoftwarePane. Note that this method is intended to be called by
the subclass itself at the start of its own init_view() implementation.
"""
# common UI elements (applist and appdetails)
# its the job of the Child class to put it into a good location
# list
self.box_app_list = Gtk.VBox()
# search aid
self.search_aid = SearchAid(self)
self.box_app_list.pack_start(self.search_aid, False, False, 0)
with ExecutionTime("SoftwarePane.AppView"):
self.app_view = AppView(self.db, self.cache, self.icons,
self.show_ratings)
self.app_view.connect("sort-method-changed",
self.on_app_view_sort_method_changed)
self.init_atk_name(self.app_view, "app_view")
self.box_app_list.pack_start(self.app_view, True, True, 0)
self.app_view.connect("application-selected",
self.on_application_selected)
self.app_view.connect("application-activated",
self.on_application_activated)
# details
self.scroll_details = Gtk.ScrolledWindow()
self.scroll_details.set_policy(Gtk.PolicyType.AUTOMATIC,
Gtk.PolicyType.AUTOMATIC)
# delayed import gives ~1s speedup until visible on the raspi
with ExecutionTime("import AppDetailsView"):
from softwarecenter.ui.gtk3.views.appdetailsview import (
AppDetailsView)
with ExecutionTime("SoftwarePane.AppDetailsView"):
self.app_details_view = AppDetailsView(self.db, self.distro,
self.icons, self.cache)
self.app_details_view.connect("different-application-selected",
self.on_application_activated)
self.scroll_details.add(self.app_details_view)
# when the cache changes, refresh the app list
self.cache.connect("cache-ready", self.on_cache_ready)
# connect signals
self.connect("app-list-changed", self.on_app_list_changed)
# db reopen
if self.db:
self.db.connect("reopen", self.on_db_reopen)
def set_active_view(self, view_id):
# no views yet
if not self.all_views:
return
# if the view switches, ensure that the global spinner is hidden
self.spinner.hide()
# emit signal
self.emit('view-changed', view_id)
page_id = self.all_views[view_id]
view_widget = self.get_view_widget(view_id)
# it *seems* that this shouldn't be called here if we want the history
# to work, but I'm not familiar with the code, so I'll leave it here
# for the mean time
# view_page = view_widget.get_current_page()
# view_state = view_widget.state
if self.search_entry.get_text() != view_widget.state.search_term:
self.search_entry.set_text_with_no_signal(
view_widget.state.search_term)
# callback = view_widget.get_callback_for_page(view_page,
# view_state)
# nav_item = NavigationItem(self, view_widget, view_page,
# view_state.copy(), callback)
# self.navhistory.append(nav_item)
self.notebook_view.set_current_page(page_id)
if view_widget:
with ExecutionTime("view_widget.init_view() (%s)" % view_widget):
view_widget.init_view()
return view_widget
def __init__(self,
db,
cache,
icons,
icon_size=48,
global_icon_cache=False):
GObject.GObject.__init__(self)
self.db = db
self.cache = cache
# get all categories
cat_parser = CategoriesParser(db)
with ExecutionTime("cat_parser.parse_applications_menu()"):
self.all_categories = cat_parser.parse_applications_menu()
# reviews stats loader
self.review_loader = get_review_loader(cache, db)
# icon jazz
self.icons = icons
self.icon_size = icon_size
self._missing_icon = None # delay this until actually needed
if global_icon_cache:
self.icon_cache = _app_icon_cache
else:
self.icon_cache = {}
def test_apthistory_rescan_big(self):
""" create big history file and ensure that on rescan the
events are still processed
"""
self._timeouts = []
new_history = os.path.join(self.basedir, "history.log.2")
try:
os.remove(new_history + ".gz")
except OSError:
pass
history = self._get_apt_history()
self.assertEqual(len(history.transactions), 186)
self._generate_big_history_file(new_history)
timer_id = GObject.timeout_add(100, self._glib_timeout)
with ExecutionTime("rescan %s byte file" %
os.path.getsize(new_history + ".gz")):
history._rescan(use_cache=False)
GObject.source_remove(timer_id)
# verify rescan
self.assertTrue(len(history.transactions) > 186)
# check the timeouts
self.assertTrue(len(self._timeouts) > 0)
for i in range(len(self._timeouts) - 1):
# check that we get a max timeout of 0.2s
if abs(self._timeouts[i] - self._timeouts[i + 1]) > 0.2:
raise
os.remove(new_history + ".gz")
def parse_query(parser, search_strings, verbose=True):
str_to_prefix = {'section': 'AE', 'type': 'AT', 'category': 'AC'}
for st in search_strings:
(search_prefix, search_term) = st.split(":")
if search_prefix == "section":
t = str_to_prefix[search_prefix]
s = search_term.lower()
query = xapian.Query(t + s)
for pre in ["universe", "multiverse", "restricted"]:
query = xapian.Query(xapian.Query.OP_OR, query,
xapian.Query("%s%s/%s" % (t, pre, s)))
query = xapian.Query(xapian.Query.OP_OR, query,
xapian.Query("XS%s/%s" % (pre, s)))
else:
query = xapian.Query(str_to_prefix[search_prefix] +
search_term.lower())
enquire = xapian.Enquire(db)
enquire.set_query(query)
with ExecutionTime("Search took"):
mset = enquire.get_mset(0, db.get_doccount())
print "Found %i documents for search '%s'" % (len(mset), st)
if verbose:
for m in mset:
doc = m.document
appname = doc.get_data()
pkgname = doc.get_value(XAPIAN_VALUE_PKGNAME)
print "%s ; %s" % (appname, pkgname)
print
def run_benchmark(db):
# test postlist
with ExecutionTime('postlist("")'):
for m in db.postlist(""):
pass
# test postlist + get_document
with ExecutionTime('postlist+get_document'):
for m in db.postlist(""):
doc = db.get_document(m.docid)
# test postlist + get_document
with ExecutionTime('postlist+get_document+get_value'):
for m in db.postlist(""):
doc = db.get_document(m.docid)
doc.get_value(XAPIAN_VALUE_PKGNAME)
def get_test_window_apptreeview():
cache = get_test_pkg_info()
db = get_test_db()
icons = get_test_gtk3_icon_cache()
# create a filter
app_filter = appfilter.AppFilter(db, cache)
app_filter.set_supported_only(False)
app_filter.set_installed_only(True)
# get the TREEstore
store = appstore2.AppTreeStore(db, cache, icons)
# populate from data
cats = get_test_categories(db)
for cat in cats[:3]:
with ExecutionTime("query cat '%s'" % cat.name):
docs = db.get_docs_from_query(cat.query)
store.set_category_documents(cat, docs)
# ok, this is confusing - the AppView contains the AppTreeView that
# is a tree or list depending on the model
app_view = appview.AppView(db, cache, icons, show_ratings=True)
app_view.set_model(store)
box = Gtk.VBox()
box.pack_start(app_view, True, True, 0)
win = get_test_window(child=box)
return win
def test_open_aptcache(self):
# mvo: for the performance, its critical to have a
# /var/cache/apt/srcpkgcache.bin - otherwise stuff will get slow
# open s-c aptcache
with ExecutionTime("s-c softwarecenter.apt.AptCache"):
self.sccache = get_pkg_info()
# cache is opened with a timeout_add() in get_pkg_info()
time.sleep(0.2)
context = GObject.main_context_default()
while context.pending():
context.iteration()
# compare with plain apt
with ExecutionTime("plain apt: apt.Cache()"):
self.cache = apt.Cache()
with ExecutionTime("plain apt: apt.Cache(memonly=True)"):
self.cache = apt.Cache(memonly=True)
def run_query(parser, search_term):
search_query = parser.parse_query(
search_term,
xapian.QueryParser.FLAG_PARTIAL | xapian.QueryParser.FLAG_BOOLEAN)
enquire = xapian.Enquire(db)
enquire.set_query(search_query)
with ExecutionTime("enquire"):
mset = enquire.get_mset(0, db.get_doccount())
print "len mset: ", len(mset)
def quick_query(self, query):
# a blocking query and does not emit "query-complete"
with ExecutionTime("enquirer.set_query() quick query"):
self.enquirer.set_query(
query,
limit=self.get_app_items_limit(),
nonapps_visible=self.nonapps_visible,
nonblocking_load=False,
filter=self.state.filter)
return len(self.enquirer.matches)
def get_estimated_matches_count(self, query):
with ExecutionTime("estimate item count for query: '%s'" % query):
enquire = xapian.Enquire(self.db.xapiandb)
enquire.set_query(query)
# no performance difference between the two
#tmp_matches = enquire.get_mset(0, 1, None, None)
#nr_pkgs = tmp_matches.get_matches_estimated()
tmp_matches = enquire.get_mset(0, len(self.db), None, None)
nr_pkgs = len(tmp_matches)
return nr_pkgs
def quick_query_len(self, query):
""" do a blocking query that only returns the amount of
matches from this query
"""
with ExecutionTime("enquirer.set_query() quick query"):
self.enquirer.set_query(query,
limit=self.get_app_items_limit(),
nonapps_visible=self.nonapps_visible,
nonblocking_load=False,
filter=self.state.filter)
return len(self.enquirer.matches)
def _refresh_apps_with_apt_cache(self, query):
LOG.debug("softwarepane query: %s" % query)
# a nonblocking query calls on_query_complete once finished
with ExecutionTime("enquirer.set_query()"):
self.enquirer.set_query(
query,
limit=self.get_app_items_limit(),
sortmode=self.get_sort_mode(),
exact=self.is_custom_list(),
nonapps_visible=self.nonapps_visible,
filter=self.state.filter)
return
def _work():
new_text = widget.get_text()
(view, enquirer) = data
with ExecutionTime("total time"):
with ExecutionTime("enquire.set_query()"):
enquirer.set_query(get_query_from_search_entry(new_text),
limit=100 * 1000,
nonapps_visible=NonAppVisibility.ALWAYS_VISIBLE)
store = view.tree_view.get_model()
if store is None:
return
with ExecutionTime("store.clear()"):
store.clear()
with ExecutionTime("store.set_from_matches()"):
store.set_from_matches(enquirer.matches)
with ExecutionTime("model settle (size=%s)" % len(store)):
do_events()
def open(self):
""" (re)open the cache, this sends cache-invalid, cache-ready signals
"""
self._ready = False
self.emit("cache-invalid")
from softwarecenter.utils import ExecutionTime
with ExecutionTime("open the apt cache (in event loop)"):
if self._cache == None:
self._cache = apt.Cache(GtkMainIterationProgress())
else:
self._cache.open(GtkMainIterationProgress())
self._ready = True
self.emit("cache-ready")
if self._cache.broken_count > 0:
self.emit("cache-broken")
def get_test_window():
import softwarecenter.log
softwarecenter.log.root.setLevel(level=logging.DEBUG)
softwarecenter.log.add_filters_from_string("performance")
fmt = logging.Formatter("%(name)s - %(message)s", None)
softwarecenter.log.handler.setFormatter(fmt)
from softwarecenter.testutils import (get_test_db, get_test_pkg_info,
get_test_gtk3_icon_cache,
get_test_categories)
cache = get_test_pkg_info()
db = get_test_db()
icons = get_test_gtk3_icon_cache()
# create a filter
from softwarecenter.db.appfilter import AppFilter
filter = AppFilter(db, cache)
filter.set_supported_only(False)
filter.set_installed_only(True)
# get the TREEstore
from softwarecenter.ui.gtk3.models.appstore2 import AppTreeStore
store = AppTreeStore(db, cache, icons)
# populate from data
cats = get_test_categories(db)
for cat in cats[:3]:
with ExecutionTime("query cat '%s'" % cat.name):
docs = db.get_docs_from_query(cat.query)
store.set_category_documents(cat, docs)
# ok, this is confusing - the AppView contains the AppTreeView that
# is a tree or list depending on the model
from softwarecenter.ui.gtk3.views.appview import AppView
app_view = AppView(db, cache, icons, show_ratings=True)
app_view.set_model(store)
box = Gtk.VBox()
box.pack_start(app_view, True, True, 0)
win = Gtk.Window()
win.add(box)
win.connect("destroy", lambda x: Gtk.main_quit())
win.set_size_request(600, 400)
win.show_all()
return win
def open(self, blocking=False):
""" (re)open the cache, this sends cache-invalid, cache-ready signals
"""
LOG.info("aptcache.open()")
self._ready = False
self.emit("cache-invalid")
if blocking:
progress = None
else:
progress = GtkMainIterationProgress()
with ExecutionTime("open the apt cache (in event loop)"):
if self._cache is None:
self._cache = apt.Cache(progress)
else:
self._cache.open(progress)
self._ready = True
self.emit("cache-ready")
if self._cache.broken_count > 0:
self.emit("cache-broken")
def get_query_for_pkgnames(self, pkgnames):
""" return a xapian query that matches exactly the list of pkgnames """
enquire = xapian.Enquire(self.xapiandb)
query = xapian.Query()
for pkgname in pkgnames:
# even on the raspberry pi this query is super quick (~0.003s)
with ExecutionTime("de-dup query_for_pkgnames"):
tmp_query = xapian.Query("AP" + pkgname)
enquire.set_query(tmp_query)
result = enquire.get_mset(0, 1)
# see bug #1043159, we need to ensure that we de-duplicate
# when there is a pkg and a app (e.g. from the s-c-agent) in the db
if len(result) == 1:
query = xapian.Query(xapian.Query.OP_OR,
query,
xapian.Query("AP" + pkgname))
else:
query = xapian.Query(xapian.Query.OP_OR,
query,
xapian.Query("XP" + pkgname))
return query
def _rescan(self, use_cache=True):
self._history_ready = False
self._transactions = []
p = os.path.join(SOFTWARE_CENTER_CACHE_DIR, "apthistory.p")
cachetime = 0
if os.path.exists(p) and use_cache:
with ExecutionTime("loading pickle cache"):
try:
self._transactions = pickle.load(open(p))
cachetime = os.path.getmtime(p)
except:
LOG.exception("failed to load cache")
for history_gz_file in sorted(glob.glob(self.history_file + ".*.gz"),
cmp=self._mtime_cmp):
if os.path.getmtime(history_gz_file) < cachetime:
LOG.debug("skipping already cached '%s'" % history_gz_file)
continue
self._scan(history_gz_file)
self._scan(self.history_file)
if use_cache:
pickle.dump(self._transactions, open(p, "w"))
self._history_ready = True
def __init__(self, db, cache, icons, show_ratings):
Gtk.VBox.__init__(self)
#~ self.set_name("app-view")
# app properties helper
with ExecutionTime("Appview.__init__ create AppPropertiesHelper"):
self.helper = AppPropertiesHelper(db, cache, icons)
# misc internal containers
self.header_hbox = Gtk.HBox()
self.header_hbox.set_border_width(StockEms.MEDIUM)
self.pack_start(self.header_hbox, False, False, 0)
self.tree_view_scroll = Gtk.ScrolledWindow()
self.pack_start(self.tree_view_scroll, True, True, 0)
# category label
self.header_label = Gtk.Label()
self.header_label.set_use_markup(True)
self.header_hbox.pack_start(self.header_label, False, False, 0)
# sort methods comboboxs
# variant 1 includes sort by search relevance
self.sort_methods_combobox = self._get_sort_methods_combobox()
combo_alignment = Gtk.Alignment.new(0.5, 0.5, 1.0, 0.0)
combo_alignment.add(self.sort_methods_combobox)
self.header_hbox.pack_end(combo_alignment, False, False, 0)
# content views
self.tree_view = AppTreeView(self, db, icons, show_ratings, store=None)
self.tree_view_scroll.add(self.tree_view)
self.appcount = None
self.vadj = 0.0
# list view sorting stuff
self._force_default_sort_method = True
self._handler = self.sort_methods_combobox.connect(
"changed", self.on_sort_method_changed)
def get_details(self, db):
with ExecutionTime("get_details for DebFileApplication"):
details = AppDetailsDebFile(db, application=self)
return details
def _blocking_perform_search(self):
# WARNING this call may run in a thread, so it's *not*
# allowed to touch gtk, otherwise hell breaks loose
# performance only: this is only needed to avoid the
# python __call__ overhead for each item if we can avoid it
# use a unique instance of both enquire and xapian database
# so concurrent queries don't result in an inconsistent database
# an alternative would be to serialise queries
enquire = xapian.Enquire(self.db.xapiandb)
if self.filter and self.filter.required:
xfilter = self.filter
else:
xfilter = None
# go over the queries
self.nr_apps, self.nr_pkgs = 0, 0
_matches = self._matches
match_docids = self.match_docids
for q in self.search_query:
LOG.debug("initial query: '%s'" % q)
# for searches we may want to disable show/hide
terms = [term for term in q]
exact_pkgname_query = (len(terms) == 1 and
terms[0].startswith("XP"))
# see if we should do a app query and skip the pkg query
# see bug #891613 and #1043159
if exact_pkgname_query:
with ExecutionTime("de-duplication"):
q_app = xapian.Query(terms[0].replace("XP", "AP"))
nr_apps, nr_pkgs = self._get_estimate_nr_apps_and_nr_pkgs(
enquire, q_app, xfilter)
if nr_apps == 1:
q = q_app
# this is a app query now
exact_pkgname_query = False
with ExecutionTime("calculate nr_apps and nr_pkgs: "):
nr_apps, nr_pkgs = self._get_estimate_nr_apps_and_nr_pkgs(
enquire, q, xfilter)
self.nr_apps += nr_apps
self.nr_pkgs += nr_pkgs
# only show apps by default (unless in always visible mode)
if self.nonapps_visible != NonAppVisibility.ALWAYS_VISIBLE:
if not exact_pkgname_query:
q = xapian.Query(xapian.Query.OP_AND,
xapian.Query("ATapplication"),
q)
LOG.debug("nearly completely filtered query: '%s'" % q)
# filter out docs of pkgs of which there exists a doc of the app
# FIXME: make this configurable again?
enquire.set_query(xapian.Query(xapian.Query.OP_AND_NOT,
q, xapian.Query("XD")))
# sort results
# cataloged time - what's new category
if self.sortmode == SortMethods.BY_CATALOGED_TIME:
sorter = xapian.MultiValueKeyMaker()
if (self.db._axi_values and
"catalogedtime" in self.db._axi_values):
sorter.add_value(
self.db._axi_values["catalogedtime"])
sorter.add_value(XapianValues.DB_CATALOGED_TIME)
enquire.set_sort_by_key(sorter, reverse=True)
elif self.sortmode == SortMethods.BY_TOP_RATED:
from softwarecenter.backend.reviews import get_review_loader
review_loader = get_review_loader(self.cache, self.db)
sorter = TopRatedSorter(self.db, review_loader)
enquire.set_sort_by_key(sorter, reverse=True)
# search ranking - when searching
elif self.sortmode == SortMethods.BY_SEARCH_RANKING:
#enquire.set_sort_by_value(XapianValues.POPCON)
# use the default enquire.set_sort_by_relevance()
pass
# display name - all categories / channels
elif (self.db._axi_values and
"display_name" in self.db._axi_values):
enquire.set_sort_by_key(LocaleSorter(self.db), reverse=False)
# fallback to pkgname - if needed?
# fallback to pkgname - if needed?
else:
enquire.set_sort_by_value_then_relevance(
XapianValues.PKGNAME, False)
#~ try:
if self.limit == 0:
matches = enquire.get_mset(0, len(self.db), None, xfilter)
else:
matches = enquire.get_mset(0, self.limit, None, xfilter)
LOG.debug("found ~%i matches" % matches.get_matches_estimated())
#~ except:
#~ logging.exception("get_mset")
#~ matches = []
# promote exact matches to a "app", this will make the
# show/hide technical items work correctly
if exact_pkgname_query and len(matches) == 1:
self.nr_apps += 1
self.nr_pkgs -= 2
# add matches, but don't duplicate docids
with ExecutionTime("append new matches to existing ones:"):
for match in matches:
if not match.docid in match_docids:
_matches.append(match)
match_docids.add(match.docid)
# if we have no results, try forcing pkgs to be displayed
# if not NonAppVisibility.NEVER_VISIBLE is set
if (not _matches and
self.nonapps_visible not in (NonAppVisibility.ALWAYS_VISIBLE,
NonAppVisibility.NEVER_VISIBLE)):
self.nonapps_visible = NonAppVisibility.ALWAYS_VISIBLE
self._blocking_perform_search()
# wake up the UI if run in a search thread
self._perform_search_complete = True
def set_query(self, search_query,
limit=DEFAULT_SEARCH_LIMIT,
sortmode=SortMethods.UNSORTED,
filter=None,
exact=False,
nonapps_visible=NonAppVisibility.MAYBE_VISIBLE,
nonblocking_load=True,
persistent_duplicate_filter=False):
"""
Set a new query
:Parameters:
- `search_query`: a single search as a xapian.Query or a list
- `limit`: how many items the search should return (0 == unlimited)
- `sortmode`: sort the result
- `filter`: filter functions that can be used to filter the
data further. A python function that gets a pkgname
- `exact`: If true, indexes of queries without matches will be
maintained in the store (useful to show e.g. a row
with "??? not found")
- `nonapps_visible`: decide whether adding non apps in the model or
not. Can be NonAppVisibility.ALWAYS_VISIBLE
/NonAppVisibility.MAYBE_VISIBLE
/NonAppVisibility.NEVER_VISIBLE
(NonAppVisibility.MAYBE_VISIBLE will return non
apps result if no matching apps is found)
- `nonblocking_load`: set to False to execute the query inside the
current thread. Defaults to True to allow the
search to be performed without blocking the UI.
- 'persistent_duplicate_filter': if True allows filtering of duplicate
matches across multiple queries
"""
self.search_query = SearchQuery(search_query)
self.limit = limit
self.sortmode = sortmode
# make a copy for good measure
if filter:
self.filter = filter.copy()
else:
self.filter = None
self.exact = exact
self.nonblocking_load = nonblocking_load
self.nonapps_visible = nonapps_visible
# no search query means "all"
if not search_query:
self.search_query = SearchQuery(xapian.Query(""))
self.sortmode = SortMethods.BY_ALPHABET
self.limit = 0
# flush old query matches
self._matches = []
if not persistent_duplicate_filter:
self.match_docids = set()
# we support single and list search_queries,
# if list, we append them one by one
with ExecutionTime("populate model from query: '%s' (threaded: %s)" % (
" ; ".join([str(q) for q in self.search_query]),
self.nonblocking_load), with_traceback=False):
if self.nonblocking_load:
self._threaded_perform_search()
else:
self._blocking_perform_search()
return True
run_query(parser, "abc")
print "with db query: a"
parser.set_database(db)
parser.add_boolean_prefix("pkg", "AP")
parser.set_default_op(xapian.Query.OP_AND)
run_query(parser, "a")
run_query(parser, "ab")
run_query(parser, "abc")
print "query for all !ATapplication"
search_query = xapian.Query(xapian.Query.OP_AND_NOT, xapian.Query(""),
xapian.Query("ATapplication"))
enquire = xapian.Enquire(db)
enquire.set_query(search_query)
with ExecutionTime("enquire"):
mset = enquire.get_mset(0, db.get_doccount())
print "len mset: ", len(mset)
print "look at all !ATapplication"
search_query = xapian.Query(xapian.Query.OP_AND_NOT, xapian.Query(""),
xapian.Query("ATapplication"))
enquire = xapian.Enquire(db)
enquire.set_query(search_query)
with ExecutionTime("enquire"):
mset = enquire.get_mset(0, db.get_doccount())
print "len mset: ", len(mset)
for m in mset:
doc = m.document
appname = doc.get_value(XAPIAN_VALUE_APPNAME)
pkgname = doc.get_value(XAPIAN_VALUE_PKGNAME)
def profiled_rebuild_oneconfview():
with ExecutionTime("rebuild_oneconfview"):
rebuild_oneconfview()
def profiled_rebuild_categorised_view():
with ExecutionTime("rebuild_categorized_view"):
rebuild_categorised_view()
parser = OptionParser()
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="print found apps/pkgs too")
(options, args) = parser.parse_args()
pathname = os.path.join(XAPIAN_BASE_PATH, "xapian")
db = xapian.Database(pathname)
axi = xapian.Database("/var/lib/apt-xapian-index/index")
db.add_database(axi)
query_set = set()
with ExecutionTime("allterms XP/AP"):
for search_term in args:
for m in db.allterms("XP"):
term = m.term
if search_term in term:
query_set.add(term)
for m in db.allterms("AP"):
term = m.term
if search_term in term:
query_set.add(term)
print "found terms: ", len(query_set)
if options.verbose:
print sorted(query_set)