def _query_available(self, series_name, arch_tag, for_qa):
if not series_name:
series_name = self.distro.get_codename()
if not arch_tag:
arch_tag = get_current_arch()
# build the command
spawner = SpawnHelper()
spawner.parent_xid = self.xid
spawner.ignore_cache = self.ignore_cache
spawner.connect("data-available", self._on_query_available_data)
spawner.connect("error", lambda spawner, err: self.emit("error", err))
if for_qa:
spawner.needs_auth = True
spawner.run_generic_piston_helper(
"SoftwareCenterAgentAPI",
"available_apps_qa",
lang=get_language(),
series=series_name,
arch=arch_tag)
else:
spawner.run_generic_piston_helper(
"SoftwareCenterAgentAPI",
"available_apps",
lang=get_language(),
series=series_name,
arch=arch_tag)
def buy_app(self):
""" initiate the purchase transaction """
lang = get_language()
distro = self.distro.get_codename()
url = self.distro.PURCHASE_APP_URL % (lang, distro, urlencode({
'archive_id' : self.appdetails.ppaname,
'arch' : get_current_arch() ,
}))
self.emit("purchase-requested", self.app, url)
def buy_app(self, app):
""" initiate the purchase transaction """
lang = get_language()
appdetails = app.get_details(self.db)
url = self.distro.PURCHASE_APP_URL % (
lang, self.distro.get_codename(),
urlencode({
'archive_id': appdetails.ppaname,
'arch': get_current_arch(),
}))
self.emit("purchase-requested", app, appdetails.icon, url)
def buy_app(self, app):
""" initiate the purchase transaction """
lang = get_language()
appdetails = app.get_details(self.db)
url = self.distro.PURCHASE_APP_URL % (
lang, self.distro.get_codename(), urlencode({
'archive_id': appdetails.ppaname,
'arch': get_current_arch()
})
)
self.emit("purchase-requested", app, appdetails.icon, url)
def _query_available(self, series_name, arch_tag, for_qa):
language = get_language()
if not series_name:
series_name = self.distro.get_codename()
if not arch_tag:
arch_tag = get_current_arch()
# build the command
cmd = self.HELPER_CMD[:]
if for_qa:
cmd.append("available_apps_qa")
else:
cmd.append("available_apps")
cmd += [language,
series_name,
arch_tag,
]
spawner = SpawnHelper()
spawner.connect("data-available", self._on_query_available_data)
spawner.connect("error", lambda spawner, err: self.emit("error", err))
spawner.run(cmd)
def _query_available(self, series_name, arch_tag, for_qa):
language = get_language()
if not series_name:
series_name = self.distro.get_codename()
if not arch_tag:
arch_tag = get_current_arch()
# build the command
cmd = self.HELPER_CMD[:]
if for_qa:
cmd.append("available_apps_qa")
else:
cmd.append("available_apps")
cmd += [language,
series_name,
arch_tag,
]
spawner = SpawnHelper()
spawner.connect("data-available", self._on_query_available_data)
spawner.connect("error", lambda spawner, err: self.emit("error", err))
spawner.run(cmd)
def make_doc(self, cache):
"""Build a Xapian document from the desktop info."""
doc = xapian.Document()
# app name is the data
name = self._set_doc_from_key(doc, AppInfoFields.NAME)
assert name is not None
doc.set_data(name)
self._set_doc_from_key(doc,
AppInfoFields.NAME_UNTRANSLATED,
translated=False)
# check if we should ignore this file
if self.is_ignored:
LOG.debug("%r.make_doc: %r is ignored.", self.__class__.__name__,
self.desktopf)
return
# architecture
pkgname_extension = ''
arches = self._set_doc_from_key(doc, AppInfoFields.ARCH)
if arches:
native_archs = get_current_arch() in arches.split(',')
foreign_archs = list(
set(arches.split(',')) & set(get_foreign_architectures()))
if not (native_archs or foreign_archs):
return
if not native_archs and foreign_archs:
pkgname_extension = ':' + foreign_archs[0]
# package name
pkgname = self._set_doc_from_key(doc,
AppInfoFields.PACKAGE,
pkgname_extension=pkgname_extension)
doc.add_value(XapianValues.DESKTOP_FILE, self.desktopf)
# display name
display_name = axi_values.get("display_name")
if display_name is not None:
doc.add_value(display_name, name)
# cataloged_times
catalogedtime = axi_values.get("catalogedtime")
if catalogedtime is not None and pkgname in cataloged_times:
doc.add_value(catalogedtime,
xapian.sortable_serialise(cataloged_times[pkgname]))
# section (mail, base, ..)
if pkgname in cache and cache[pkgname].candidate:
section = cache[pkgname].section
doc.add_term("AE" + section)
fields = (
AppInfoFields.CHANNEL, # channel (third party stuff)
AppInfoFields.DEB_LINE, # deb-line (third party)
AppInfoFields.DESCRIPTION, # description software-center extension
AppInfoFields.GETTEXT_DOMAIN, # check gettext domain
AppInfoFields.ICON, # icon
AppInfoFields.LICENSE, # license (third party)
AppInfoFields.LICENSE_KEY, # license key (third party)
AppInfoFields.LICENSE_KEY_PATH, # license keypath (third party)
AppInfoFields.PPA, # PPA (third party stuff)
AppInfoFields.PURCHASED_DATE, # purchased date
AppInfoFields.SCREENSHOT_URLS, # screenshot (for third party)
AppInfoFields.SECTION, # pocket (main, restricted, ...)
AppInfoFields.SIGNING_KEY_ID, # signing key (third party)
AppInfoFields.SUPPORT_URL, # support url (mainly pay stuff)
AppInfoFields.SUPPORTED_DISTROS, # supported distros
AppInfoFields.THUMBNAIL_URL, # thumbnail (for third party)
AppInfoFields.VERSION, # version support (for e.g. the scagent)
AppInfoFields.VIDEO_URL, # video support (for third party mostly)
AppInfoFields.WEBSITE, # homepage url (developer website)
)
for field in fields:
self._set_doc_from_key(doc, field)
# date published
date_published_str = self._set_doc_from_key(
doc, AppInfoFields.DATE_PUBLISHED)
# we use the date published value for the cataloged time as well
if date_published_str is not None:
LOG.debug("pkgname: %s, date_published cataloged time is: %s",
pkgname, date_published_str)
date_published = time.mktime(
time.strptime(date_published_str, "%Y-%m-%d %H:%M:%S"))
# a value for our own DB
doc.add_value(XapianValues.DB_CATALOGED_TIME,
xapian.sortable_serialise(date_published))
if "catalogedtime" in axi_values:
# compat with a-x-i
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(date_published))
# icon (for third party)
url = self._set_doc_from_key(doc, AppInfoFields.ICON_URL)
if url and self.get_value(AppInfoFields.ICON) is None:
# prefix pkgname to avoid name clashes
doc.add_value(XapianValues.ICON,
"%s-icon-%s" % (pkgname, os.path.basename(url)))
# price (pay stuff)
price = self._set_doc_from_key(doc, AppInfoFields.PRICE)
if price:
# this is a commercial app, indicate it in the component value
doc.add_value(XapianValues.ARCHIVE_SECTION, "commercial")
# this is hard-coded to US dollar for now, but if the server
# ever changes we can update
doc.add_value(XapianValues.CURRENCY, "US$")
# add donwload size as string (its send as int)
download_size = self.get_value(AppInfoFields.DOWNLOAD_SIZE)
if download_size is not None:
doc.add_value(XapianValues.DOWNLOAD_SIZE,
xapian.sortable_serialise((download_size)))
# write out categories
for cat in self.get_categories():
doc.add_term("AC" + cat.lower())
categories_string = ";".join(self.get_categories())
doc.add_value(XapianValues.CATEGORIES, categories_string)
# mimetypes
for mime in self.get_mimetypes():
doc.add_term("AM" + mime.lower())
# get type (to distinguish between apps and packages)
app_type = self.get_value(AppInfoFields.TYPE)
if app_type:
doc.add_term("AT" + app_type.lower())
# (deb)tags (in addition to the pkgname debtags)
tags_string = self.get_value(AppInfoFields.TAGS)
if tags_string:
# convert to list and register
tags = [tag.strip().lower() for tag in tags_string.split(",")]
for tag in tags:
doc.add_term("XT" + tag)
# ENFORCE region blacklist/whitelist by not registering
# the app at all
region = get_region_cached()
if region:
countrycode = region["countrycode"].lower()
blacklist = [
t.replace(REGION_BLACKLIST_TAG, "") for t in tags
if t.startswith(REGION_BLACKLIST_TAG)
]
whitelist = [
t.replace(REGION_WHITELIST_TAG, "") for t in tags
if t.startswith(REGION_WHITELIST_TAG)
]
if countrycode in blacklist:
if countrycode in whitelist:
LOG.debug(
"%r.make_doc: %r black AND whitelisted for "
"region %r. Treating as blacklisted.",
self.__class__.__name__, name, countrycode)
LOG.debug(
"%r.make_doc: skipping region restricted app %r "
"(blacklisted)", self.__class__.__name__, name)
return
if len(whitelist) > 0 and countrycode not in whitelist:
LOG.debug(
"%r.make_doc: skipping region restricted "
"app %r (region not whitelisted)",
self.__class__.__name__, name)
return
# popcon
# FIXME: popularity not only based on popcon but also
# on archive section, third party app etc
popcon = self._set_doc_from_key(doc, AppInfoFields.POPCON)
if popcon is not None:
global popcon_max
popcon_max = max(popcon_max, popcon)
# comment goes into the summary data if there is one,
# otherwise we try GenericName and if nothing else,
# the summary of the candidate package
summary = self._set_doc_from_key(doc, AppInfoFields.SUMMARY, name=name)
if summary is None and pkgname in cache and cache[pkgname].candidate:
summary = cache[pkgname].candidate.summary
doc.add_value(XapianValues.SUMMARY, summary)
return doc
def make_doc(self, cache):
"""Build a Xapian document from the desktop info."""
doc = xapian.Document()
# app name is the data
name = self._set_doc_from_key(doc, AppInfoFields.NAME)
assert name is not None
doc.set_data(name)
self._set_doc_from_key(doc, AppInfoFields.NAME_UNTRANSLATED,
translated=False)
# check if we should ignore this file
if self.is_ignored:
LOG.debug("%r.make_doc: %r is ignored.",
self.__class__.__name__, self.desktopf)
return
# architecture
pkgname_extension = ''
arches = self._set_doc_from_key(doc, AppInfoFields.ARCH)
if arches:
native_archs = get_current_arch() in arches.split(',')
foreign_archs = list(set(arches.split(',')) &
set(get_foreign_architectures()))
if not (native_archs or foreign_archs):
return
if not native_archs and foreign_archs:
pkgname_extension = ':' + foreign_archs[0]
# package name
pkgname = self._set_doc_from_key(doc, AppInfoFields.PACKAGE,
pkgname_extension=pkgname_extension)
doc.add_value(XapianValues.DESKTOP_FILE, self.desktopf)
# display name
display_name = axi_values.get("display_name")
if display_name is not None:
doc.add_value(display_name, name)
# cataloged_times
catalogedtime = axi_values.get("catalogedtime")
if catalogedtime is not None and pkgname in cataloged_times:
doc.add_value(catalogedtime,
xapian.sortable_serialise(cataloged_times[pkgname]))
# section (mail, base, ..)
if pkgname in cache and cache[pkgname].candidate:
section = cache[pkgname].section
doc.add_term("AE" + section)
fields = (
AppInfoFields.CHANNEL, # channel (third party stuff)
AppInfoFields.DEB_LINE, # deb-line (third party)
AppInfoFields.DESCRIPTION, # description software-center extension
AppInfoFields.GETTEXT_DOMAIN, # check gettext domain
AppInfoFields.ICON, # icon
AppInfoFields.LICENSE, # license (third party)
AppInfoFields.LICENSE_KEY, # license key (third party)
AppInfoFields.LICENSE_KEY_PATH, # license keypath (third party)
AppInfoFields.PPA, # PPA (third party stuff)
AppInfoFields.PURCHASED_DATE, # purchased date
AppInfoFields.SCREENSHOT_URLS, # screenshot (for third party)
AppInfoFields.SECTION, # pocket (main, restricted, ...)
AppInfoFields.SIGNING_KEY_ID, # signing key (third party)
AppInfoFields.SUPPORT_URL, # support url (mainly pay stuff)
AppInfoFields.SUPPORTED_DISTROS, # supported distros
AppInfoFields.THUMBNAIL_URL, # thumbnail (for third party)
AppInfoFields.VERSION, # version support (for e.g. the scagent)
AppInfoFields.VIDEO_URL, # video support (for third party mostly)
AppInfoFields.WEBSITE, # homepage url (developer website)
)
for field in fields:
self._set_doc_from_key(doc, field)
# date published
date_published_str = self._set_doc_from_key(
doc, AppInfoFields.DATE_PUBLISHED)
# we use the date published value for the cataloged time as well
if date_published_str is not None:
LOG.debug("pkgname: %s, date_published cataloged time is: %s",
pkgname, date_published_str)
date_published = time.mktime(time.strptime(date_published_str,
"%Y-%m-%d %H:%M:%S"))
# a value for our own DB
doc.add_value(XapianValues.DB_CATALOGED_TIME,
xapian.sortable_serialise(date_published))
if "catalogedtime" in axi_values:
# compat with a-x-i
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(date_published))
# icon (for third party)
url = self._set_doc_from_key(doc, AppInfoFields.ICON_URL)
if url and self.get_value(AppInfoFields.ICON) is None:
# prefix pkgname to avoid name clashes
doc.add_value(XapianValues.ICON,
"%s-icon-%s" % (pkgname, os.path.basename(url)))
# price (pay stuff)
price = self._set_doc_from_key(doc, AppInfoFields.PRICE)
if price:
# this is a commercial app, indicate it in the component value
doc.add_value(XapianValues.ARCHIVE_SECTION, "commercial")
# this is hard-coded to US dollar for now, but if the server
# ever changes we can update
doc.add_value(XapianValues.CURRENCY, "US$")
# add donwload size as string (its send as int)
download_size = self.get_value(AppInfoFields.DOWNLOAD_SIZE)
if download_size is not None:
doc.add_value(XapianValues.DOWNLOAD_SIZE,
xapian.sortable_serialise((download_size)))
# write out categories
for cat in self.get_categories():
doc.add_term("AC" + cat.lower())
categories_string = ";".join(self.get_categories())
doc.add_value(XapianValues.CATEGORIES, categories_string)
# mimetypes
for mime in self.get_mimetypes():
doc.add_term("AM" + mime.lower())
# get type (to distinguish between apps and packages)
app_type = self.get_value(AppInfoFields.TYPE)
if app_type:
doc.add_term("AT" + app_type.lower())
# (deb)tags (in addition to the pkgname debtags)
tags_string = self.get_value(AppInfoFields.TAGS)
if tags_string:
# convert to list and register
tags = [tag.strip().lower() for tag in tags_string.split(",")]
for tag in tags:
doc.add_term("XT" + tag)
# ENFORCE region blacklist/whitelist by not registering
# the app at all
region = get_region_cached()
if region:
countrycode = region["countrycode"].lower()
blacklist = [t.replace(REGION_BLACKLIST_TAG, "")
for t in tags if
t.startswith(REGION_BLACKLIST_TAG)]
whitelist = [t.replace(REGION_WHITELIST_TAG, "")
for t in tags if
t.startswith(REGION_WHITELIST_TAG)]
if countrycode in blacklist:
if countrycode in whitelist:
LOG.debug("%r.make_doc: %r black AND whitelisted for "
"region %r. Treating as blacklisted.",
self.__class__.__name__, name, countrycode)
LOG.debug("%r.make_doc: skipping region restricted app %r "
"(blacklisted)", self.__class__.__name__, name)
return
if len(whitelist) > 0 and countrycode not in whitelist:
LOG.debug("%r.make_doc: skipping region restricted "
"app %r (region not whitelisted)",
self.__class__.__name__, name)
return
# popcon
# FIXME: popularity not only based on popcon but also
# on archive section, third party app etc
popcon = self._set_doc_from_key(doc, AppInfoFields.POPCON)
if popcon is not None:
global popcon_max
popcon_max = max(popcon_max, popcon)
# comment goes into the summary data if there is one,
# otherwise we try GenericName and if nothing else,
# the summary of the candidate package
summary = self._set_doc_from_key(doc, AppInfoFields.SUMMARY, name=name)
if summary is None and pkgname in cache and cache[pkgname].candidate:
summary = cache[pkgname].candidate.summary
doc.add_value(XapianValues.SUMMARY, summary)
return doc
def make_doc_from_parser(parser, cache):
# XXX 2012-01-19 michaeln I'm just pulling this code out from
# index_app_info_from_parser, but it'd be great to further
# refactor it - it looks quite scary :-)
doc = xapian.Document()
# app name is the data
if parser.has_option_desktop("X-Ubuntu-Software-Center-Name"):
name = parser.get_desktop("X-Ubuntu-Software-Center-Name")
untranslated_name = parser.get_desktop("X-Ubuntu-Software-Center-Name",
translated=False)
elif parser.has_option_desktop("X-GNOME-FullName"):
name = parser.get_desktop("X-GNOME-FullName")
untranslated_name = parser.get_desktop("X-GNOME-FullName",
translated=False)
else:
name = parser.get_desktop("Name")
untranslated_name = parser.get_desktop("Name", translated=False)
doc.set_data(name)
doc.add_value(XapianValues.APPNAME_UNTRANSLATED, untranslated_name)
# check if we should ignore this file
if parser.has_option_desktop("X-AppInstall-Ignore"):
ignore = parser.get_desktop("X-AppInstall-Ignore")
if ignore.strip().lower() == "true":
LOG.debug("X-AppInstall-Ignore found for '%s'" % parser.desktopf)
return
# architecture
pkgname_extension = ''
if parser.has_option_desktop("X-AppInstall-Architectures"):
arches = parser.get_desktop("X-AppInstall-Architectures")
doc.add_value(XapianValues.ARCHIVE_ARCH, arches)
native_archs = get_current_arch() in arches.split(',')
foreign_archs = list(set(arches.split(',')) &
set(get_foreign_architectures()))
if not (native_archs or foreign_archs):
return
if not native_archs and foreign_archs:
pkgname_extension = ':' + foreign_archs[0]
# package name
pkgname = parser.get_desktop("X-AppInstall-Package") + pkgname_extension
doc.add_term("AP" + pkgname)
if '-' in pkgname:
# we need this to work around xapian oddness
doc.add_term(pkgname.replace('-', '_'))
doc.add_value(XapianValues.PKGNAME, pkgname)
doc.add_value(XapianValues.DESKTOP_FILE, parser.desktopf)
# display name
if "display_name" in axi_values:
doc.add_value(axi_values["display_name"], name)
# cataloged_times
if "catalogedtime" in axi_values:
if pkgname in cataloged_times:
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(cataloged_times[pkgname]))
# pocket (main, restricted, ...)
if parser.has_option_desktop("X-AppInstall-Section"):
archive_section = parser.get_desktop("X-AppInstall-Section")
doc.add_term("AS" + archive_section)
doc.add_value(XapianValues.ARCHIVE_SECTION, archive_section)
# section (mail, base, ..)
if pkgname in cache and cache[pkgname].candidate:
section = cache[pkgname].section
doc.add_term("AE" + section)
# channel (third party stuff)
if parser.has_option_desktop("X-AppInstall-Channel"):
archive_channel = parser.get_desktop("X-AppInstall-Channel")
doc.add_term("AH" + archive_channel)
doc.add_value(XapianValues.ARCHIVE_CHANNEL, archive_channel)
# signing key (third party)
if parser.has_option_desktop("X-AppInstall-Signing-Key-Id"):
keyid = parser.get_desktop("X-AppInstall-Signing-Key-Id")
doc.add_value(XapianValues.ARCHIVE_SIGNING_KEY_ID, keyid)
# license (third party)
if parser.has_option_desktop("X-AppInstall-License"):
license = parser.get_desktop("X-AppInstall-License")
doc.add_value(XapianValues.LICENSE, license)
# date published
if parser.has_option_desktop("X-AppInstall-Date-Published"):
date_published = parser.get_desktop("X-AppInstall-Date-Published")
if (date_published and
re.match("\d+-\d+-\d+ \d+:\d+:\d+", date_published)):
# strip the subseconds from the end of the published date string
date_published = str(date_published).split(".")[0]
doc.add_value(XapianValues.DATE_PUBLISHED,
date_published)
# we use the date published value for the cataloged time as well
if "catalogedtime" in axi_values:
LOG.debug(
("pkgname: %s, date_published cataloged time is: %s" %
(pkgname, parser.get_desktop("date_published"))))
date_published_sec = time.mktime(
time.strptime(date_published,
"%Y-%m-%d %H:%M:%S"))
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(date_published_sec))
# purchased date
if parser.has_option_desktop("X-AppInstall-Purchased-Date"):
date = parser.get_desktop("X-AppInstall-Purchased-Date")
# strip the subseconds from the end of the date string
doc.add_value(XapianValues.PURCHASED_DATE, str(date).split(".")[0])
# deb-line (third party)
if parser.has_option_desktop("X-AppInstall-Deb-Line"):
debline = parser.get_desktop("X-AppInstall-Deb-Line")
doc.add_value(XapianValues.ARCHIVE_DEB_LINE, debline)
# license key (third party)
if parser.has_option_desktop("X-AppInstall-License-Key"):
key = parser.get_desktop("X-AppInstall-License-Key")
doc.add_value(XapianValues.LICENSE_KEY, key)
# license keypath (third party)
if parser.has_option_desktop("X-AppInstall-License-Key-Path"):
path = parser.get_desktop("X-AppInstall-License-Key-Path")
doc.add_value(XapianValues.LICENSE_KEY_PATH, path)
# PPA (third party stuff)
if parser.has_option_desktop("X-AppInstall-PPA"):
archive_ppa = parser.get_desktop("X-AppInstall-PPA")
if archive_ppa:
doc.add_value(XapianValues.ARCHIVE_PPA, archive_ppa)
# add archive origin data here so that its available even if
# the PPA is not (yet) enabled
doc.add_term("XOO" + "lp-ppa-%s" % archive_ppa.replace("/", "-"))
# screenshot (for third party)
if parser.has_option_desktop("X-AppInstall-Screenshot-Url"):
url = parser.get_desktop("X-AppInstall-Screenshot-Url")
doc.add_value(XapianValues.SCREENSHOT_URLS, url)
# thumbnail (for third party)
if parser.has_option_desktop("X-AppInstall-Thumbnail-Url"):
url = parser.get_desktop("X-AppInstall-Thumbnail-Url")
doc.add_value(XapianValues.THUMBNAIL_URL, url)
# video support (for third party mostly)
if parser.has_option_desktop("X-AppInstall-Video-Url"):
url = parser.get_desktop("X-AppInstall-Video-Url")
doc.add_value(XapianValues.VIDEO_URL, url)
# icon (for third party)
if parser.has_option_desktop("X-AppInstall-Icon-Url"):
url = parser.get_desktop("X-AppInstall-Icon-Url")
doc.add_value(XapianValues.ICON_URL, url)
if not parser.has_option_desktop("X-AppInstall-Icon"):
# prefix pkgname to avoid name clashes
doc.add_value(XapianValues.ICON, "%s-icon-%s" % (
pkgname, os.path.basename(url)))
# price (pay stuff)
if parser.has_option_desktop("X-AppInstall-Price"):
price = parser.get_desktop("X-AppInstall-Price")
doc.add_value(XapianValues.PRICE, price)
# since this is a commercial app, indicate it in the component value
doc.add_value(XapianValues.ARCHIVE_SECTION, "commercial")
# support url (mainly pay stuff)
if parser.has_option_desktop("X-AppInstall-Support-Url"):
url = parser.get_desktop("X-AppInstall-Support-Url")
doc.add_value(XapianValues.SUPPORT_SITE_URL, url)
# icon
if parser.has_option_desktop("Icon"):
icon = parser.get_desktop("Icon")
doc.add_value(XapianValues.ICON, icon)
# write out categories
for cat in parser.get_desktop_categories():
doc.add_term("AC" + cat.lower())
categories_string = ";".join(parser.get_desktop_categories())
doc.add_value(XapianValues.CATEGORIES, categories_string)
for mime in parser.get_desktop_mimetypes():
doc.add_term("AM" + mime.lower())
# get type (to distinguish between apps and packages
if parser.has_option_desktop("Type"):
type = parser.get_desktop("Type")
doc.add_term("AT" + type.lower())
# check gettext domain
if parser.has_option_desktop("X-Ubuntu-Gettext-Domain"):
domain = parser.get_desktop("X-Ubuntu-Gettext-Domain")
doc.add_value(XapianValues.GETTEXT_DOMAIN, domain)
# Description (software-center extension)
if parser.has_option_desktop("X-AppInstall-Description"):
descr = parser.get_desktop("X-AppInstall-Description")
doc.add_value(XapianValues.SC_DESCRIPTION, descr)
if parser.has_option_desktop("Supported-Distros"):
doc.add_value(XapianValues.SC_SUPPORTED_DISTROS,
json.dumps(parser.get_desktop("Supported-Distros")))
# version support (for e.g. the scagent)
if parser.has_option_desktop("X-AppInstall-Version"):
ver = parser.get_desktop("X-AppInstall-Version")
doc.add_value(XapianValues.VERSION_INFO, ver)
# (deb)tags (in addition to the pkgname debtags
if parser.has_option_desktop("X-AppInstall-Tags"):
# register tags
tags_string = parser.get_desktop("X-AppInstall-Tags")
if tags_string:
tags = [tag.strip().lower() for tag in tags_string.split(",")]
for tag in tags:
doc.add_term("XT" + tag)
region = get_region_cached()
if region:
# ENFORCE region blacklist/whitelist by not registering
# the app at all
countrycode = region["countrycode"].lower()
if "%s%s" % (REGION_BLACKLIST_TAG, countrycode) in tags:
LOG.info("skipping region restricted app: '%s'"
" (blacklisted) " % name)
return
# whitelist
for tag in tags:
if (tag.startswith(REGION_WHITELIST_TAG) and not
"%s%s" % (REGION_WHITELIST_TAG, countrycode) in tag):
LOG.info("skipping region restricted app: '%s'"
" (not whitelisted)" % name)
return
# popcon
# FIXME: popularity not only based on popcon but also
# on archive section, third party app etc
if parser.has_option_desktop("X-AppInstall-Popcon"):
popcon = float(parser.get_desktop("X-AppInstall-Popcon"))
# sort_by_value uses string compare, so we need to pad here
doc.add_value(XapianValues.POPCON,
xapian.sortable_serialise(popcon))
global popcon_max
popcon_max = max(popcon_max, popcon)
# comment goes into the summary data if there is one,
# other wise we try GenericName and if nothing else,
# the summary of the package
if parser.has_option_desktop("Comment"):
s = parser.get_desktop("Comment")
doc.add_value(XapianValues.SUMMARY, s)
elif parser.has_option_desktop("GenericName"):
s = parser.get_desktop("GenericName")
if s != name:
doc.add_value(XapianValues.SUMMARY, s)
elif pkgname in cache and cache[pkgname].candidate:
s = cache[pkgname].candidate.summary
doc.add_value(XapianValues.SUMMARY, s)
return doc
def make_doc_from_parser(parser, cache):
# XXX 2012-01-19 michaeln I'm just pulling this code out from
# index_app_info_from_parser, but it'd be great to further
# refactor it - it looks quite scary :-)
doc = xapian.Document()
# app name is the data
if parser.has_option_desktop("X-Ubuntu-Software-Center-Name"):
name = parser.get_desktop("X-Ubuntu-Software-Center-Name")
untranslated_name = parser.get_desktop("X-Ubuntu-Software-Center-Name",
translated=False)
elif parser.has_option_desktop("X-GNOME-FullName"):
name = parser.get_desktop("X-GNOME-FullName")
untranslated_name = parser.get_desktop("X-GNOME-FullName",
translated=False)
else:
name = parser.get_desktop("Name")
untranslated_name = parser.get_desktop("Name", translated=False)
doc.set_data(name)
doc.add_value(XapianValues.APPNAME_UNTRANSLATED, untranslated_name)
# check if we should ignore this file
if parser.has_option_desktop("X-AppInstall-Ignore"):
ignore = parser.get_desktop("X-AppInstall-Ignore")
if ignore.strip().lower() == "true":
LOG.debug("X-AppInstall-Ignore found for '%s'" % parser.desktopf)
return
# architecture
pkgname_extension = ''
if parser.has_option_desktop("X-AppInstall-Architectures"):
arches = parser.get_desktop("X-AppInstall-Architectures")
doc.add_value(XapianValues.ARCHIVE_ARCH, arches)
native_archs = get_current_arch() in arches.split(',')
foreign_archs = list(set(arches.split(',')) &
set(get_foreign_architectures()))
if not (native_archs or foreign_archs):
return
if not native_archs and foreign_archs:
pkgname_extension = ':' + foreign_archs[0]
# package name
pkgname = parser.get_desktop("X-AppInstall-Package") + pkgname_extension
doc.add_term("AP" + pkgname)
if '-' in pkgname:
# we need this to work around xapian oddness
doc.add_term(pkgname.replace('-', '_'))
doc.add_value(XapianValues.PKGNAME, pkgname)
doc.add_value(XapianValues.DESKTOP_FILE, parser.desktopf)
# display name
if "display_name" in axi_values:
doc.add_value(axi_values["display_name"], name)
# cataloged_times
if "catalogedtime" in axi_values:
if pkgname in cataloged_times:
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(cataloged_times[pkgname]))
# pocket (main, restricted, ...)
if parser.has_option_desktop("X-AppInstall-Section"):
archive_section = parser.get_desktop("X-AppInstall-Section")
doc.add_term("AS" + archive_section)
doc.add_value(XapianValues.ARCHIVE_SECTION, archive_section)
# section (mail, base, ..)
if pkgname in cache and cache[pkgname].candidate:
section = cache[pkgname].section
doc.add_term("AE" + section)
# channel (third party stuff)
if parser.has_option_desktop("X-AppInstall-Channel"):
archive_channel = parser.get_desktop("X-AppInstall-Channel")
doc.add_term("AH" + archive_channel)
doc.add_value(XapianValues.ARCHIVE_CHANNEL, archive_channel)
# signing key (third party)
if parser.has_option_desktop("X-AppInstall-Signing-Key-Id"):
keyid = parser.get_desktop("X-AppInstall-Signing-Key-Id")
doc.add_value(XapianValues.ARCHIVE_SIGNING_KEY_ID, keyid)
# license (third party)
if parser.has_option_desktop("X-AppInstall-License"):
license = parser.get_desktop("X-AppInstall-License")
doc.add_value(XapianValues.LICENSE, license)
# date published
if parser.has_option_desktop("X-AppInstall-Date-Published"):
date_published = parser.get_desktop("X-AppInstall-Date-Published")
if (date_published and
re.match("\d+-\d+-\d+ \d+:\d+:\d+", date_published)):
# strip the subseconds from the end of the published date string
date_published = str(date_published).split(".")[0]
doc.add_value(XapianValues.DATE_PUBLISHED,
date_published)
# we use the date published value for the cataloged time as well
if "catalogedtime" in axi_values:
LOG.debug(
("pkgname: %s, date_published cataloged time is: %s" %
(pkgname, parser.get_desktop("date_published"))))
date_published_sec = time.mktime(
time.strptime(date_published,
"%Y-%m-%d %H:%M:%S"))
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(date_published_sec))
# purchased date
if parser.has_option_desktop("X-AppInstall-Purchased-Date"):
date = parser.get_desktop("X-AppInstall-Purchased-Date")
# strip the subseconds from the end of the date string
doc.add_value(XapianValues.PURCHASED_DATE, str(date).split(".")[0])
# deb-line (third party)
if parser.has_option_desktop("X-AppInstall-Deb-Line"):
debline = parser.get_desktop("X-AppInstall-Deb-Line")
doc.add_value(XapianValues.ARCHIVE_DEB_LINE, debline)
# license key (third party)
if parser.has_option_desktop("X-AppInstall-License-Key"):
key = parser.get_desktop("X-AppInstall-License-Key")
doc.add_value(XapianValues.LICENSE_KEY, key)
# license keypath (third party)
if parser.has_option_desktop("X-AppInstall-License-Key-Path"):
path = parser.get_desktop("X-AppInstall-License-Key-Path")
doc.add_value(XapianValues.LICENSE_KEY_PATH, path)
# PPA (third party stuff)
if parser.has_option_desktop("X-AppInstall-PPA"):
archive_ppa = parser.get_desktop("X-AppInstall-PPA")
if archive_ppa:
doc.add_value(XapianValues.ARCHIVE_PPA, archive_ppa)
# add archive origin data here so that its available even if
# the PPA is not (yet) enabled
doc.add_term("XOO" + "lp-ppa-%s" % archive_ppa.replace("/", "-"))
# screenshot (for third party)
if parser.has_option_desktop("X-AppInstall-Screenshot-Url"):
url = parser.get_desktop("X-AppInstall-Screenshot-Url")
doc.add_value(XapianValues.SCREENSHOT_URLS, url)
# thumbnail (for third party)
if parser.has_option_desktop("X-AppInstall-Thumbnail-Url"):
url = parser.get_desktop("X-AppInstall-Thumbnail-Url")
doc.add_value(XapianValues.THUMBNAIL_URL, url)
# video support (for third party mostly)
if parser.has_option_desktop("X-AppInstall-Video-Url"):
url = parser.get_desktop("X-AppInstall-Video-Url")
doc.add_value(XapianValues.VIDEO_URL, url)
# icon (for third party)
if parser.has_option_desktop("X-AppInstall-Icon-Url"):
url = parser.get_desktop("X-AppInstall-Icon-Url")
doc.add_value(XapianValues.ICON_URL, url)
if not parser.has_option_desktop("X-AppInstall-Icon"):
# prefix pkgname to avoid name clashes
doc.add_value(XapianValues.ICON, "%s-icon-%s" % (
pkgname, os.path.basename(url)))
# price (pay stuff)
if parser.has_option_desktop("X-AppInstall-Price"):
price = parser.get_desktop("X-AppInstall-Price")
doc.add_value(XapianValues.PRICE, price)
# since this is a commercial app, indicate it in the component value
doc.add_value(XapianValues.ARCHIVE_SECTION, "commercial")
# support url (mainly pay stuff)
if parser.has_option_desktop("X-AppInstall-Support-Url"):
url = parser.get_desktop("X-AppInstall-Support-Url")
doc.add_value(XapianValues.SUPPORT_SITE_URL, url)
# icon
if parser.has_option_desktop("Icon"):
icon = parser.get_desktop("Icon")
doc.add_value(XapianValues.ICON, icon)
# write out categories
for cat in parser.get_desktop_categories():
doc.add_term("AC" + cat.lower())
categories_string = ";".join(parser.get_desktop_categories())
doc.add_value(XapianValues.CATEGORIES, categories_string)
for mime in parser.get_desktop_mimetypes():
doc.add_term("AM" + mime.lower())
# get type (to distinguish between apps and packages
if parser.has_option_desktop("Type"):
type = parser.get_desktop("Type")
doc.add_term("AT" + type.lower())
# check gettext domain
if parser.has_option_desktop("X-Ubuntu-Gettext-Domain"):
domain = parser.get_desktop("X-Ubuntu-Gettext-Domain")
doc.add_value(XapianValues.GETTEXT_DOMAIN, domain)
# Description (software-center extension)
if parser.has_option_desktop("X-AppInstall-Description"):
descr = parser.get_desktop("X-AppInstall-Description")
doc.add_value(XapianValues.SC_DESCRIPTION, descr)
if parser.has_option_desktop("Supported-Distros"):
doc.add_value(XapianValues.SC_SUPPORTED_DISTROS,
json.dumps(parser.get_desktop("Supported-Distros")))
# version support (for e.g. the scagent)
if parser.has_option_desktop("X-AppInstall-Version"):
ver = parser.get_desktop("X-AppInstall-Version")
doc.add_value(XapianValues.VERSION_INFO, ver)
# (deb)tags (in addition to the pkgname debtags
if parser.has_option_desktop("X-AppInstall-Tags"):
# register tags
tags_string = parser.get_desktop("X-AppInstall-Tags")
if tags_string:
tags = [tag.strip().lower() for tag in tags_string.split(",")]
for tag in tags:
doc.add_term("XT" + tag)
region = get_region_cached()
if region:
# ENFORCE region blacklist/whitelist by not registering
# the app at all
countrycode = region["countrycode"].lower()
if "%s%s" % (REGION_BLACKLIST_TAG, countrycode) in tags:
LOG.info("skipping region restricted app: '%s'"
" (blacklisted) " % name)
return
# whitelist
for tag in tags:
if (tag.startswith(REGION_WHITELIST_TAG) and not
"%s%s" % (REGION_WHITELIST_TAG, countrycode) in tag):
LOG.info("skipping region restricted app: '%s'"
" (not whitelisted)" % name)
return
# popcon
# FIXME: popularity not only based on popcon but also
# on archive section, third party app etc
if parser.has_option_desktop("X-AppInstall-Popcon"):
popcon = float(parser.get_desktop("X-AppInstall-Popcon"))
# sort_by_value uses string compare, so we need to pad here
doc.add_value(XapianValues.POPCON,
xapian.sortable_serialise(popcon))
global popcon_max
popcon_max = max(popcon_max, popcon)
# comment goes into the summary data if there is one,
# other wise we try GenericName and if nothing else,
# the summary of the package
if parser.has_option_desktop("Comment"):
s = parser.get_desktop("Comment")
doc.add_value(XapianValues.SUMMARY, s)
elif parser.has_option_desktop("GenericName"):
s = parser.get_desktop("GenericName")
if s != name:
doc.add_value(XapianValues.SUMMARY, s)
elif pkgname in cache and cache[pkgname].candidate:
s = cache[pkgname].candidate.summary
doc.add_value(XapianValues.SUMMARY, s)
return doc
def index_app_info_from_parser(parser, db, cache):
term_generator = xapian.TermGenerator()
term_generator.set_database(db)
try:
# this tests if we have spelling suggestions (there must be
# a better way?!?) - this is needed as inmemory does not have
# spelling corrections, but it allows setting the flag and will
# raise a exception much later
db.add_spelling("test")
db.remove_spelling("test")
# this enables the flag for it (we only reach this line if
# the db supports spelling suggestions)
term_generator.set_flags(xapian.TermGenerator.FLAG_SPELLING)
except xapian.UnimplementedError:
pass
doc = xapian.Document()
term_generator.set_document(doc)
# app name is the data
if parser.has_option_desktop("X-Ubuntu-Software-Center-Name"):
name = parser.get_desktop("X-Ubuntu-Software-Center-Name")
untranslated_name = parser.get_desktop("X-Ubuntu-Software-Center-Name",
translated=False)
elif parser.has_option_desktop("X-GNOME-FullName"):
name = parser.get_desktop("X-GNOME-FullName")
untranslated_name = parser.get_desktop("X-GNOME-FullName",
translated=False)
else:
name = parser.get_desktop("Name")
untranslated_name = parser.get_desktop("Name", translated=False)
if name in seen:
LOG.debug("duplicated name '%s' (%s)" % (name, parser.desktopf))
LOG.debug("indexing app '%s'" % name)
seen.add(name)
doc.set_data(name)
index_name(doc, name, term_generator)
doc.add_value(XapianValues.APPNAME_UNTRANSLATED, untranslated_name)
# check if we should ignore this file
if parser.has_option_desktop("X-AppInstall-Ignore"):
ignore = parser.get_desktop("X-AppInstall-Ignore")
if ignore.strip().lower() == "true":
LOG.debug("X-AppInstall-Ignore found for '%s'" % parser.desktopf)
return
# architecture
pkgname_extension = ''
if parser.has_option_desktop("X-AppInstall-Architectures"):
arches = parser.get_desktop("X-AppInstall-Architectures")
doc.add_value(XapianValues.ARCHIVE_ARCH, arches)
native_archs = get_current_arch() in arches.split(',')
foreign_archs = list(
set(arches.split(',')) & set(get_foreign_architectures()))
if not (native_archs or foreign_archs): return
if not native_archs and foreign_archs:
pkgname_extension = ':' + foreign_archs[0]
# package name
pkgname = parser.get_desktop("X-AppInstall-Package") + pkgname_extension
doc.add_term("AP" + pkgname)
if '-' in pkgname:
# we need this to work around xapian oddness
doc.add_term(pkgname.replace('-', '_'))
doc.add_value(XapianValues.PKGNAME, pkgname)
doc.add_value(XapianValues.DESKTOP_FILE, parser.desktopf)
# display name
if "display_name" in axi_values:
doc.add_value(axi_values["display_name"], name)
# cataloged_times
if "catalogedtime" in axi_values:
if pkgname in cataloged_times:
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(cataloged_times[pkgname]))
else:
# also catalog apps not found in axi (e.g. for-purchase apps)
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(time.time()))
# pocket (main, restricted, ...)
if parser.has_option_desktop("X-AppInstall-Section"):
archive_section = parser.get_desktop("X-AppInstall-Section")
doc.add_term("AS" + archive_section)
doc.add_value(XapianValues.ARCHIVE_SECTION, archive_section)
# section (mail, base, ..)
if pkgname in cache and cache[pkgname].candidate:
section = cache[pkgname].section
doc.add_term("AE" + section)
# channel (third party stuff)
if parser.has_option_desktop("X-AppInstall-Channel"):
archive_channel = parser.get_desktop("X-AppInstall-Channel")
doc.add_term("AH" + archive_channel)
doc.add_value(XapianValues.ARCHIVE_CHANNEL, archive_channel)
# signing key (third party)
if parser.has_option_desktop("X-AppInstall-Signing-Key-Id"):
keyid = parser.get_desktop("X-AppInstall-Signing-Key-Id")
doc.add_value(XapianValues.ARCHIVE_SIGNING_KEY_ID, keyid)
# license (third party)
if parser.has_option_desktop("X-AppInstall-License"):
license = parser.get_desktop("X-AppInstall-License")
doc.add_value(XapianValues.LICENSE, license)
# purchased date
if parser.has_option_desktop("X-AppInstall-Purchased-Date"):
date = parser.get_desktop("X-AppInstall-Purchased-Date")
# strip the subseconds from the end of the date string
doc.add_value(XapianValues.PURCHASED_DATE, str(date).split(".")[0])
# deb-line (third party)
if parser.has_option_desktop("X-AppInstall-Deb-Line"):
debline = parser.get_desktop("X-AppInstall-Deb-Line")
doc.add_value(XapianValues.ARCHIVE_DEB_LINE, debline)
# license key (third party)
if parser.has_option_desktop("X-AppInstall-License-Key"):
key = parser.get_desktop("X-AppInstall-License-Key")
doc.add_value(XapianValues.LICENSE_KEY, key)
# license keypath (third party)
if parser.has_option_desktop("X-AppInstall-License-Key-Path"):
path = parser.get_desktop("X-AppInstall-License-Key-Path")
doc.add_value(XapianValues.LICENSE_KEY_PATH, path)
# PPA (third party stuff)
if parser.has_option_desktop("X-AppInstall-PPA"):
archive_ppa = parser.get_desktop("X-AppInstall-PPA")
doc.add_value(XapianValues.ARCHIVE_PPA, archive_ppa)
# add archive origin data here so that its available even if
# the PPA is not (yet) enabled
doc.add_term("XOO" + "lp-ppa-%s" % archive_ppa.replace("/", "-"))
# screenshot (for third party)
if parser.has_option_desktop("X-AppInstall-Screenshot-Url"):
url = parser.get_desktop("X-AppInstall-Screenshot-Url")
doc.add_value(XapianValues.SCREENSHOT_URL, url)
# thumbnail (for third party)
if parser.has_option_desktop("X-AppInstall-Thumbnail-Url"):
url = parser.get_desktop("X-AppInstall-Thumbnail-Url")
doc.add_value(XapianValues.THUMBNAIL_URL, url)
# video support (for third party mostly)
if parser.has_option_desktop("X-AppInstall-Video-Url"):
url = parser.get_desktop("X-AppInstall-Video-Url")
doc.add_value(XapianValues.VIDEO_URL, url)
# icon (for third party)
if parser.has_option_desktop("X-AppInstall-Icon-Url"):
url = parser.get_desktop("X-AppInstall-Icon-Url")
doc.add_value(XapianValues.ICON_URL, url)
if not parser.has_option_desktop("X-AppInstall-Icon"):
doc.add_value(XapianValues.ICON, os.path.basename(url))
# price (pay stuff)
if parser.has_option_desktop("X-AppInstall-Price"):
price = parser.get_desktop("X-AppInstall-Price")
doc.add_value(XapianValues.PRICE, price)
# since this is a commercial app, indicate it in the component value
doc.add_value(XapianValues.ARCHIVE_SECTION, "commercial")
# icon
if parser.has_option_desktop("Icon"):
icon = parser.get_desktop("Icon")
doc.add_value(XapianValues.ICON, icon)
# write out categories
for cat in parser.get_desktop_categories():
doc.add_term("AC" + cat.lower())
categories_string = ";".join(parser.get_desktop_categories())
doc.add_value(XapianValues.CATEGORIES, categories_string)
for mime in parser.get_desktop_mimetypes():
doc.add_term("AM" + mime.lower())
# get type (to distinguish between apps and packages
if parser.has_option_desktop("Type"):
type = parser.get_desktop("Type")
doc.add_term("AT" + type.lower())
# check gettext domain
if parser.has_option_desktop("X-Ubuntu-Gettext-Domain"):
domain = parser.get_desktop("X-Ubuntu-Gettext-Domain")
doc.add_value(XapianValues.GETTEXT_DOMAIN, domain)
# Description (software-center extension)
if parser.has_option_desktop("X-AppInstall-Description"):
descr = parser.get_desktop("X-AppInstall-Description")
doc.add_value(XapianValues.SC_DESCRIPTION, descr)
# popcon
# FIXME: popularity not only based on popcon but also
# on archive section, third party app etc
if parser.has_option_desktop("X-AppInstall-Popcon"):
popcon = float(parser.get_desktop("X-AppInstall-Popcon"))
# sort_by_value uses string compare, so we need to pad here
doc.add_value(XapianValues.POPCON, xapian.sortable_serialise(popcon))
global popcon_max
popcon_max = max(popcon_max, popcon)
# comment goes into the summary data if there is one,
# other wise we try GenericName and if nothing else,
# the summary of the package
if parser.has_option_desktop("Comment"):
s = parser.get_desktop("Comment")
doc.add_value(XapianValues.SUMMARY, s)
elif parser.has_option_desktop("GenericName"):
s = parser.get_desktop("GenericName")
if s != name:
doc.add_value(XapianValues.SUMMARY, s)
elif pkgname in cache and cache[pkgname].candidate:
s = cache[pkgname].candidate.summary
doc.add_value(XapianValues.SUMMARY, s)
# add packagename as meta-data too
term_generator.index_text_without_positions(pkgname, WEIGHT_APT_PKGNAME)
# now add search data from the desktop file
for key in ["GenericName", "Comment", "X-AppInstall-Description"]:
if not parser.has_option_desktop(key):
continue
s = parser.get_desktop(key)
# we need the ascii_upper here for e.g. turkish locales, see
# bug #581207
k = "WEIGHT_DESKTOP_" + ascii_upper(key.replace(" ", ""))
if k in globals():
w = globals()[k]
else:
LOG.debug("WEIGHT %s not found" % k)
w = 1
term_generator.index_text_without_positions(s, w)
# add data from the apt cache
if pkgname in cache and cache[pkgname].candidate:
s = cache[pkgname].candidate.summary
term_generator.index_text_without_positions(s, WEIGHT_APT_SUMMARY)
s = cache[pkgname].candidate.description
term_generator.index_text_without_positions(s, WEIGHT_APT_DESCRIPTION)
for origin in cache[pkgname].candidate.origins:
doc.add_term("XOA" + origin.archive)
doc.add_term("XOC" + origin.component)
doc.add_term("XOL" + origin.label)
doc.add_term("XOO" + origin.origin)
doc.add_term("XOS" + origin.site)
# add our keywords (with high priority)
if parser.has_option_desktop("X-AppInstall-Keywords"):
keywords = parser.get_desktop("X-AppInstall-Keywords")
for s in keywords.split(";"):
if s:
term_generator.index_text_without_positions(
s, WEIGHT_DESKTOP_KEYWORD)
# now add it
db.add_document(doc)
def index_app_info_from_parser(parser, db, cache):
term_generator = xapian.TermGenerator()
term_generator.set_database(db)
try:
# this tests if we have spelling suggestions (there must be
# a better way?!?) - this is needed as inmemory does not have
# spelling corrections, but it allows setting the flag and will
# raise a exception much later
db.add_spelling("test")
db.remove_spelling("test")
# this enables the flag for it (we only reach this line if
# the db supports spelling suggestions)
term_generator.set_flags(xapian.TermGenerator.FLAG_SPELLING)
except xapian.UnimplementedError:
pass
doc = xapian.Document()
term_generator.set_document(doc)
# app name is the data
if parser.has_option_desktop("X-Ubuntu-Software-Center-Name"):
name = parser.get_desktop("X-Ubuntu-Software-Center-Name")
untranslated_name = parser.get_desktop("X-Ubuntu-Software-Center-Name", translated=False)
elif parser.has_option_desktop("X-GNOME-FullName"):
name = parser.get_desktop("X-GNOME-FullName")
untranslated_name = parser.get_desktop("X-GNOME-FullName", translated=False)
else:
name = parser.get_desktop("Name")
untranslated_name = parser.get_desktop("Name", translated=False)
if name in seen:
LOG.debug("duplicated name '%s' (%s)" % (name, parser.desktopf))
LOG.debug("indexing app '%s'" % name)
seen.add(name)
doc.set_data(name)
index_name(doc, name, term_generator)
doc.add_value(XapianValues.APPNAME_UNTRANSLATED, untranslated_name)
# check if we should ignore this file
if parser.has_option_desktop("X-AppInstall-Ignore"):
ignore = parser.get_desktop("X-AppInstall-Ignore")
if ignore.strip().lower() == "true":
LOG.debug("X-AppInstall-Ignore found for '%s'" % parser.desktopf)
return
# architecture
pkgname_extension = ''
if parser.has_option_desktop("X-AppInstall-Architectures"):
arches = parser.get_desktop("X-AppInstall-Architectures")
doc.add_value(XapianValues.ARCHIVE_ARCH, arches)
native_archs = get_current_arch() in arches.split(',')
foreign_archs = list(set(arches.split(',')) & set(get_foreign_architectures()))
if not (native_archs or foreign_archs): return
if not native_archs and foreign_archs:
pkgname_extension = ':' + foreign_archs[0]
# package name
pkgname = parser.get_desktop("X-AppInstall-Package") + pkgname_extension
doc.add_term("AP"+pkgname)
if '-' in pkgname:
# we need this to work around xapian oddness
doc.add_term(pkgname.replace('-','_'))
doc.add_value(XapianValues.PKGNAME, pkgname)
doc.add_value(XapianValues.DESKTOP_FILE, parser.desktopf)
# display name
if "display_name" in axi_values:
doc.add_value(axi_values["display_name"], name)
# cataloged_times
if "catalogedtime" in axi_values:
if pkgname in cataloged_times:
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(cataloged_times[pkgname]))
else:
# also catalog apps not found in axi (e.g. for-purchase apps)
doc.add_value(axi_values["catalogedtime"],
xapian.sortable_serialise(time.time()))
# pocket (main, restricted, ...)
if parser.has_option_desktop("X-AppInstall-Section"):
archive_section = parser.get_desktop("X-AppInstall-Section")
doc.add_term("AS"+archive_section)
doc.add_value(XapianValues.ARCHIVE_SECTION, archive_section)
# section (mail, base, ..)
if pkgname in cache and cache[pkgname].candidate:
section = cache[pkgname].section
doc.add_term("AE"+section)
# channel (third party stuff)
if parser.has_option_desktop("X-AppInstall-Channel"):
archive_channel = parser.get_desktop("X-AppInstall-Channel")
doc.add_term("AH"+archive_channel)
doc.add_value(XapianValues.ARCHIVE_CHANNEL, archive_channel)
# signing key (third party)
if parser.has_option_desktop("X-AppInstall-Signing-Key-Id"):
keyid = parser.get_desktop("X-AppInstall-Signing-Key-Id")
doc.add_value(XapianValues.ARCHIVE_SIGNING_KEY_ID, keyid)
# license (third party)
if parser.has_option_desktop("X-AppInstall-License"):
license = parser.get_desktop("X-AppInstall-License")
doc.add_value(XapianValues.LICENSE, license)
# purchased date
if parser.has_option_desktop("X-AppInstall-Purchased-Date"):
date = parser.get_desktop("X-AppInstall-Purchased-Date")
# strip the subseconds from the end of the date string
doc.add_value(XapianValues.PURCHASED_DATE, str(date).split(".")[0])
# deb-line (third party)
if parser.has_option_desktop("X-AppInstall-Deb-Line"):
debline = parser.get_desktop("X-AppInstall-Deb-Line")
doc.add_value(XapianValues.ARCHIVE_DEB_LINE, debline)
# license key (third party)
if parser.has_option_desktop("X-AppInstall-License-Key"):
key = parser.get_desktop("X-AppInstall-License-Key")
doc.add_value(XapianValues.LICENSE_KEY, key)
# license keypath (third party)
if parser.has_option_desktop("X-AppInstall-License-Key-Path"):
path = parser.get_desktop("X-AppInstall-License-Key-Path")
doc.add_value(XapianValues.LICENSE_KEY_PATH, path)
# PPA (third party stuff)
if parser.has_option_desktop("X-AppInstall-PPA"):
archive_ppa = parser.get_desktop("X-AppInstall-PPA")
doc.add_value(XapianValues.ARCHIVE_PPA, archive_ppa)
# add archive origin data here so that its available even if
# the PPA is not (yet) enabled
doc.add_term("XOO"+"lp-ppa-%s" % archive_ppa.replace("/", "-"))
# screenshot (for third party)
if parser.has_option_desktop("X-AppInstall-Screenshot-Url"):
url = parser.get_desktop("X-AppInstall-Screenshot-Url")
doc.add_value(XapianValues.SCREENSHOT_URL, url)
# thumbnail (for third party)
if parser.has_option_desktop("X-AppInstall-Thumbnail-Url"):
url = parser.get_desktop("X-AppInstall-Thumbnail-Url")
doc.add_value(XapianValues.THUMBNAIL_URL, url)
# video support (for third party mostly)
if parser.has_option_desktop("X-AppInstall-Video-Url"):
url = parser.get_desktop("X-AppInstall-Video-Url")
doc.add_value(XapianValues.VIDEO_URL, url)
# icon (for third party)
if parser.has_option_desktop("X-AppInstall-Icon-Url"):
url = parser.get_desktop("X-AppInstall-Icon-Url")
doc.add_value(XapianValues.ICON_URL, url)
if not parser.has_option_desktop("X-AppInstall-Icon"):
doc.add_value(XapianValues.ICON, os.path.basename(url))
# price (pay stuff)
if parser.has_option_desktop("X-AppInstall-Price"):
price = parser.get_desktop("X-AppInstall-Price")
doc.add_value(XapianValues.PRICE, price)
# since this is a commercial app, indicate it in the component value
doc.add_value(XapianValues.ARCHIVE_SECTION, "commercial")
# icon
if parser.has_option_desktop("Icon"):
icon = parser.get_desktop("Icon")
doc.add_value(XapianValues.ICON, icon)
# write out categories
for cat in parser.get_desktop_categories():
doc.add_term("AC"+cat.lower())
categories_string = ";".join(parser.get_desktop_categories())
doc.add_value(XapianValues.CATEGORIES, categories_string)
for mime in parser.get_desktop_mimetypes():
doc.add_term("AM"+mime.lower())
# get type (to distinguish between apps and packages
if parser.has_option_desktop("Type"):
type = parser.get_desktop("Type")
doc.add_term("AT"+type.lower())
# check gettext domain
if parser.has_option_desktop("X-Ubuntu-Gettext-Domain"):
domain = parser.get_desktop("X-Ubuntu-Gettext-Domain")
doc.add_value(XapianValues.GETTEXT_DOMAIN, domain)
# Description (software-center extension)
if parser.has_option_desktop("X-AppInstall-Description"):
descr = parser.get_desktop("X-AppInstall-Description")
doc.add_value(XapianValues.SC_DESCRIPTION, descr)
# popcon
# FIXME: popularity not only based on popcon but also
# on archive section, third party app etc
if parser.has_option_desktop("X-AppInstall-Popcon"):
popcon = float(parser.get_desktop("X-AppInstall-Popcon"))
# sort_by_value uses string compare, so we need to pad here
doc.add_value(XapianValues.POPCON,
xapian.sortable_serialise(popcon))
global popcon_max
popcon_max = max(popcon_max, popcon)
# comment goes into the summary data if there is one,
# other wise we try GenericName and if nothing else,
# the summary of the package
if parser.has_option_desktop("Comment"):
s = parser.get_desktop("Comment")
doc.add_value(XapianValues.SUMMARY, s)
elif parser.has_option_desktop("GenericName"):
s = parser.get_desktop("GenericName")
if s != name:
doc.add_value(XapianValues.SUMMARY, s)
elif pkgname in cache and cache[pkgname].candidate:
s = cache[pkgname].candidate.summary
doc.add_value(XapianValues.SUMMARY, s)
# add packagename as meta-data too
term_generator.index_text_without_positions(pkgname, WEIGHT_APT_PKGNAME)
# now add search data from the desktop file
for key in ["GenericName","Comment", "X-AppInstall-Description"]:
if not parser.has_option_desktop(key):
continue
s = parser.get_desktop(key)
# we need the ascii_upper here for e.g. turkish locales, see
# bug #581207
k = "WEIGHT_DESKTOP_" + ascii_upper(key.replace(" ", ""))
if k in globals():
w = globals()[k]
else:
LOG.debug("WEIGHT %s not found" % k)
w = 1
term_generator.index_text_without_positions(s, w)
# add data from the apt cache
if pkgname in cache and cache[pkgname].candidate:
s = cache[pkgname].candidate.summary
term_generator.index_text_without_positions(s, WEIGHT_APT_SUMMARY)
s = cache[pkgname].candidate.description
term_generator.index_text_without_positions(s, WEIGHT_APT_DESCRIPTION)
for origin in cache[pkgname].candidate.origins:
doc.add_term("XOA"+origin.archive)
doc.add_term("XOC"+origin.component)
doc.add_term("XOL"+origin.label)
doc.add_term("XOO"+origin.origin)
doc.add_term("XOS"+origin.site)
# add our keywords (with high priority)
if parser.has_option_desktop("X-AppInstall-Keywords"):
keywords = parser.get_desktop("X-AppInstall-Keywords")
for s in keywords.split(";"):
if s:
term_generator.index_text_without_positions(s, WEIGHT_DESKTOP_KEYWORD)
# now add it
db.add_document(doc)
