class Build:
def __init__(self):
self.cfg = Config()
self.application_ids = []
self.has_valid_content = False
self.completed = {}
def add_application(self, app):
# application is blacklisted
blacklisted = False
for b in self.cfg.get_id_blacklist():
if fnmatch.fnmatch(app.app_id, b):
app.log.write(LoggerItem.INFO, "application is blacklisted")
blacklisted = True
break
if blacklisted:
return False
# packages that ship .desktop files in /usr/share/applications
# *and* /usr/share/applications/kde4 do not need multiple entries
if app.app_id in self.application_ids:
app.log.write(LoggerItem.INFO, "duplicate ID in package %s" % app.pkgnames[0])
return False
self.application_ids.append(app.app_id)
# add AppData
if not app.add_appdata_file() and app.requires_appdata:
app.log.write(LoggerItem.INFO, "%s requires AppData" % app.app_id_full)
return False
# use the homepage to filter out same more generic apps
homepage_url = None
if app.urls.has_key('homepage'):
homepage_url = app.urls['homepage']
if homepage_url and not app.project_group:
# GNOME
project_urls = [ 'http*://*.gnome.org*',
'http://gnome-*.sourceforge.net/']
for m in project_urls:
if fnmatch.fnmatch(homepage_url, m):
app.project_group = u'GNOME'
# KDE
project_urls = [ 'http*://*.kde.org*',
'http://*kde-apps.org/*' ]
for m in project_urls:
if fnmatch.fnmatch(homepage_url, m):
app.project_group = u'KDE'
# XFCE
project_urls = [ 'http://*xfce.org*' ]
for m in project_urls:
if fnmatch.fnmatch(homepage_url, m):
app.project_group = u'XFCE'
# LXDE
project_urls = [ 'http://lxde.org*',
'http://lxde.sourceforge.net/*' ]
for m in project_urls:
if fnmatch.fnmatch(homepage_url, m):
app.project_group = u'LXDE'
# MATE
project_urls = [ 'http://*mate-desktop.org*' ]
for m in project_urls:
if fnmatch.fnmatch(homepage_url, m):
app.project_group = u'MATE'
# log that we auto-added it
if app.project_group:
app.log.write(LoggerItem.INFO, "assigned %s" % app.project_group)
# Do not include apps without a name
if not 'C' in app.names:
app.log.write(LoggerItem.INFO, "ignored as no Name")
return False
# Do not include apps without a summary
if not 'C' in app.comments:
app.log.write(LoggerItem.INFO, "ignored as no Comment")
return False
# Do not include apps without an icon
if not app.icon:
app.log.write(LoggerItem.INFO, "ignored as no Icon")
return False
# do we have screeshot overrides?
extra_screenshots = os.path.join('../screenshots-extra', app.app_id)
if os.path.exists(extra_screenshots):
app.screenshots = []
overrides = glob.glob(extra_screenshots + "/*.png")
app.log.write(LoggerItem.INFO, "adding %i screenshot overrides" % len(overrides))
overrides.sort()
for f in overrides:
app.add_screenshot_filename(f)
# we got something useful
if not self.has_valid_content:
self.has_valid_content = True
return True
def add_completed(self, pkg, app):
key = pkg.sourcerpm.replace('.src.rpm', '')
if key in self.completed:
self.completed[key].append(app)
else:
self.completed[key] = [app]
def build(self, filename):
# check the package has .desktop files
pkg = Package(filename)
for b in self.cfg.get_package_blacklist():
if fnmatch.fnmatch(pkg.name, b):
pkg.log.write(LoggerItem.INFO, "package is blacklisted")
return
# set up state
if not os.path.exists('./appstream'):
os.makedirs('./appstream')
if not os.path.exists('./icons'):
os.makedirs('./icons')
if not os.path.exists('./screenshot-cache'):
os.makedirs('./screenshot-cache')
if not os.path.exists('./screenshots'):
os.makedirs('./screenshots')
if not os.path.exists('./screenshots/source'):
os.makedirs('./screenshots/source')
for size in self.cfg.get_screenshot_thumbnail_sizes():
path = './screenshots/' + str(size[0]) + 'x' + str(size[1])
if not os.path.exists(path):
os.makedirs(path)
# remove tmp
if os.path.exists('./tmp'):
shutil.rmtree('./tmp')
os.makedirs('./tmp')
# decompress main file and search for desktop files
package_decompress(pkg)
files = []
for f in self.cfg.get_interesting_installed_files():
files.extend(glob.glob("./tmp" + f))
files.sort()
# we only need to install additional files if we're not running on
# the builders
for c in self.cfg.get_package_data_list():
if fnmatch.fnmatch(pkg.name, c[0]):
extra_files = glob.glob("./packages/%s*.rpm" % c[1])
for f in extra_files:
extra_pkg = Package(f)
pkg.log.write(LoggerItem.INFO, "adding extra package %s" % extra_pkg.name)
package_decompress(extra_pkg)
# check for duplicate apps in the package
self.has_valid_content = False
# check for codecs
if pkg.name.startswith('gstreamer'):
app = Codec(pkg, self.cfg)
if app.parse_files(files):
if self.add_application(app):
self.add_completed(pkg, app)
else:
# process each desktop file in the original package
for f in files:
pkg.log.write(LoggerItem.INFO, "reading %s" % f)
fi = Gio.file_new_for_path(f)
info = fi.query_info('standard::content-type', 0, None)
# create the right object depending on the content type
content_type = info.get_content_type()
if content_type == 'inode/symlink':
continue
if content_type == 'application/x-font-ttf':
app = FontFile(pkg, self.cfg)
elif content_type == 'application/x-font-otf':
app = FontFile(pkg, self.cfg)
elif content_type == 'application/x-desktop':
app = DesktopFile(pkg, self.cfg)
elif content_type == 'application/xml':
app = InputMethodComponent(pkg, self.cfg)
elif content_type == 'application/x-sqlite3':
app = InputMethodTable(pkg, self.cfg)
else:
pkg.log.write(LoggerItem.INFO, "content type %s not supported" % content_type)
continue
# the ID is the filename
app_id = os.path.basename(f).decode('utf-8')
app.set_id(app_id)
# parse file
if not app.parse_file(f):
continue
# write the application
if self.add_application(app):
self.add_completed(pkg, app)
def write_appstream(self):
valid_apps = []
for key in self.completed:
log = LoggerItem(key)
valid_apps = self.completed[key]
# group fonts of the same family
fltr = FontFileFilter()
valid_apps = fltr.merge(valid_apps)
# create AppStream XML and icons
filename = './appstream/' + key + '.xml'
filename_icons = "./appstream/%s-icons.tar" % key
root = ET.Element("applications")
root.set("version", "0.1")
for app in valid_apps:
try:
app.build_xml(root)
#app.write_status()
except UnicodeEncodeError, e:
log.write(LoggerItem.WARNING,
"Failed to build %s: %s" % (app.app_id, str(e)))
continue
except TypeError, e:
log.write(LoggerItem.WARNING,
"Failed to build %s: %s" % (app.app_id, str(e)))
continue
log.write(LoggerItem.INFO,
"writing %s and %s" % (filename, filename_icons))
try:
ET.ElementTree(root).write(filename,
encoding='UTF-8',
xml_declaration=True)
except UnicodeDecodeError, e:
log.write(LoggerItem.WARNING,
"Failed to write %s: %s" % (filename, str(e)))
def update(repos, reponame):
# create if we're starting from nothing
if not os.path.exists('./packages'):
os.makedirs('./packages')
# get extra packages needed for some applications
cfg = Config()
extra_packages = []
for e in cfg.get_package_data_list():
extra_packages.append(e[1])
# find out what we've got already
files = glob.glob("./packages/*.rpm")
files.sort()
existing = {}
for f in files:
fd = os.open(f, os.O_RDONLY)
try:
hdr = _ts.hdrFromFdno(fd)
except Exception as e:
pass
else:
existing[hdr.name] = f
os.close(fd)
print "INFO:\t\tFound %i existing packages for %s" % (len(existing), reponame)
# setup yum
yb = yum.YumBase()
yb.preconf.releasever = reponame[1:]
yb.doConfigSetup(errorlevel=-1, debuglevel=-1)
yb.conf.cache = 0
# reget the metadata every day
for repo in yb.repos.listEnabled():
repo.metadata_expire = 60 * 60 * 24 # 24 hours
# what is native for this arch
basearch = rpmUtils.arch.getBaseArch()
if basearch == 'i386':
basearch_list = ['i386', 'i486', 'i586', 'i686']
else:
basearch_list = [basearch]
basearch_list.append('noarch')
# find all packages
downloaded = {}
try:
pkgs = yb.pkgSack
except yum.Errors.NoMoreMirrorsRepoError as e:
print "FAILED:\t\t" + str(e)
sys.exit(1)
for pkg in _do_newest_filtering(pkgs):
# not our repo
if pkg.repoid not in repos:
continue
# not our arch
if pkg.arch not in basearch_list:
continue
# don't download blacklisted packages
if pkg.name in cfg.get_package_blacklist():
continue
# make sure the metadata exists
repo = yb.repos.getRepo(pkg.repoid)
# find out if any of the files ship a desktop file
interesting_files = []
for instfile in pkg.returnFileEntries():
for match in cfg.get_interesting_installed_files():
if fnmatch.fnmatch(instfile, match):
interesting_files.append(instfile)
break
# don't download packages without desktop files
if len(interesting_files) == 0 and pkg.name not in extra_packages:
continue
# get base name without the slash
relativepath = pkg.returnSimple('relativepath')
pos = relativepath.rfind('/')
if pos != -1:
relativepath = relativepath[pos+1:]
# is in cache?
path = './packages/' + relativepath
if os.path.exists(path) and os.path.getsize(path) == int(pkg.returnSimple('packagesize')):
print 'INFO:\t\t' + pkg.name + ' already in cache'
downloaded[pkg.name] = True
else:
pkg.localpath = path
# download now
print 'DOWNLOAD:\t', path
repo.getPackage(pkg)
# do we have an old version of this?
if existing.has_key(pkg.name):
print 'DELETE:\t\t', existing[pkg.name]
os.remove(existing[pkg.name])
downloaded[pkg.name] = True
# have any packages been removed?
for i in existing:
if not downloaded.has_key(i):
print 'DELETE:\t\t' + existing[i]
os.remove(existing[i])
class Build:
def __init__(self):
self.cfg = Config()
def build(self, filename):
# check the package has .desktop files
print 'SOURCE\t', filename
pkg = Package(filename)
for b in self.cfg.get_package_blacklist():
if fnmatch.fnmatch(pkg.name, b):
print 'IGNORE\t', filename, '\t', "package is blacklisted:", pkg.name
return
# set up state
if not os.path.exists('./appstream'):
os.makedirs('./appstream')
if not os.path.exists('./icons'):
os.makedirs('./icons')
if not os.path.exists('./screenshot-cache'):
os.makedirs('./screenshot-cache')
if not os.path.exists('./screenshots'):
os.makedirs('./screenshots')
if not os.path.exists('./screenshots/source'):
os.makedirs('./screenshots/source')
for size in self.cfg.get_screenshot_thumbnail_sizes():
path = './screenshots/' + str(size[0]) + 'x' + str(size[1])
if not os.path.exists(path):
os.makedirs(path)
# remove tmp
if os.path.exists('./tmp'):
shutil.rmtree('./tmp')
os.makedirs('./tmp')
# decompress main file and search for desktop files
package_decompress(pkg)
files = []
for f in self.cfg.get_interesting_installed_files():
files.extend(glob.glob("./tmp" + f))
files.sort()
# we only need to install additional files if we're not running on
# the builders
for c in self.cfg.get_package_data_list():
if fnmatch.fnmatch(pkg.name, c[0]):
extra_files = glob.glob("./packages/%s*.rpm" % c[1])
for f in extra_files:
extra_pkg = Package(f)
print "INFO\tAdding extra package %s for %s" % (extra_pkg.name, pkg.name)
package_decompress(extra_pkg)
# open the AppStream file for writing
xml_output_file = './appstream/' + pkg.name + '.xml'
xml = open(xml_output_file, 'w')
xml.write("<?xml version=\"1.0\"?>\n")
xml.write("<applications version=\"0.1\">\n")
# check for duplicate apps in the package
application_ids = []
has_valid_content = False
# process each desktop file in the original package
for f in files:
print 'PROCESS\t', f
fi = Gio.file_new_for_path(f)
info = fi.query_info('standard::content-type', 0, None)
# create the right object depending on the content type
content_type = info.get_content_type()
if content_type == 'inode/symlink':
continue
if content_type == 'application/x-font-ttf':
app = FontFile(pkg, self.cfg)
elif content_type == 'application/x-font-otf':
app = FontFile(pkg, self.cfg)
elif content_type == 'application/x-desktop':
app = DesktopFile(pkg, self.cfg)
elif content_type == 'application/xml':
app = InputMethodComponent(pkg, self.cfg)
elif content_type == 'application/x-sqlite3':
app = InputMethodTable(pkg, self.cfg)
else:
print 'IGNORE\t', f, '\t', "content type " + content_type + " not supported"
continue
# the ID is the filename
app.set_id(f.split('/')[-1])
# application is blacklisted
blacklisted = False
for b in self.cfg.get_id_blacklist():
if fnmatch.fnmatch(app.app_id, b):
print 'IGNORE\t', f, '\t', "application is blacklisted:", app.app_id
blacklisted = True
break
if blacklisted:
continue
# packages that ship .desktop files in /usr/share/applications
# *and* /usr/share/applications/kde4 do not need multiple entries
if app.app_id in application_ids:
print 'IGNORE\t', f, '\t', app.app_id, 'duplicate ID in package'
continue
application_ids.append(app.app_id)
# parse desktop file
if not app.parse_file(f):
continue
# do we have an AppData file?
appdata_file = './tmp/usr/share/appdata/' + app.app_id + '.appdata.xml'
appdata_extra_file = './appdata-extra/' + app.type_id + '/' + app.app_id + '.appdata.xml'
if os.path.exists(appdata_file) and os.path.exists(appdata_extra_file):
print 'DELETE\t', appdata_extra_file, 'as upstream AppData file exists'
os.remove(appdata_extra_file)
# just use the extra file in places of the missing upstream one
if os.path.exists(appdata_extra_file):
appdata_file = appdata_extra_file
# need to extract details
if os.path.exists(appdata_file):
data = AppData()
data.extract(appdata_file)
# check AppData file validates
if os.path.exists('/usr/bin/appdata-validate'):
env = os.environ
p = subprocess.Popen(['/usr/bin/appdata-validate',
'--relax', appdata_file],
cwd='.', env=env, stdout=subprocess.PIPE)
p.wait()
if p.returncode:
for line in p.stdout:
line = line.replace('\n', '')
print 'WARNING\tAppData did not validate: ' + line
# check the id matches
if data.get_id() != app.app_id and data.get_id() != app.app_id_full:
raise StandardError('The AppData id does not match: ' + app.app_id)
# check the licence is okay
if data.get_licence() not in self.cfg.get_content_licences():
raise StandardError('The AppData licence is not okay for ' +
app.app_id + ': \'' +
data.get_licence() + '\'')
# if we have an override, use it for all languages
tmp = data.get_names()
if tmp:
app.names = tmp
# if we have an override, use it for all languages
tmp = data.get_summaries()
if tmp:
app.comments = tmp
# get optional bits
tmp = data.get_url()
if tmp:
app.homepage_url = tmp
tmp = data.get_project_group()
if tmp:
app.project_group = tmp
app.descriptions = data.get_descriptions()
# get screenshots
tmp = data.get_screenshots()
for image in tmp:
print 'DOWNLOADING\t', image
app.add_screenshot_url(image)
elif app.requires_appdata:
print 'IGNORE\t', f, '\t', app.app_id_full, 'requires AppData to be included'
continue
# use the homepage to filter out same more generic apps
if not app.project_group:
# GNOME
project_urls = [ 'http*://*.gnome.org*',
'http://gnome-*.sourceforge.net/']
for m in project_urls:
if fnmatch.fnmatch(app.homepage_url, m):
app.project_group = "GNOME"
# KDE
project_urls = [ 'http*://*.kde.org*',
'http://*kde-apps.org/*' ]
for m in project_urls:
if fnmatch.fnmatch(app.homepage_url, m):
app.project_group = "KDE"
# XFCE
project_urls = [ 'http://*xfce.org*' ]
for m in project_urls:
if fnmatch.fnmatch(app.homepage_url, m):
app.project_group = "XFCE"
# LXDE
project_urls = [ 'http://lxde.org*',
'http://lxde.sourceforge.net/*' ]
for m in project_urls:
if fnmatch.fnmatch(app.homepage_url, m):
app.project_group = "LXDE"
# MATE
project_urls = [ 'http://*mate-desktop.org*' ]
for m in project_urls:
if fnmatch.fnmatch(app.homepage_url, m):
app.project_group = "MATE"
# print that we auto-added it
if app.project_group:
print 'INFO\t', f, '\t', app.app_id, 'assigned', app.project_group
# we got something useful
if not has_valid_content:
has_valid_content = True
# Do not include apps without a name
if not 'C' in app.names:
print 'IGNORE\t', f, '\t', "no Name"
continue
# Do not include apps without a summary
if not 'C' in app.comments:
print 'IGNORE\t', f, '\t', "no Comment"
continue
# Do not include apps without an icon
if not app.icon:
print 'IGNORE\t', f, '\t', "Icon unspecified"
continue
# write content
app.write(xml)
# create AppStream XML
xml.write("</applications>\n")
xml.close()
if not has_valid_content:
os.remove(xml_output_file)
# create AppStream icon tar
if has_valid_content:
output_file = "./appstream/%s-icons.tar" % pkg.name
print 'WRITING\t', output_file
tar = tarfile.open(output_file, "w")
files = glob.glob("./icons/*.png")
for f in files:
tar.add(f, arcname=f.split('/')[-1])
tar.close()
# remove tmp
if not os.getenv('APPSTREAM_DEBUG'):
shutil.rmtree('./tmp')
shutil.rmtree('./icons')
