def main():
# check we're not top level
if os.path.exists('./application.py'):
print 'You cannot run these tools from the top level directory'
sys.exit(1)
# remove appstream
if os.path.exists('./appstream'):
shutil.rmtree('./appstream')
if os.path.exists('./icons'):
shutil.rmtree('./icons')
# the status HTML page goes here too
if not os.path.exists('./screenshots'):
os.makedirs('./screenshots')
files_all = glob.glob("./packages/*.rpm")
files = _do_newest_filtering(files_all)
files.sort()
log = LoggerItem()
job = Build()
for f in files:
log.update_key(f)
try:
job.build(f)
except Exception as e:
log.write(LoggerItem.WARNING, str(e))
job.write_appstream()
def write_appstream(self):
valid_apps = []
for key in self.completed:
log = LoggerItem(key)
valid_apps = self.completed[key]
# group fonts of the same family
fltr = FontFileFilter()
valid_apps = fltr.merge(valid_apps)
# create AppStream XML and icons
filename = './appstream/' + key + '.xml'
filename_icons = "./appstream/%s-icons.tar" % key
root = ET.Element("applications")
root.set("version", "0.1")
for app in valid_apps:
try:
app.build_xml(root)
#app.write_status()
except UnicodeEncodeError, e:
log.write(LoggerItem.WARNING,
"Failed to build %s: %s" % (app.app_id, str(e)))
continue
except TypeError, e:
log.write(LoggerItem.WARNING,
"Failed to build %s: %s" % (app.app_id, str(e)))
continue
def __init__(self, pkg, cfg):
self.log = LoggerItem()
self.app_id = None
self.app_id_full = None
self.names = {}
self.categories = []
self.descriptions = {}
self.comments = {}
self.mimetypes = None
self.urls = {}
self.licence = None
self.pkgnames = []
self.metadata = {}
if pkg:
self.licence = pkg.licence
self.pkgnames = [pkg.name]
if pkg.homepage_url:
self.urls[u'homepage'] = pkg.homepage_url
self.icon = None
self.keywords = []
self.cached_icon = False
self.cfg = cfg
self.screenshots = []
self.compulsory_for_desktop = []
self.type_id = None
self.project_group = None
self.requires_appdata = False
self.thumbnail_screenshots = True
def __init__(self):
self.cfg = config.Config()
self.log = LoggerItem()
self.application_ids = {}
# used as a temp location
if os.path.exists('./tmp'):
shutil.rmtree('./tmp')
os.makedirs('./tmp')
if os.path.exists('./' + self.cfg.distro_name + '.xml.gz'):
os.remove('./' + self.cfg.distro_name + '.xml.gz')
if os.path.exists('./' + self.cfg.distro_name + '-icons.tar.gz'):
os.remove('./' + self.cfg.distro_name + '-icons.tar.gz')
class Compose:
def __init__(self):
self.cfg = config.Config()
self.log = LoggerItem()
self.application_ids = {}
# used as a temp location
if os.path.exists('./tmp'):
shutil.rmtree('./tmp')
os.makedirs('./tmp')
if os.path.exists('./' + self.cfg.distro_name + '.xml.gz'):
os.remove('./' + self.cfg.distro_name + '.xml.gz')
if os.path.exists('./' + self.cfg.distro_name + '-icons.tar.gz'):
os.remove('./' + self.cfg.distro_name + '-icons.tar.gz')
def run(self):
# setup the output XML
master_root = ET.Element("applications")
master_root.set("version", "0.1")
master_tree = ET.ElementTree(master_root)
# find any extra appstream files
files = glob.glob("../appstream-extra/*.xml")
for f in files:
tree = ET.parse(f)
root = tree.getroot()
self.log.update_key(os.path.basename(f))
for app in root:
app_id = app.find('id')
if app_id is None:
self.log.write(LoggerItem.WARNING, "appstream id not found")
continue
# add everything
new = ET.SubElement(master_root, 'application')
for elem in app:
new.append(elem)
# check for screenshots in ../screenshots-extra/${id}/*
tmp = Application(None, self.cfg)
tmp.set_id(app_id.text)
self.log.write(LoggerItem.INFO, "adding %s" % tmp.app_id_full)
overrides = glob.glob("../screenshots-extra/%s/*.png" % tmp.app_id)
if len(overrides) > 0:
self.log.write(LoggerItem.INFO,
"adding %i screenshot overrides" % len(overrides))
for ss_fn in overrides:
tmp.add_screenshot_filename(ss_fn)
tmp.build_xml_screenshots(new)
# add the generated appstream files
files = glob.glob("./appstream/*.xml")
files.sort()
recognised_types = ['desktop', 'codec', 'font', 'inputmethod']
for filename in files:
self.log.update_key(filename)
try:
tree = ET.parse(filename)
except ET.ParseError, e:
self.log.write(LoggerItem.WARNING, "XML could not be parsed: %s" % str(e))
continue
root = tree.getroot()
for app in root:
app_id = app.find('id')
# check type is known
app_id_type = app_id.get('type')
if app_id_type not in recognised_types:
self.log.write(LoggerItem.WARNING,
"appstream id type %s not recognised" % app_id_type)
continue
# detect duplicate IDs in the data
if self.application_ids.has_key(app_id):
found = self.application_ids[app_id.text]
self.log.write(LoggerItem.WARNING,
"duplicate ID found in %s and %s" % (filename, found))
continue
# add everything that isn't private
new = ET.SubElement(master_root, 'application')
for elem in app:
if elem.tag.startswith("X-"):
continue
new.append(elem)
# success
self.application_ids[app_id.text] = filename
self.log.write(LoggerItem.INFO, "adding %s" % app_id.text)
# write to compressed file
master = gzip.open('./' + self.cfg.distro_name + '.xml.gz', 'wb')
master_tree.write(master, 'UTF-8')
master.close()
# we have to do this as "tar --concatenate" is broken
files = glob.glob("./appstream/*.tar")
files.sort()
for f in files:
tar = tarfile.open(f, "r")
tar.extractall(path='./tmp')
tar.close()
# create master icons file
tar = tarfile.open('./' + self.cfg.distro_name + '-icons.tar', "w")
files = glob.glob("./tmp/*.png")
for f in files:
tar.add(f, arcname=f.split('/')[-1])
tar.close()
# compress to save a few Mb
f_in = open('./' + self.cfg.distro_name + '-icons.tar', 'rb')
f_out = gzip.open('./' + self.cfg.distro_name + '-icons.tar.gz', 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.remove('./' + self.cfg.distro_name + '-icons.tar')
def main():
log = LoggerItem()
cfg = Config()
# read in AppStream file into several Application objects
f = gzip.open(sys.argv[1], 'rb')
tree = ET.parse(f)
apps = []
for app in tree.getroot():
a = Application(None, cfg)
for elem in app:
if elem.tag == 'id':
a.set_id(elem.text)
a.type_id = elem.get('type')
log.update_key(a.app_id_full)
log.write(LoggerItem.INFO, "parsing")
elif elem.tag == 'name':
if elem.get(XML_LANG):
continue
a.names['C'] = ensure_unicode(elem.text)
elif elem.tag == 'summary':
if elem.get(XML_LANG):
continue
a.comments['C'] = ensure_unicode(elem.text)
elif elem.tag == 'pkgname':
a.pkgnames.append(ensure_unicode(elem.text))
elif elem.tag == 'appcategories':
for elem2 in elem:
a.categories.append(ensure_unicode(elem2.text))
elif elem.tag == 'keywords':
for elem2 in elem:
a.keywords.append(ensure_unicode(elem2.text))
elif elem.tag == 'url':
a.urls[elem.get('type')] = ensure_unicode(elem.text)
elif elem.tag == 'compulsory_for_desktop':
a.compulsory_for_desktop.append(ensure_unicode(elem.text))
elif elem.tag == 'project_group':
a.project_group = ensure_unicode(elem.text)
elif elem.tag == 'description':
description = ''
if len(elem._children):
for elem2 in elem:
description += elem2.text + u' '
else:
description = elem.text
a.descriptions['C'] = ensure_unicode(description)
elif elem.tag == 'screenshots':
if a.type_id == 'font':
continue
for elem2 in elem:
if elem2.tag != 'screenshot':
continue
caption = None
for elem3 in elem2:
if elem3.tag == 'caption':
caption = elem3.text
elif elem3.tag == 'image':
if elem3.get('type') != 'source':
continue
s = Screenshot(a.app_id, None, caption)
s.basename = os.path.basename(elem3.text)
a.screenshots.append(s)
apps.append(a)
f.close()
# build status page
status = open('./screenshots/status.html', 'w')
status.write('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 ' +
'Transitional//EN" ' +
'"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n')
status.write('<html xmlns="http://www.w3.org/1999/xhtml">\n')
status.write('<head>\n')
status.write('<meta http-equiv="Content-Type" content="text/html; ' +
'charset=UTF-8" />\n')
status.write('<title>Application Data Review</title>\n')
status.write('</head>\n')
status.write('<body>\n')
status.write('<h1>Executive summary</h1>\n')
status.write('<ul>\n')
# long descriptions
cnt = 0
total = len(apps)
for app in apps:
if len(app.descriptions) > 0:
cnt += 1
tmp = 100 * cnt / total
status.write("<li>Applications in Fedora with long descriptions: %i (%i%%)</li>" % (cnt, tmp))
# keywords
cnt = 0
total = len(apps)
for app in apps:
if len(app.keywords) > 0:
cnt += 1
tmp = 100 * cnt / total
status.write("<li>Applications in Fedora with keywords: %i (%i%%)</li>" % (cnt, tmp))
# categories
cnt = 0
total = len(apps)
for app in apps:
if len(app.categories) > 0:
cnt += 1
tmp = 100 * cnt / total
status.write("<li>Applications in Fedora with categories: %i (%i%%)</li>" % (cnt, tmp))
# screenshots
cnt = 0
total = len(apps)
for app in apps:
if len(app.screenshots) > 0:
cnt += 1
tmp = 100 * cnt / total
status.write("<li>Applications in Fedora with screenshots: %i (%i%%)</li>" % (cnt, tmp))
# project apps with appdata
for project_group in ['GNOME', 'KDE', 'XFCE']:
cnt = 0
total = 0
for app in apps:
if app.project_group != project_group:
continue
total += 1
if len(app.screenshots) > 0 or len(app.descriptions) > 0:
cnt += 1
tmp = 0
if total > 0:
tmp = 100 * cnt / total
status.write("<li>Applications in %s with AppData: %i (%i%%)</li>" % (project_group, cnt, tmp))
status.write('</ul>\n')
# write applications
status.write('<h1>Applications</h1>\n')
for app in apps:
if app.type_id == 'font':
continue
if app.type_id == 'inputmethod':
continue
if app.type_id == 'codec':
continue
log.update_key(app.app_id_full)
log.write(LoggerItem.INFO, "writing")
try:
status.write(_to_utf8(_to_html(app)))
except AttributeError, e:
log.write(LoggerItem.WARNING, "failed to write %s: %s" % (app, str(e)))
continue
def update():
log = LoggerItem()
# create if we're starting from nothing
if not os.path.exists('./packages'):
os.makedirs('./packages')
# get extra packages needed for some applications
cfg = Config()
extra_packages = []
for e in cfg.get_package_data_list():
extra_packages.append(e[1])
# find out what we've got already
files = glob.glob("./packages/*.rpm")
files.sort()
existing = {}
for f in files:
fd = os.open(f, os.O_RDONLY)
try:
hdr = _ts.hdrFromFdno(fd)
except Exception as e:
pass
else:
existing[hdr.name] = f
os.close(fd)
log.write(LoggerItem.INFO, "found %i existing packages for %s" % (len(existing), cfg.distro_tag))
# setup yum
yb = yum.YumBase()
yb.preconf.releasever = cfg.distro_tag[1:]
yb.doConfigSetup(errorlevel=-1, debuglevel=-1)
yb.conf.cache = 0
# reget the metadata every day
for repo in yb.repos.listEnabled():
repo.metadata_expire = 60 * 60 * 24 # 24 hours
# what is native for this arch
basearch = rpmUtils.arch.getBaseArch()
if basearch == 'i386':
basearch_list = ['i386', 'i486', 'i586', 'i686']
else:
basearch_list = [basearch]
basearch_list.append('noarch')
# ensure all the repos are enabled
for repo_id in cfg.repo_ids:
repo = yb.repos.getRepo(repo_id)
repo.enable()
# find all packages
downloaded = {}
try:
pkgs = yb.pkgSack
except yum.Errors.NoMoreMirrorsRepoError as e:
log.write(LoggerItem.FAILED, str(e))
sys.exit(1)
newest_packages = _do_newest_filtering(pkgs)
newest_packages.sort()
for pkg in newest_packages:
log.update_key(pkg.nvra)
# not our repo
if pkg.repoid not in cfg.repo_ids:
continue
# not our arch
if pkg.arch not in basearch_list:
continue
# don't download blacklisted packages
if pkg.name in cfg.get_package_blacklist():
continue
# make sure the metadata exists
repo = yb.repos.getRepo(pkg.repoid)
# find out if any of the files ship a desktop file
interesting_files = []
for instfile in pkg.returnFileEntries():
for match in cfg.get_interesting_installed_files():
if fnmatch.fnmatch(instfile, match):
interesting_files.append(instfile)
break
# don't download packages without desktop files
if len(interesting_files) == 0 and pkg.name not in extra_packages:
continue
# get base name without the slash
relativepath = pkg.returnSimple('relativepath')
pos = relativepath.rfind('/')
if pos != -1:
relativepath = relativepath[pos+1:]
# is in cache?
path = './packages/' + relativepath
if os.path.exists(path) and os.path.getsize(path) == int(pkg.returnSimple('packagesize')):
log.write(LoggerItem.INFO, "up to date")
downloaded[pkg.name] = True
else:
pkg.localpath = path
# download now
log.update_key(os.path.basename(path))
log.write(LoggerItem.INFO, "downloading")
repo.getPackage(pkg)
# do we have an old version of this?
if existing.has_key(pkg.name):
log.update_key(os.path.basename(existing[pkg.name]))
log.write(LoggerItem.INFO, "deleting")
os.remove(existing[pkg.name])
downloaded[pkg.name] = True
if len(downloaded) == 0:
log.update_key()
log.write(LoggerItem.INFO, "no packages downloaded for %s" % cfg.distro_tag)
return
# have any packages been removed?
log.update_key()
for i in existing:
if not downloaded.has_key(i):
log.write(LoggerItem.INFO, "deleting %s" % existing[i])
os.remove(existing[i])
class Application:
def __init__(self, pkg, cfg):
self.log = LoggerItem()
self.app_id = None
self.app_id_full = None
self.names = {}
self.categories = []
self.descriptions = {}
self.comments = {}
self.mimetypes = None
self.urls = {}
self.licence = None
self.pkgnames = []
self.metadata = {}
if pkg:
self.licence = pkg.licence
self.pkgnames = [pkg.name]
if pkg.homepage_url:
self.urls[u'homepage'] = pkg.homepage_url
self.icon = None
self.keywords = []
self.cached_icon = False
self.cfg = cfg
self.screenshots = []
self.compulsory_for_desktop = []
self.type_id = None
self.project_group = None
self.requires_appdata = False
self.thumbnail_screenshots = True
def __str__(self):
s = {}
s['app-id'] = self.app_id_full
s['type_id'] = self.type_id
s['names'] = self.names
s['comments'] = self.comments
s['categories'] = self.categories
s['descriptions'] = self.descriptions
s['mimetypes'] = self.mimetypes
s['urls'] = self.urls
s['metadata'] = self.metadata
s['licence'] = self.licence
s['pkgnames'] = self.pkgnames
s['icon'] = self.icon
s['keywords'] = self.keywords
s['cached_icon'] = self.cached_icon
s['screenshots'] = self.screenshots
s['compulsory_for_desktop'] = self.compulsory_for_desktop
s['project_group'] = self.project_group
s['requires_appdata'] = self.requires_appdata
return str(s)
def add_appdata_file(self):
# do we have an AppData file?
filename = './tmp/usr/share/appdata/' + self.app_id + '.appdata.xml'
fn_extra = '../appdata-extra/' + self.type_id + '/' + self.app_id + '.appdata.xml'
if os.path.exists(filename) and os.path.exists(fn_extra):
self.log.write(LoggerItem.INFO,
"deleting %s as upstream AppData file exists" % fn_extra)
os.remove(fn_extra)
# just use the extra file in places of the missing upstream one
if os.path.exists(fn_extra):
filename = fn_extra
# need to extract details
if not os.path.exists(filename):
return False
data = AppData()
if not data.extract(filename):
self.log.write(LoggerItem.WARNING,
"AppData file '%s' could not be parsed" % filename)
return False
# check AppData file validates
enable_validation = self.type_id != 'font'
if enable_validation and os.path.exists('/usr/bin/appdata-validate'):
env = os.environ
p = subprocess.Popen(['/usr/bin/appdata-validate',
'--relax', filename],
cwd='.', env=env, stdout=subprocess.PIPE)
p.wait()
if p.returncode:
for line in p.stdout:
line = line.replace('\n', '').decode('utf-8')
self.log.write(LoggerItem.WARNING,
"AppData did not validate: %s" % line)
# check the id matches
if data.get_id() != self.app_id and data.get_id() != self.app_id_full:
self.log.write(LoggerItem.WARNING,
"The AppData id does not match: " + self.app_id)
return False
# check the licence is okay
if data.get_licence() not in self.cfg.get_content_licences():
self.log.write(LoggerItem.WARNING,
"The AppData licence is not okay for " +
self.app_id + ': \'' +
data.get_licence() + '\'')
return False
# if we have an override, use it for all languages
tmp = data.get_names()
if tmp:
self.names = tmp
# if we have an override, use it for all languages
tmp = data.get_summaries()
if tmp:
self.comments = tmp
# get metadata
tmp = data.get_metadata()
if tmp:
# and extra packages we want to add in?
if 'ExtraPackages' in tmp:
for pkg in tmp['ExtraPackages'].split(','):
if pkg not in self.pkgnames:
self.pkgnames.append(pkg)
del tmp['ExtraPackages']
self.metadata.update(tmp)
# get optional bits
tmp = data.get_urls()
if tmp:
for key in tmp:
self.urls[key] = tmp[key]
tmp = data.get_project_group()
if tmp:
self.project_group = tmp
try:
self.descriptions = data.get_descriptions()
except StandardError, e:
self.log.write(LoggerItem.WARNING,
"failed to add description: %s" % str(e))
# get screenshots
tmp = data.get_screenshots()
for image in tmp:
self.log.write(LoggerItem.INFO, "downloading %s" % image)
self.add_screenshot_url(image)
# get compulsory_for_desktop
for c in data.get_compulsory_for_desktop():
if c not in self.compulsory_for_desktop:
self.compulsory_for_desktop.append(c)
return True