class Build: def __init__(self): self.cfg = Config() self.application_ids = [] self.has_valid_content = False self.completed = {} def add_application(self, app): # application is blacklisted blacklisted = False for b in self.cfg.get_id_blacklist(): if fnmatch.fnmatch(app.app_id, b): app.log.write(LoggerItem.INFO, "application is blacklisted") blacklisted = True break if blacklisted: return False # packages that ship .desktop files in /usr/share/applications # *and* /usr/share/applications/kde4 do not need multiple entries if app.app_id in self.application_ids: app.log.write(LoggerItem.INFO, "duplicate ID in package %s" % app.pkgnames[0]) return False self.application_ids.append(app.app_id) # add AppData if not app.add_appdata_file() and app.requires_appdata: app.log.write(LoggerItem.INFO, "%s requires AppData" % app.app_id_full) return False # use the homepage to filter out same more generic apps homepage_url = None if app.urls.has_key('homepage'): homepage_url = app.urls['homepage'] if homepage_url and not app.project_group: # GNOME project_urls = [ 'http*://*.gnome.org*', 'http://gnome-*.sourceforge.net/'] for m in project_urls: if fnmatch.fnmatch(homepage_url, m): app.project_group = u'GNOME' # KDE project_urls = [ 'http*://*.kde.org*', 'http://*kde-apps.org/*' ] for m in project_urls: if fnmatch.fnmatch(homepage_url, m): app.project_group = u'KDE' # XFCE project_urls = [ 'http://*xfce.org*' ] for m in project_urls: if fnmatch.fnmatch(homepage_url, m): app.project_group = u'XFCE' # LXDE project_urls = [ 'http://lxde.org*', 'http://lxde.sourceforge.net/*' ] for m in project_urls: if fnmatch.fnmatch(homepage_url, m): app.project_group = u'LXDE' # MATE project_urls = [ 'http://*mate-desktop.org*' ] for m in project_urls: if fnmatch.fnmatch(homepage_url, m): app.project_group = u'MATE' # log that we auto-added it if app.project_group: app.log.write(LoggerItem.INFO, "assigned %s" % app.project_group) # Do not include apps without a name if not 'C' in app.names: app.log.write(LoggerItem.INFO, "ignored as no Name") return False # Do not include apps without a summary if not 'C' in app.comments: app.log.write(LoggerItem.INFO, "ignored as no Comment") return False # Do not include apps without an icon if not app.icon: app.log.write(LoggerItem.INFO, "ignored as no Icon") return False # do we have screeshot overrides? extra_screenshots = os.path.join('../screenshots-extra', app.app_id) if os.path.exists(extra_screenshots): app.screenshots = [] overrides = glob.glob(extra_screenshots + "/*.png") app.log.write(LoggerItem.INFO, "adding %i screenshot overrides" % len(overrides)) overrides.sort() for f in overrides: app.add_screenshot_filename(f) # we got something useful if not self.has_valid_content: self.has_valid_content = True return True def add_completed(self, pkg, app): key = pkg.sourcerpm.replace('.src.rpm', '') if key in self.completed: self.completed[key].append(app) else: self.completed[key] = [app] def build(self, filename): # check the package has .desktop files pkg = Package(filename) for b in self.cfg.get_package_blacklist(): if fnmatch.fnmatch(pkg.name, b): pkg.log.write(LoggerItem.INFO, "package is blacklisted") return # set up state if not os.path.exists('./appstream'): os.makedirs('./appstream') if not os.path.exists('./icons'): os.makedirs('./icons') if not os.path.exists('./screenshot-cache'): os.makedirs('./screenshot-cache') if not os.path.exists('./screenshots'): os.makedirs('./screenshots') if not os.path.exists('./screenshots/source'): os.makedirs('./screenshots/source') for size in self.cfg.get_screenshot_thumbnail_sizes(): path = './screenshots/' + str(size[0]) + 'x' + str(size[1]) if not os.path.exists(path): os.makedirs(path) # remove tmp if os.path.exists('./tmp'): shutil.rmtree('./tmp') os.makedirs('./tmp') # decompress main file and search for desktop files package_decompress(pkg) files = [] for f in self.cfg.get_interesting_installed_files(): files.extend(glob.glob("./tmp" + f)) files.sort() # we only need to install additional files if we're not running on # the builders for c in self.cfg.get_package_data_list(): if fnmatch.fnmatch(pkg.name, c[0]): extra_files = glob.glob("./packages/%s*.rpm" % c[1]) for f in extra_files: extra_pkg = Package(f) pkg.log.write(LoggerItem.INFO, "adding extra package %s" % extra_pkg.name) package_decompress(extra_pkg) # check for duplicate apps in the package self.has_valid_content = False # check for codecs if pkg.name.startswith('gstreamer'): app = Codec(pkg, self.cfg) if app.parse_files(files): if self.add_application(app): self.add_completed(pkg, app) else: # process each desktop file in the original package for f in files: pkg.log.write(LoggerItem.INFO, "reading %s" % f) fi = Gio.file_new_for_path(f) info = fi.query_info('standard::content-type', 0, None) # create the right object depending on the content type content_type = info.get_content_type() if content_type == 'inode/symlink': continue if content_type == 'application/x-font-ttf': app = FontFile(pkg, self.cfg) elif content_type == 'application/x-font-otf': app = FontFile(pkg, self.cfg) elif content_type == 'application/x-desktop': app = DesktopFile(pkg, self.cfg) elif content_type == 'application/xml': app = InputMethodComponent(pkg, self.cfg) elif content_type == 'application/x-sqlite3': app = InputMethodTable(pkg, self.cfg) else: pkg.log.write(LoggerItem.INFO, "content type %s not supported" % content_type) continue # the ID is the filename app_id = os.path.basename(f).decode('utf-8') app.set_id(app_id) # parse file if not app.parse_file(f): continue # write the application if self.add_application(app): self.add_completed(pkg, app) def write_appstream(self): valid_apps = [] for key in self.completed: log = LoggerItem(key) valid_apps = self.completed[key] # group fonts of the same family fltr = FontFileFilter() valid_apps = fltr.merge(valid_apps) # create AppStream XML and icons filename = './appstream/' + key + '.xml' filename_icons = "./appstream/%s-icons.tar" % key root = ET.Element("applications") root.set("version", "0.1") for app in valid_apps: try: app.build_xml(root) #app.write_status() except UnicodeEncodeError, e: log.write(LoggerItem.WARNING, "Failed to build %s: %s" % (app.app_id, str(e))) continue except TypeError, e: log.write(LoggerItem.WARNING, "Failed to build %s: %s" % (app.app_id, str(e))) continue log.write(LoggerItem.INFO, "writing %s and %s" % (filename, filename_icons)) try: ET.ElementTree(root).write(filename, encoding='UTF-8', xml_declaration=True) except UnicodeDecodeError, e: log.write(LoggerItem.WARNING, "Failed to write %s: %s" % (filename, str(e)))
def update(repos, reponame): # create if we're starting from nothing if not os.path.exists('./packages'): os.makedirs('./packages') # get extra packages needed for some applications cfg = Config() extra_packages = [] for e in cfg.get_package_data_list(): extra_packages.append(e[1]) # find out what we've got already files = glob.glob("./packages/*.rpm") files.sort() existing = {} for f in files: fd = os.open(f, os.O_RDONLY) try: hdr = _ts.hdrFromFdno(fd) except Exception as e: pass else: existing[hdr.name] = f os.close(fd) print "INFO:\t\tFound %i existing packages for %s" % (len(existing), reponame) # setup yum yb = yum.YumBase() yb.preconf.releasever = reponame[1:] yb.doConfigSetup(errorlevel=-1, debuglevel=-1) yb.conf.cache = 0 # reget the metadata every day for repo in yb.repos.listEnabled(): repo.metadata_expire = 60 * 60 * 24 # 24 hours # what is native for this arch basearch = rpmUtils.arch.getBaseArch() if basearch == 'i386': basearch_list = ['i386', 'i486', 'i586', 'i686'] else: basearch_list = [basearch] basearch_list.append('noarch') # find all packages downloaded = {} try: pkgs = yb.pkgSack except yum.Errors.NoMoreMirrorsRepoError as e: print "FAILED:\t\t" + str(e) sys.exit(1) for pkg in _do_newest_filtering(pkgs): # not our repo if pkg.repoid not in repos: continue # not our arch if pkg.arch not in basearch_list: continue # don't download blacklisted packages if pkg.name in cfg.get_package_blacklist(): continue # make sure the metadata exists repo = yb.repos.getRepo(pkg.repoid) # find out if any of the files ship a desktop file interesting_files = [] for instfile in pkg.returnFileEntries(): for match in cfg.get_interesting_installed_files(): if fnmatch.fnmatch(instfile, match): interesting_files.append(instfile) break # don't download packages without desktop files if len(interesting_files) == 0 and pkg.name not in extra_packages: continue # get base name without the slash relativepath = pkg.returnSimple('relativepath') pos = relativepath.rfind('/') if pos != -1: relativepath = relativepath[pos+1:] # is in cache? path = './packages/' + relativepath if os.path.exists(path) and os.path.getsize(path) == int(pkg.returnSimple('packagesize')): print 'INFO:\t\t' + pkg.name + ' already in cache' downloaded[pkg.name] = True else: pkg.localpath = path # download now print 'DOWNLOAD:\t', path repo.getPackage(pkg) # do we have an old version of this? if existing.has_key(pkg.name): print 'DELETE:\t\t', existing[pkg.name] os.remove(existing[pkg.name]) downloaded[pkg.name] = True # have any packages been removed? for i in existing: if not downloaded.has_key(i): print 'DELETE:\t\t' + existing[i] os.remove(existing[i])
class Build: def __init__(self): self.cfg = Config() def build(self, filename): # check the package has .desktop files print 'SOURCE\t', filename pkg = Package(filename) for b in self.cfg.get_package_blacklist(): if fnmatch.fnmatch(pkg.name, b): print 'IGNORE\t', filename, '\t', "package is blacklisted:", pkg.name return # set up state if not os.path.exists('./appstream'): os.makedirs('./appstream') if not os.path.exists('./icons'): os.makedirs('./icons') if not os.path.exists('./screenshot-cache'): os.makedirs('./screenshot-cache') if not os.path.exists('./screenshots'): os.makedirs('./screenshots') if not os.path.exists('./screenshots/source'): os.makedirs('./screenshots/source') for size in self.cfg.get_screenshot_thumbnail_sizes(): path = './screenshots/' + str(size[0]) + 'x' + str(size[1]) if not os.path.exists(path): os.makedirs(path) # remove tmp if os.path.exists('./tmp'): shutil.rmtree('./tmp') os.makedirs('./tmp') # decompress main file and search for desktop files package_decompress(pkg) files = [] for f in self.cfg.get_interesting_installed_files(): files.extend(glob.glob("./tmp" + f)) files.sort() # we only need to install additional files if we're not running on # the builders for c in self.cfg.get_package_data_list(): if fnmatch.fnmatch(pkg.name, c[0]): extra_files = glob.glob("./packages/%s*.rpm" % c[1]) for f in extra_files: extra_pkg = Package(f) print "INFO\tAdding extra package %s for %s" % (extra_pkg.name, pkg.name) package_decompress(extra_pkg) # open the AppStream file for writing xml_output_file = './appstream/' + pkg.name + '.xml' xml = open(xml_output_file, 'w') xml.write("<?xml version=\"1.0\"?>\n") xml.write("<applications version=\"0.1\">\n") # check for duplicate apps in the package application_ids = [] has_valid_content = False # process each desktop file in the original package for f in files: print 'PROCESS\t', f fi = Gio.file_new_for_path(f) info = fi.query_info('standard::content-type', 0, None) # create the right object depending on the content type content_type = info.get_content_type() if content_type == 'inode/symlink': continue if content_type == 'application/x-font-ttf': app = FontFile(pkg, self.cfg) elif content_type == 'application/x-font-otf': app = FontFile(pkg, self.cfg) elif content_type == 'application/x-desktop': app = DesktopFile(pkg, self.cfg) elif content_type == 'application/xml': app = InputMethodComponent(pkg, self.cfg) elif content_type == 'application/x-sqlite3': app = InputMethodTable(pkg, self.cfg) else: print 'IGNORE\t', f, '\t', "content type " + content_type + " not supported" continue # the ID is the filename app.set_id(f.split('/')[-1]) # application is blacklisted blacklisted = False for b in self.cfg.get_id_blacklist(): if fnmatch.fnmatch(app.app_id, b): print 'IGNORE\t', f, '\t', "application is blacklisted:", app.app_id blacklisted = True break if blacklisted: continue # packages that ship .desktop files in /usr/share/applications # *and* /usr/share/applications/kde4 do not need multiple entries if app.app_id in application_ids: print 'IGNORE\t', f, '\t', app.app_id, 'duplicate ID in package' continue application_ids.append(app.app_id) # parse desktop file if not app.parse_file(f): continue # do we have an AppData file? appdata_file = './tmp/usr/share/appdata/' + app.app_id + '.appdata.xml' appdata_extra_file = './appdata-extra/' + app.type_id + '/' + app.app_id + '.appdata.xml' if os.path.exists(appdata_file) and os.path.exists(appdata_extra_file): print 'DELETE\t', appdata_extra_file, 'as upstream AppData file exists' os.remove(appdata_extra_file) # just use the extra file in places of the missing upstream one if os.path.exists(appdata_extra_file): appdata_file = appdata_extra_file # need to extract details if os.path.exists(appdata_file): data = AppData() data.extract(appdata_file) # check AppData file validates if os.path.exists('/usr/bin/appdata-validate'): env = os.environ p = subprocess.Popen(['/usr/bin/appdata-validate', '--relax', appdata_file], cwd='.', env=env, stdout=subprocess.PIPE) p.wait() if p.returncode: for line in p.stdout: line = line.replace('\n', '') print 'WARNING\tAppData did not validate: ' + line # check the id matches if data.get_id() != app.app_id and data.get_id() != app.app_id_full: raise StandardError('The AppData id does not match: ' + app.app_id) # check the licence is okay if data.get_licence() not in self.cfg.get_content_licences(): raise StandardError('The AppData licence is not okay for ' + app.app_id + ': \'' + data.get_licence() + '\'') # if we have an override, use it for all languages tmp = data.get_names() if tmp: app.names = tmp # if we have an override, use it for all languages tmp = data.get_summaries() if tmp: app.comments = tmp # get optional bits tmp = data.get_url() if tmp: app.homepage_url = tmp tmp = data.get_project_group() if tmp: app.project_group = tmp app.descriptions = data.get_descriptions() # get screenshots tmp = data.get_screenshots() for image in tmp: print 'DOWNLOADING\t', image app.add_screenshot_url(image) elif app.requires_appdata: print 'IGNORE\t', f, '\t', app.app_id_full, 'requires AppData to be included' continue # use the homepage to filter out same more generic apps if not app.project_group: # GNOME project_urls = [ 'http*://*.gnome.org*', 'http://gnome-*.sourceforge.net/'] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "GNOME" # KDE project_urls = [ 'http*://*.kde.org*', 'http://*kde-apps.org/*' ] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "KDE" # XFCE project_urls = [ 'http://*xfce.org*' ] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "XFCE" # LXDE project_urls = [ 'http://lxde.org*', 'http://lxde.sourceforge.net/*' ] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "LXDE" # MATE project_urls = [ 'http://*mate-desktop.org*' ] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "MATE" # print that we auto-added it if app.project_group: print 'INFO\t', f, '\t', app.app_id, 'assigned', app.project_group # we got something useful if not has_valid_content: has_valid_content = True # Do not include apps without a name if not 'C' in app.names: print 'IGNORE\t', f, '\t', "no Name" continue # Do not include apps without a summary if not 'C' in app.comments: print 'IGNORE\t', f, '\t', "no Comment" continue # Do not include apps without an icon if not app.icon: print 'IGNORE\t', f, '\t', "Icon unspecified" continue # write content app.write(xml) # create AppStream XML xml.write("</applications>\n") xml.close() if not has_valid_content: os.remove(xml_output_file) # create AppStream icon tar if has_valid_content: output_file = "./appstream/%s-icons.tar" % pkg.name print 'WRITING\t', output_file tar = tarfile.open(output_file, "w") files = glob.glob("./icons/*.png") for f in files: tar.add(f, arcname=f.split('/')[-1]) tar.close() # remove tmp if not os.getenv('APPSTREAM_DEBUG'): shutil.rmtree('./tmp') shutil.rmtree('./icons')