def __init__(self): # load packages configuration self._packages = BaseConfigFile(CONFIG_PACKAGES) # load cache database self._cache = JsonDatabase() self._cache.load(CACHE_PACKAGES) # set cache if set(self._cache.keys()) != set( ("downstream", "compare", "upstream")): logging.debug("Invalid cache, purging it") self._cache.clear() self._cache["upstream"] = {} self._cache["downstream"] = {} self._cache["compare"] = {}
def __init__(self): # load packages configuration self._packages = BaseConfigFile(CONFIG_PACKAGES) # load cache database self._cache = JsonDatabase() self._cache.load(CACHE_PACKAGES) # set cache if set(self._cache.keys()) != set(("downstream", "compare", "upstream")): logging.debug("Invalid cache, purging it") self._cache.clear() self._cache["upstream"] = {} self._cache["downstream"] = {} self._cache["compare"] = {}
class VersionController(object): ''' Handle version detection of packages ''' def __init__(self): # load packages configuration self._packages = BaseConfigFile(CONFIG_PACKAGES) # load cache database self._cache = JsonDatabase() self._cache.load(CACHE_PACKAGES) # set cache if set(self._cache.keys()) != set(("downstream", "compare", "upstream")): logging.debug("Invalid cache, purging it") self._cache.clear() self._cache["upstream"] = {} self._cache["downstream"] = {} self._cache["compare"] = {} @property def packages(self): '''Return list of packages augmented with aliases''' pkgs = [] for name, data in self._packages.items(): pkgs.append(name) pkgs += self.alias(data) return pkgs @property def versions(self): '''Return upstream versions of a package (use cache)''' ver = OrderedDict() for name, v_upstream, v_downstream in self.compare(): ver[name] = (v_upstream, v_downstream) return ver def select(self, packages): ''' Remove packages not listed in packages from the processing of controller future actions ''' packages = set(packages) for name, data in OrderedDict(self._packages).items(): names = set((name,)) | set(self.alias(data)) if len(packages & names) == 0: self._packages.pop(name, None) def sort(self): ''' Sort packages by name Make packages to be upgraded/displayed by alpha order ''' self._packages = self.sort_dict(self._packages) # do not sort self._cache by recreating the cache object # destructor is used to save the cache content def sync(self): ''' Synchronise local cache with external states Retrieve upstream and downstream versions and store them ''' for name, value in self._packages.items(): try: logging.debug("Syncing versions of package %s" % name) # get upstream version v_upstream = self.get_version_upstream(name, value) # apply eval to upstream e_upstream = value.get("eval_upstream", None) if e_upstream is not None: v_upstream = eval(e_upstream, {}, {"version": v_upstream}) logging.debug("eval_upstream produce version: %s" % v_upstream) # save upstream version if self._cache["upstream"].get(name, {}).get("version", None) != v_upstream: logging.debug("caching upstream version %s" % v_upstream) self._cache["upstream"][name] = {"version": v_upstream, "epoch": int(time())} else: logging.debug("already cached upstream version %s" % v_upstream) # get downstream mode mode = value.get("downstream", None) if mode is None: logging.warning("%s: Invalid downstream mode: %s." % (name, mode)) continue # get downstream version v_downstream = self.get_version_downstream(name, value, mode) # apply eval to downstream e_compare = value.get("eval_downstream", None) if e_compare is not None: v_compare = eval(e_compare, {}, {"version": v_compare}) logging.debug("eval_downstream produce version: %s" % v_downstream) # save downstream version if self._cache["downstream"].get(name, {}).get("version", None) != v_downstream: logging.debug("caching downstream version %s" % v_downstream) self._cache["downstream"][name] = {"version": v_downstream, "epoch": int(time())} else: logging.debug("already cached downstream version %s" % v_downstream) except Exception as exp: logging.error("Sync of %s: %s" % (name, exp)) def compare(self, only_new=False, only_fresh=False): ''' Compare versions according compare mode Return an iterator over all packages and their aliases with upstream and downstream versions. ''' for name, value in self._packages.items(): logging.debug("Comparing versions of package %s" % name) # get upstream in cache v_upstream = self._cache["upstream"].get(name, {}).get("version", None) if v_upstream is None: logging.warning("%s: Upstream version not found in cache" % name) continue # get downstream in cache v_downstream = self._cache["downstream"].get(name, {}).get("version", None) if v_downstream is None: logging.warning("%s: Downstream version not found in cache" % name) continue # only new version mode if only_new and v_upstream == v_downstream: logging.debug("%s: skipped by only new mode" % name) continue # only fresh version mode if only_fresh: last_cmp = self._cache["compare"].get(name, -1) last_up = self._cache["upstream"].get(name, {}).get("epoch", 0) last_down = self._cache["downstream"].get(name, {}).get("epoch", 0) if (last_cmp >= last_up and last_cmp >= last_down): logging.debug("%s: skipped by only fresh mode" % name) continue # save our compare in cache self._cache["compare"][name] = int(time()) # gen main pacakge yield (name, v_upstream, v_downstream) # gen aliases package for alias in self.alias(value): yield (alias, v_upstream, v_downstream) @staticmethod def alias(pkg): '''Return the list of aliases of a package''' return [ al for al in pkg.get("alias", "").split(" ") if al != "" ] @staticmethod def sort_dict(larousse): '''Sort a dictionary into and OrderedDict''' return OrderedDict(sorted(larousse.items(), key=lambda t: t[0])) @staticmethod def get_version_upstream(name, value): '''Fetch upstream version''' logging.debug("Get upstream version") # check upstream param if "url" not in value: logging.error("No url specified for %s" % name) raise InvalidConfigFile("Missing url in config file") url = value["url"] regex = value.get("regex", "%s[-_]v?(%s)%s" % ( value.get("regex_name", name), value.get("regex_version", "[-.\w]+"), value.get("regex_ext", "\.(?:tar(?:\.gz|\.bz2|\.xz)?|tgz|tbz2|zip)"))) # retrieve config timeout timeout = float(value["timeout"]) if "timeout" in value else None # do it retry time + 1 ntry = int(value.get("retry", 0)) + 1 # do the job for n in range(1, ntry + 1): try: logging.debug("Requesting url: %s (try %d/%d)" % (url, n, ntry)) logging.debug("Timeout is %s" % timeout) url_req = Request(url, headers=HTTP_HEADERS) url_fd = urlopen(url_req, timeout=timeout) logging.debug("Version regex: %s" % regex) v = re.findall(regex, url_fd.read().decode("utf-8", "ignore")) if v is None or len(v) == 0: raise VersionNotFound("No regex match on upstream") # remove duplicity v = set(v) # list all found versions logging.debug("Found versions: %s" % v) # exclude versions regex_exclude = value.get("regex_exclude", ".*(rc|beta|alpha|pre).*") if regex_exclude != "": logging.debug("Exclusion regex: %s" % regex_exclude) v -= set(filter(lambda x: re.search(regex_exclude, x), v)) logging.debug("Found versions after exclusion: %s" % v) # latest version is the highest v = max(v, key=VersionKey) # list selected version logging.debug("Upstream version is : %s" % v) return v except Exception as exp: if n == ntry: raise VersionNotFound("Upstream check failed: %s" % exp) assert(False) @staticmethod def get_version_downstream(name, value, mode): '''Return dowstream version''' try: return getattr(VersionController, "get_version_downstream_%s" % mode)(name, value) except AttributeError: raise InvalidConfigFile("Invalid dowstream mode") @staticmethod def get_version_downstream_pacman(name, value): '''Return pacman version''' logging.debug("Get pacman version") # Load pacman pacman = Pacman() # filter if repo is provided allowed_repos = value.get("repo").split(",") if "repo" in value else None # looking into db for package name db, pkg = pacman.find_pkg(name, allowed_repos) if pkg is not None: epoch, pkgver, pkgrel = re.match("^(?:(\d+)\:)?([^-:]*)(?:-(\d+))?", pkg.version).groups() logging.debug("pacman version in %s: %s" % (db.name, pkgver)) return pkgver raise VersionNotFound("No pacman package found") @staticmethod def get_version_downstream_archweb(name, value): '''Return archweb version''' logging.debug("Get archweb version") # if arch is specified archs = value.get("arch", "x86_64,i686,any").split(",") # if archweb repository is specified repos = value.get("repo", "community-testing,community,testing,extra,core" ).split(",") # retrieve config timeout timeout = float(value["timeout"]) if "timeout" in value else None for arch in archs: for repo in repos: url = "http://www.archlinux.org/packages/%s/%s/%s/json" % ( repo, arch, name) url_req = Request(url, headers=HTTP_HEADERS) logging.debug("Requesting url: %s" % url) logging.debug("Timeout is %s" % timeout) try: url_fd = urlopen(url_req, timeout=timeout) d = json.loads(url_fd.read().decode("utf-8", "ignore")) v = d["pkgver"] logging.debug("Archweb version is : %s" % v) return v except Exception as exp: logging.debug("Archweb check failed: %s" % exp) raise VersionNotFound("No Archweb package found") @staticmethod def get_version_downstream_aur(name, value): '''Return archlinux user repository version''' logging.debug("Get AUR version") try: # retrieve config timeout timeout = float(value["timeout"]) if "timeout" in value else None url = "http://aur.archlinux.org/rpc.php?type=info&arg=%s" % name url_req = Request(url, headers=HTTP_HEADERS) logging.debug("Requesting url: %s" % url) logging.debug("Timeout is %s" % timeout) url_fd = urlopen(url_req, timeout=timeout) d = json.loads(url_fd.read().decode("utf-8", "ignore")) if "version" not in d or d["version"] != 1: raise VersionNotFound("Unsupported AUR version") if len(d["results"]) == 0: raise VersionNotFound("No such package") v = d["results"]["Version"].rsplit("-")[0] logging.debug("AUR version is : %s" % v) return v except Exception as exp: raise VersionNotFound("AUR check failed: %s" % exp) assert(False) @staticmethod def get_version_downstream_abs(name, value): '''Return abs version''' logging.debug("Get ABS version") # Get ABS tree path abspath = value.get("abs_path", "/var/abs") # Map db and name repos = [d for d in os.listdir(abspath) if os.path.isdir(os.path.join(abspath, d))] # filter if repo is provided if "repo" in value: allowed_repos = value.get("repo").split(",") for r in list(repos): if r not in allowed_repos: repos.remove(r) # looking into db for package name for repo in repos: logging.debug("Looking into directory %s" % repo) repopath = os.path.join(abspath, repo) packages = [d for d in os.listdir(repopath)] if name in packages: pkgpath = os.path.join(repopath, name, "PKGBUILD") if os.path.isfile(pkgpath): # use bash to export vars. # WARNING: CODE IS EXECUTED pkgdict = parse_pkgbuild(pkgpath) if "pkgver" in pkgdict: v = pkgdict["pkgver"] logging.debug("ABS version is : %s" % v) return v raise VersionNotFound("No ABS package found") @staticmethod def get_version_downstream_none(name, value): '''Return none version''' return "" def print_names(self): '''Print packages name''' for name in self.packages: print(name) @staticmethod def print_modes(): '''Print comparaison modes''' for mode in fnmatch.filter(dir(VersionController), "get_version_downstream_*"): print(mode[23:]) def print_versions(self, only_new=False, only_fresh=False): '''Print versions''' for name, v_upstream, v_downstream in self.compare(only_new, only_fresh): self.print_version(name, v_upstream, v_downstream) def print_version(self, name, v1, v2=None): '''Handle printing of 2 versions''' # define used color c_blue = c_white = c_yellow = c_compare = c_reset = '' if sys.stdout.isatty(): if v2 is None: c_compare = '\033[1;33m' elif v1 == v2: c_compare = '\033[1;32m' else: c_compare = '\033[1;31m' c_blue = '\033[1;34m' c_white = '\033[1;37m' c_yellow = '\033[1;33m' c_reset = '\033[m' # print package name toprint = "%s[%s%s%s]" % (c_blue, c_white, name, c_blue) # print upstream toprint += " %sup: %s " % (c_yellow, v1) # print downstream if v2 is not "": # print separator toprint += "%s|" % c_blue origin = self._packages.get(name,{}).get("downstream", "downstream") toprint += " %s%s: %s" % (c_compare, origin, v2) toprint += c_reset print(toprint)
class VersionController(object): ''' Handle version detection of packages ''' def __init__(self): # load packages configuration self._packages = BaseConfigFile(CONFIG_PACKAGES) # load cache database self._cache = JsonDatabase() self._cache.load(CACHE_PACKAGES) # set cache if set(self._cache.keys()) != set( ("downstream", "compare", "upstream")): logging.debug("Invalid cache, purging it") self._cache.clear() self._cache["upstream"] = {} self._cache["downstream"] = {} self._cache["compare"] = {} @property def packages(self): '''Return list of packages augmented with aliases''' pkgs = [] for name, data in self._packages.items(): pkgs.append(name) pkgs += self.alias(data) return pkgs @property def versions(self): '''Return upstream versions of a package (use cache)''' ver = OrderedDict() for name, v_upstream, v_downstream in self.compare(): ver[name] = (v_upstream, v_downstream) return ver def select(self, packages): ''' Remove packages not listed in packages from the processing of controller future actions ''' packages = set(packages) for name, data in OrderedDict(self._packages).items(): names = set((name, )) | set(self.alias(data)) if len(packages & names) == 0: self._packages.pop(name, None) def sort(self): ''' Sort packages by name Make packages to be upgraded/displayed by alpha order ''' self._packages = self.sort_dict(self._packages) # do not sort self._cache by recreating the cache object # destructor is used to save the cache content def sync(self): ''' Synchronise local cache with external states Retrieve upstream and downstream versions and store them ''' for name, value in self._packages.items(): try: logging.debug("Syncing versions of package %s" % name) # get upstream version v_upstream = self.get_version_upstream(name, value) # apply eval to upstream e_upstream = value.get("eval_upstream", None) if e_upstream is not None: v_upstream = eval(e_upstream, {}, {"version": v_upstream}) logging.debug("eval_upstream produce version: %s" % v_upstream) # save upstream version if self._cache["upstream"].get(name, {}).get( "version", None) != v_upstream: logging.debug("caching upstream version %s" % v_upstream) self._cache["upstream"][name] = { "version": v_upstream, "epoch": int(time()) } else: logging.debug("already cached upstream version %s" % v_upstream) # get downstream mode mode = value.get("downstream", None) if mode is None: logging.warning("%s: Invalid downstream mode: %s." % (name, mode)) continue # get downstream version v_downstream = self.get_version_downstream(name, value, mode) # apply eval to downstream e_compare = value.get("eval_downstream", None) if e_compare is not None: v_compare = eval(e_compare, {}, {"version": v_compare}) logging.debug("eval_downstream produce version: %s" % v_downstream) # save downstream version if self._cache["downstream"].get(name, {}).get( "version", None) != v_downstream: logging.debug("caching downstream version %s" % v_downstream) self._cache["downstream"][name] = { "version": v_downstream, "epoch": int(time()) } else: logging.debug("already cached downstream version %s" % v_downstream) except Exception as exp: logging.error("Sync of %s: %s" % (name, exp)) def compare(self, only_new=False, only_fresh=False): ''' Compare versions according compare mode Return an iterator over all packages and their aliases with upstream and downstream versions. ''' for name, value in self._packages.items(): logging.debug("Comparing versions of package %s" % name) # get upstream in cache v_upstream = self._cache["upstream"].get(name, {}).get("version", None) if v_upstream is None: logging.warning("%s: Upstream version not found in cache" % name) continue # get downstream in cache v_downstream = self._cache["downstream"].get(name, {}).get( "version", None) if v_downstream is None: logging.warning("%s: Downstream version not found in cache" % name) continue # only new version mode if only_new and v_upstream == v_downstream: logging.debug("%s: skipped by only new mode" % name) continue # only fresh version mode if only_fresh: last_cmp = self._cache["compare"].get(name, -1) last_up = self._cache["upstream"].get(name, {}).get("epoch", 0) last_down = self._cache["downstream"].get(name, {}).get("epoch", 0) if (last_cmp >= last_up and last_cmp >= last_down): logging.debug("%s: skipped by only fresh mode" % name) continue # save our compare in cache self._cache["compare"][name] = int(time()) # gen main pacakge yield (name, v_upstream, v_downstream) # gen aliases package for alias in self.alias(value): yield (alias, v_upstream, v_downstream) @staticmethod def alias(pkg): '''Return the list of aliases of a package''' return [al for al in pkg.get("alias", "").split(" ") if al != ""] @staticmethod def sort_dict(larousse): '''Sort a dictionary into and OrderedDict''' return OrderedDict(sorted(larousse.items(), key=lambda t: t[0])) @staticmethod def get_version_upstream(name, value): '''Fetch upstream version''' logging.debug("Get upstream version") # check upstream param if "url" not in value: logging.error("No url specified for %s" % name) raise InvalidConfigFile("Missing url in config file") url = value["url"] regex = value.get( "regex", "%s[-_]v?(%s)%s" % (value.get("regex_name", name), value.get("regex_version", "[-.\w]+"), value.get("regex_ext", "\.(?:tar(?:\.gz|\.bz2|\.xz)?|tgz|tbz2|zip)"))) # retrieve config timeout timeout = float(value["timeout"]) if "timeout" in value else None # do it retry time + 1 ntry = int(value.get("retry", 0)) + 1 # do the job for n in range(1, ntry + 1): try: logging.debug("Requesting url: %s (try %d/%d)" % (url, n, ntry)) logging.debug("Timeout is %s" % timeout) url_req = Request(url, headers=HTTP_HEADERS) url_fd = urlopen(url_req, timeout=timeout) logging.debug("Version regex: %s" % regex) v = re.findall(regex, url_fd.read().decode("utf-8", "ignore")) if v is None or len(v) == 0: raise VersionNotFound("No regex match on upstream") # remove duplicity v = set(v) # list all found versions logging.debug("Found versions: %s" % v) # exclude versions regex_exclude = value.get("regex_exclude", ".*(rc|beta|alpha|pre).*") if regex_exclude != "": logging.debug("Exclusion regex: %s" % regex_exclude) v -= set(filter(lambda x: re.search(regex_exclude, x), v)) logging.debug("Found versions after exclusion: %s" % v) # latest version is the highest v = max(v, key=VersionKey) # list selected version logging.debug("Upstream version is : %s" % v) return v except Exception as exp: if n == ntry: raise VersionNotFound("Upstream check failed: %s" % exp) assert (False) @staticmethod def get_version_downstream(name, value, mode): '''Return dowstream version''' try: return getattr(VersionController, "get_version_downstream_%s" % mode)(name, value) except AttributeError: raise InvalidConfigFile("Invalid dowstream mode") @staticmethod def get_version_downstream_pacman(name, value): '''Return pacman version''' logging.debug("Get pacman version") # Load pacman pacman = Pacman() # filter if repo is provided allowed_repos = value.get("repo").split( ",") if "repo" in value else None # looking into db for package name db, pkg = pacman.find_pkg(name, allowed_repos) if pkg is not None: epoch, pkgver, pkgrel = re.match( "^(?:(\d+)\:)?([^-:]*)(?:-(\d+))?", pkg.version).groups() logging.debug("pacman version in %s: %s" % (db.name, pkgver)) return pkgver raise VersionNotFound("No pacman package found") @staticmethod def get_version_downstream_archweb(name, value): '''Return archweb version''' logging.debug("Get archweb version") # if arch is specified archs = value.get("arch", "x86_64,i686,any").split(",") # if archweb repository is specified repos = value.get( "repo", "community-testing,community,testing,extra,core").split(",") # retrieve config timeout timeout = float(value["timeout"]) if "timeout" in value else None for arch in archs: for repo in repos: url = "http://www.archlinux.org/packages/%s/%s/%s/json" % ( repo, arch, name) url_req = Request(url, headers=HTTP_HEADERS) logging.debug("Requesting url: %s" % url) logging.debug("Timeout is %s" % timeout) try: url_fd = urlopen(url_req, timeout=timeout) d = json.loads(url_fd.read().decode("utf-8", "ignore")) v = d["pkgver"] logging.debug("Archweb version is : %s" % v) return v except Exception as exp: logging.debug("Archweb check failed: %s" % exp) raise VersionNotFound("No Archweb package found") @staticmethod def get_version_downstream_aur(name, value): '''Return archlinux user repository version''' logging.debug("Get AUR version") try: # retrieve config timeout timeout = float(value["timeout"]) if "timeout" in value else None url = "http://aur.archlinux.org/rpc.php?type=info&arg=%s" % name url_req = Request(url, headers=HTTP_HEADERS) logging.debug("Requesting url: %s" % url) logging.debug("Timeout is %s" % timeout) url_fd = urlopen(url_req, timeout=timeout) d = json.loads(url_fd.read().decode("utf-8", "ignore")) if "version" not in d or d["version"] != 1: raise VersionNotFound("Unsupported AUR version") if len(d["results"]) == 0: raise VersionNotFound("No such package") v = d["results"]["Version"].rsplit("-")[0] logging.debug("AUR version is : %s" % v) return v except Exception as exp: raise VersionNotFound("AUR check failed: %s" % exp) assert (False) @staticmethod def get_version_downstream_abs(name, value): '''Return abs version''' logging.debug("Get ABS version") # Get ABS tree path abspath = value.get("abs_path", "/var/abs") # Map db and name repos = [ d for d in os.listdir(abspath) if os.path.isdir(os.path.join(abspath, d)) ] # filter if repo is provided if "repo" in value: allowed_repos = value.get("repo").split(",") for r in list(repos): if r not in allowed_repos: repos.remove(r) # looking into db for package name for repo in repos: logging.debug("Looking into directory %s" % repo) repopath = os.path.join(abspath, repo) packages = [d for d in os.listdir(repopath)] if name in packages: pkgpath = os.path.join(repopath, name, "PKGBUILD") if os.path.isfile(pkgpath): # use bash to export vars. # WARNING: CODE IS EXECUTED pkgdict = parse_pkgbuild(pkgpath) if "pkgver" in pkgdict: v = pkgdict["pkgver"] logging.debug("ABS version is : %s" % v) return v raise VersionNotFound("No ABS package found") @staticmethod def get_version_downstream_none(name, value): '''Return none version''' return "" def print_names(self): '''Print packages name''' for name in self.packages: print(name) @staticmethod def print_modes(): '''Print comparaison modes''' for mode in fnmatch.filter(dir(VersionController), "get_version_downstream_*"): print(mode[23:]) def print_versions(self, only_new=False, only_fresh=False): '''Print versions''' for name, v_upstream, v_downstream in self.compare( only_new, only_fresh): self.print_version(name, v_upstream, v_downstream) def print_version(self, name, v1, v2=None): '''Handle printing of 2 versions''' # define used color c_blue = c_white = c_yellow = c_compare = c_reset = '' if sys.stdout.isatty(): if v2 is None: c_compare = '\033[1;33m' elif v1 == v2: c_compare = '\033[1;32m' else: c_compare = '\033[1;31m' c_blue = '\033[1;34m' c_white = '\033[1;37m' c_yellow = '\033[1;33m' c_reset = '\033[m' # print package name toprint = "%s[%s%s%s]" % (c_blue, c_white, name, c_blue) # print upstream toprint += " %sup: %s " % (c_yellow, v1) # print downstream if v2 is not "": # print separator toprint += "%s|" % c_blue origin = self._packages.get(name, {}).get("downstream", "downstream") toprint += " %s%s: %s" % (c_compare, origin, v2) toprint += c_reset print(toprint)