Exemple #1
0
    def get_max_version(self, category, name):
        """
        Get the recent available version of a package.

        Args:
            category: Category name.
            name: package name.

        Returns:
            The recent version of a package.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)

        if not category in self.database \
           or not name in self.database[category]['packages']:
            raise InvalidKeyError('No such package: ' + category + '/' + name)

        pkgname = category + '/' + name
        versions = list(self.database[category]['packages'][name])
        max_ver = versions[0]
        for version in versions[1:]:
            if portage.pkgcmp(portage.pkgsplit(pkgname + '-' + version),
                              portage.pkgsplit(pkgname + '-' + max_ver)) > 0:
                max_ver = version
        return max_ver
Exemple #2
0
def revisionMatch(revisionAtom, portdb, match_type="default"):
	"""
	handler for the special >~, >=~, <=~ and <~ atoms that are supposed to behave
	as > and < except that they are limited to the same version, the range only
	applies to the revision part.

	@type	revisionAtom: string
	@param	revisionAtom: a <~ or >~ atom that contains the atom to match against
	@type	portdb: portage.dbapi
	@param	portdb:	one of the portage databases to use as information source
	@type	match_type: string
	@param	match_type: if != "default" passed as first argument to portdb.xmatch
				to apply the wanted visibility filters

	@rtype:		list of strings
	@return:	a list with the matching versions
	"""
	if match_type == "default" or not hasattr(portdb, "xmatch"):
		if ":" in revisionAtom:
			mylist = portdb.match(re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:]))
		else:
			mylist = portdb.match(re.sub("-r[0-9]+$", "", revisionAtom[2:]))
	else:
		if ":" in revisionAtom:
			mylist = portdb.xmatch(match_type, re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:]))
		else:
			mylist = portdb.xmatch(match_type, re.sub("-r[0-9]+$", "", revisionAtom[2:]))
	rValue = []
	for v in mylist:
		r1 = portage.pkgsplit(v)[-1][1:]
		r2 = portage.pkgsplit(revisionAtom[3:])[-1][1:]
		if eval(r1+" "+revisionAtom[0:2]+" "+r2):
			rValue.append(v)
	return rValue
Exemple #3
0
    def get_max_version(self, category, name):
        """
        Get the recent available version of a package.

        Args:
            category: Category name.
            name: package name.

        Returns:
            The recent version of a package.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)

        if not category in self.database \
           or not name in self.database[category]['packages']:
            raise InvalidKeyError('No such package: ' + category + '/' + name)

        pkgname = category + '/' + name
        versions = list(self.database[category]['packages'][name])
        max_ver = versions[0]
        for version in versions[1:]:
            if portage.pkgcmp(portage.pkgsplit(pkgname + '-' + version),
                              portage.pkgsplit(pkgname + '-' + max_ver)) > 0:
                max_ver = version
        return max_ver
Exemple #4
0
def scan_url(pkg, url, options):
    output.einfo("Using BerliOS handler")

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    project, filename = re.search(berlios_regex, url).groups()

    project_page = "http://developer.berlios.de/projects/%s" % project
    content = urllib.request.urlopen(project_page).read()

    project_id = re.search(r"/project/filelist.php\?group_id=(\d+)",
                           content).group(1)

    base_url = (
        "http://developer.berlios.de/project/filelist.php?group_id=%s" %
        project_id)

    file_pattern = regex_from_template(filename.replace(ver, "${PV}"))

    result = url_scan(pkg, base_url, file_pattern)

    ret = []
    for found_url, pv, _, _ in result:
        found_url = found_url.replace("prdownload", "download")
        ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #5
0
def scan(cpv, url):
    'http://wiki.python.org/moin/PyPiXmlRpc'


    package = guess_package(cpv, url)

    euscan.output.einfo("Using PyPi XMLRPC: " + package)

    client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    versions = client.package_releases(package)

    if not versions:
        return versions

    versions.reverse()

    cp, ver, rev = portage.pkgsplit(cpv)

    ret = []

    for version in versions:
        if helpers.version_filtered(cp, ver, version):
            continue
        urls = client.release_urls(package, version)
        urls = " ".join([ infos['url'] for infos in urls ])
        ret.append(( urls, version ))

    return ret
Exemple #6
0
def scan_url(pkg, url, options):
    output.einfo("Using BerliOS handler")

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    project, filename = re.search(berlios_regex, url).groups()

    project_page = "http://developer.berlios.de/projects/%s" % project
    content = urllib.urlopen(project_page).read()

    project_id = re.search(
        r"/project/filelist.php\?group_id=(\d+)",
        content
    ).group(1)

    base_url = (
        "http://developer.berlios.de/project/filelist.php?group_id=%s" %
        project_id
    )

    file_pattern = regex_from_template(
        filename.replace(ver, "${PV}")
    )

    result = url_scan(pkg, base_url, file_pattern)

    ret = []
    for found_url, pv, _, _ in result:
        found_url = found_url.replace("prdownload", "download")
        ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #7
0
def scan_pkg(pkg, options):
    gem = options['data']
    url = 'http://rubygems.org/api/v1/versions/%s.json' % gem

    try:
        fp = helpers.urlopen(url)
    except urllib.error.URLError:
        return []
    except IOError:
        return []

    if not fp:
        return []

    data = fp.read()
    versions = json.loads(data)

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    ret = []
    for version in versions:
        up_pv = version['number']
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
        url = mangling.mangle_url(url, options)
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #8
0
def scan_pkg(pkg, options):
    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    packages_url, package_name = options['data'].strip().split(" ", 1)

    output.einfo("Using Debian Packages: " + packages_url)

    fp = urllib.urlopen(packages_url)
    content = fp.read()

    # Support for .gz and .bz2 Packages file
    if packages_url.endswith(".bz2"):
        content = bz2.decompress(content)
    if packages_url.endswith(".gz"):
        content = zlib.decompress(content, 16 + zlib.MAX_WBITS)

    content = content.split("\n\n")

    result = []

    for package_info in content:
        package_line = re.search(r"^Package: (.*)$", package_info, re.M)
        version_line = re.search(r"^Version: (.*)$", package_info, re.M)
        if package_line and package_line.group(1) == package_name:
            if version_line:
                result.append(version_line.group(1))

    ret = []
    for up_pv in result:
        url = ""  # TODO: How to find the url?
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #9
0
def re_find_package_name(package, regexps, uri):
    if not isinstance(regexps, (list, tuple)):
        regexps = ((regexps, 1),)

    match = None
    group = -1

    for regexp in regexps:
        if isinstance(regexp, (list, tuple)):
            regexp, group = regexp
        else:
            group = 1
        match = regexp.match(uri)
        if match:
            break

    if not match:
        sys.stderr.write(pp.warn("Can't find package name in '%s'" % uri))
        return None

    package_name = match.group(group)

    # Try to strip version, if present
    if "-%s" % package.version in package_name:
        package_name.replace("-%s" % package.version, "")
    cpv = "fake/" + package_name
    cpv = portage.pkgsplit(cpv)

    if cpv:
        package_name = cpv[0].replace("fake/", "", 1)
    return package_name
Exemple #10
0
def scan_url(pkg, url, options):
    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    remote_pkg = guess_package(cp, url)

    output.einfo("Using CPAN API: %s", remote_pkg)

    return scan_pkg(pkg, {'data': remote_pkg})
Exemple #11
0
def scan_url(pkg, url, options):
    'http://developer.github.com/v3/repos/downloads/'

    user, project, filename = guess_package(pkg.cpv, url)

    # find out where version is expected to be found
    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    if ver not in filename:
        return

    # now create a filename-matching regexp
    # XXX: supposedly replace first with (?P<foo>...)
    # and remaining ones with (?P=foo)
    fnre = re.compile('^%s$' % \
                      re.escape(filename).replace(re.escape(ver), '(.*?)'))

    output.einfo("Using github API for: project=%s user=%s filename=%s" % \
                 (project, user, filename))

    dlreq = urllib.request.urlopen('https://api.github.com/repos/%s/%s/downloads' % \
                            (user, project))
    dls = json.load(dlreq)

    ret = []
    for dl in dls:
        m = fnre.match(dl['name'])

        if m:
            pv = mangling.mangle_version(m.group(1), options)
            if helpers.version_filtered(cp, ver, pv):
                continue

            url = mangling.mangle_url(dl['html_url'], options)
            ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #12
0
def scan_pkg(pkg, options):
    package = options['data']

    output.einfo("Using PyPi XMLRPC: " + package)

    client = xmlrpc.client.ServerProxy('https://pypi.python.org/pypi')
    versions = client.package_releases(package)

    if not versions:
        return versions

    versions.reverse()

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    ret = []
    for up_pv in versions:
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        urls = client.release_urls(package, up_pv)
        urls = " ".join([mangling.mangle_url(infos['url'], options)
                         for infos in urls])
        ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #13
0
def scan_pkg(pkg, options):
    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    package = options['data'].strip()

    output.einfo("Using FreeCode handler: " + package)

    fp = urllib.urlopen("http://freecode.com/projects/%s/releases" % package)
    content = fp.read()

    result = re.findall(
        r'<a href="/projects/%s/releases/(\d+)">([^<]+)</a>' % package,
        content
    )

    ret = []
    for release_id, up_pv in result:
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        fp = urllib.urlopen("http://freecode.com/projects/%s/releases/%s" %
                            (package, release_id))
        content = fp.read()
        download_page = re.findall(r'<a href="(/urls/[^"]+)"', content)[0]
        fp = urllib.urlopen("http://freecode.com%s" % download_page)
        content = fp.read()
        url = re.findall(
            r'In case it doesn\'t, click here: <a href="([^"]+)"',
            content
        )[0]
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #14
0
def scan_pkg(pkg, options):
    gem = options['data']
    url = 'http://rubygems.org/api/v1/versions/%s.json' % gem

    try:
        fp = helpers.urlopen(url)
    except urllib2.URLError:
        return []
    except IOError:
        return []

    if not fp:
        return []

    data = fp.read()
    versions = json.loads(data)

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    ret = []
    for version in versions:
        up_pv = version['number']
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
        url = mangling.mangle_url(url, options)
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #15
0
def scan(cpv, url):
    'http://developer.github.com/v3/repos/downloads/'

    user, project, filename = guess_package(cpv, url)

    # find out where version is expected to be found
    cp, ver, rev = portage.pkgsplit(cpv)
    if ver not in filename:
        return

    # now create a filename-matching regexp
    # XXX: supposedly replace first with (?P<foo>...)
    # and remaining ones with (?P=foo)
    fnre = re.compile('^%s$' % re.escape(filename).replace(re.escape(ver), '(.*?)'))

    output.einfo("Using github API for: " + '/'.join(package))

    dlreq = urllib2.urlopen('https://api.github.com/repos/%s/%s/downloads' % (user, project))
    dls = json.load(dlreq)

    for dl in dls:
        m = fnre.match(dl['name'])

        if m:
            pv = helpers.gentoo_mangle_version(m.group(1))
            if helpers.version_filtered(cp, ver, pv):
                continue
            yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
Exemple #16
0
def scan_pkg(pkg, options):
    package = options['data']

    output.einfo("Using PyPi XMLRPC: " + package)

    client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    versions = client.package_releases(package)

    if not versions:
        return versions

    versions.reverse()

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    ret = []
    for up_pv in versions:
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        urls = client.release_urls(package, up_pv)
        urls = " ".join([mangling.mangle_url(infos['url'], options)
                         for infos in urls])
        ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #17
0
def scan_upstream_urls(cpv, urls):
    versions = []

    for filename in urls:
        for url in urls[filename]:
            if not CONFIG['quiet']:
                pp.uprint()
            euscan.output.einfo("SRC_URI is '%s'" % url)

            if '://' not in url:
                euscan.output.einfo("Invalid url '%s'" % url)
                continue

            ''' Try normal scan '''
            if CONFIG["scan-dir"]:
                versions.extend(handlers.scan(cpv, url))

            if versions and CONFIG['oneshot']:
                break

            ''' Brute Force '''
            if CONFIG["brute-force"] > 0:
                versions.extend(handlers.brute_force(cpv, url))

            if versions and CONFIG['oneshot']:
                break

    cp, ver, rev = portage.pkgsplit(cpv)
    return filter_versions(cp, versions)
Exemple #18
0
def scan(cpv, url):
    for bu in SCANDIR_BLACKLIST_URLS:
        if re.match(bu, url):
            euscan.output.einfo("%s is blacklisted by rule %s" % (url, bu))
            return []

    resolved_url = helpers.parse_mirror(url)
    if not resolved_url:
        return []

    cp, ver, rev = portage.pkgsplit(cpv)

    # 'Hack' for _beta/_rc versions where _ is used instead of -
    if ver not in resolved_url:
        newver = helpers.version_change_end_sep(ver)
        if newver and newver in resolved_url:
            euscan.output.einfo(
                "Version: using %s instead of %s" % (newver, ver)
            )
            ver = newver

    template = helpers.template_from_url(resolved_url, ver)
    if '${' not in template:
        euscan.output.einfo(
            "Url doesn't seems to depend on version: %s not found in %s" %
            (ver, resolved_url)
        )
        return []
    else:
        euscan.output.einfo("Scanning: %s" % template)

    steps = helpers.generate_scan_paths(template)
    return scan_directory_recursive(cp, ver, rev, "", steps, url)
Exemple #19
0
def scan(cpv, url):
    'http://wiki.python.org/moin/PyPiXmlRpc'

    package = guess_package(cpv, url)

    output.einfo("Using PyPi XMLRPC: " + package)

    client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    versions = client.package_releases(package)

    if not versions:
        return versions

    versions.reverse()

    cp, ver, rev = portage.pkgsplit(cpv)

    ret = []

    for up_pv in versions:
        pv = helpers.gentoo_mangle_version(up_pv)
        if helpers.version_filtered(cp, ver, pv):
            continue
        urls = client.release_urls(package, up_pv)
        urls = " ".join([infos['url'] for infos in urls])
        ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))

    return ret
Exemple #20
0
def scan_url(pkg, url, options):
    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    remote_pkg = guess_package(cp, url)

    output.einfo("Using CPAN API: %s", remote_pkg)

    return scan_pkg(pkg, {"data": remote_pkg})
Exemple #21
0
def scan_url(pkg, url, options):
    'http://developer.github.com/v3/repos/downloads/'

    user, project, filename = guess_package(pkg.cpv, url)

    # find out where version is expected to be found
    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    if ver not in filename:
        return

    # now create a filename-matching regexp
    # XXX: supposedly replace first with (?P<foo>...)
    # and remaining ones with (?P=foo)
    fnre = re.compile('^%s$' % \
                      re.escape(filename).replace(re.escape(ver), '(.*?)'))

    output.einfo("Using github API for: project=%s user=%s filename=%s" % \
                 (project, user, filename))

    dlreq = urllib2.urlopen('https://api.github.com/repos/%s/%s/downloads' % \
                            (user, project))
    dls = json.load(dlreq)

    ret = []
    for dl in dls:
        m = fnre.match(dl['name'])

        if m:
            pv = mangling.mangle_version(m.group(1), options)
            if helpers.version_filtered(cp, ver, pv):
                continue

            url = mangling.mangle_url(dl['html_url'], options)
            ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #22
0
def scan_pkg(pkg, options):
    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    package = options['data'].strip()

    output.einfo("Using FreeCode handler: " + package)

    fp = urllib.request.urlopen("http://freecode.com/projects/%s/releases" %
                                package)
    content = str(fp.read())

    result = re.findall(
        r'<a href="/projects/%s/releases/(\d+)">([^<]+)</a>' % package,
        content)

    ret = []
    for release_id, up_pv in result:
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        fp = urllib.request.urlopen(
            "http://freecode.com/projects/%s/releases/%s" %
            (package, release_id))
        content = str(fp.read())
        download_page = re.findall(r'<a href="(/urls/[^"]+)"', content)[0]
        fp = urllib.request.urlopen("http://freecode.com%s" % download_page)
        content = str(fp.read())
        url = re.findall(r'In case it doesn\'t, click here: <a href="([^"]+)"',
                         content)[0]
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #23
0
def search_ebuilds(path, portdir=True, searchdef="", repo_num="",
        config=None, data=None):
    pv = ""
    pkgs = []
    nr = len(data['ebuilds']) + 1

    if portdir:
        rep = darkgreen("Portage    ")
    else:
        rep = red("Overlay "+str(repo_num)+"  ")

    if isdir(path):
        filelist = listdir(path)

        for file in filelist:
            if file[-7:] == ".ebuild":
                pv = file[:-7]
                pkgs.append(list(pkgsplit(pv)))
                pkgs[-1].append(path + file)
                if searchdef != "" and pv == searchdef:
                    data['defebuild'] = (searchdef, pkgs[-1][3])
        if not portdir:
            config['found_in_overlay'] = True
        pkgs.sort(key=cmp_sort_key(mypkgcmp))
        for pkg in pkgs:
            rev = ""
            if pkg[2] != "r0":
                rev = "-" + pkg[2]
            data['output'].append(" " + rep + " [" + bold(str(nr)) + "] " +
                pkg[0] + "-" + pkg[1] + rev + "\n")
            data['ebuilds'].append(pkg[len(pkg)-1])
            nr += 1
Exemple #24
0
def can_handle(pkg, url=None):
    if not url:
        return False

    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    if ver not in url:
        return False

    return re.search(berlios_regex, url)
Exemple #25
0
def can_handle(pkg, url=None):
    if not url:
        return False

    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    if ver not in url:
        return False

    return re.search(berlios_regex, url)
Exemple #26
0
def can_handle(pkg, url=None):
    if not url:
        return False

    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    if ver not in url:
        return False

    return re.match(package_name_regex, url)
Exemple #27
0
def can_handle(pkg, url=None):
    if not url:
        return False

    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    if ver not in url:
        return False

    return "mirror://sourceforge/" in url
def get_v(cpv):
    "cpv can be anything"
    if portage.isjustname(cpv):
        raise Exception('Input (%s) has no version!' % cpv)
    pv = portage.pkgsplit(cpv)[-2:]
    if pv[1] == 'r0':
        return pv[0]
    else:
        return '%s-%s' % (pv[0], pv[1])
Exemple #29
0
def guess_gem(cpv, url):
    match = re.search('mirror://rubygems/(.*).gem', url)
    if match:
        cpv = 'fake/%s' % match.group(1)

    cp, ver, rev = portage.pkgsplit(cpv)
    cat, pkg = cp.split("/")

    return pkg
Exemple #30
0
def can_handle(pkg, url=None):
    if not url:
        return False

    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    if ver not in url:
        return False

    return "mirror://sourceforge/" in url
Exemple #31
0
def get_v(cpv):
    "cpv can be anything"
    if portage.isjustname(cpv):
        raise Exception('Input (%s) has no version!' % cpv)
    pv = portage.pkgsplit(cpv)[-2:]
    if pv[1] == 'r0':
        return pv[0]
    else:
        return '%s-%s' % (pv[0], pv[1])
Exemple #32
0
def can_handle(pkg, url=None):
    if not url:
        return False

    cp, ver, rev = portage.pkgsplit(pkg.cpv)
    if ver not in url:
        return False

    return re.match(package_name_regex, url)
Exemple #33
0
def scan(cpv, url):
    cp, ver, rev = portage.pkgsplit(cpv)
    pkg = guess_package(cp, url)

    orig_url = url
    url = 'http://search.cpan.org/api/dist/%s' % pkg

    euscan.output.einfo("Using: " + url)

    try:
        fp = helpers.urlopen(url)
    except urllib2.URLError:
        return []
    except IOError:
        return []

    if not fp:
        return []

    data = fp.read()
    data = json.loads(data)

    if 'releases' not in data:
        return []

    ret = []

    for version in data['releases']:
        #if version['status'] == 'testing':
        #    continue

        up_pv = version['version']
        up_pv = cpan_trim_version(up_pv)
        pv = gentoo_mangle_version(up_pv)
        up_ver = cpan_mangle_version(ver)

        if helpers.version_filtered(cp, up_ver, up_pv, cpan_vercmp):
            continue

        url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % (
            version['cpanid'][0],
            version['cpanid'][0:1],
            version['cpanid'],
            version['archive']
        )

        if url == orig_url:
            continue

        ret.append((url, pv))

    return ret
Exemple #34
0
def guess_gem(cpv, url):
    match = re.search('mirror://rubygems/(.*).gem', url)
    if match:
        cpv = 'fake/%s' % match.group(1)

    ret = portage.pkgsplit(cpv)
    if not ret:
        return None

    cp, ver, rev = ret
    cat, pkg = cp.split("/")

    return pkg
Exemple #35
0
def scan_upstream_urls(cpv, urls, on_progress):
    versions = []

    maxval = len(urls) + 5
    curval = 1

    for filename in urls:
        curval += 1
        if on_progress:
            on_progress(maxval, curval)

        for url in urls[filename]:
            if not CONFIG['quiet'] and not CONFIG['format']:
                pp.uprint()
            output.einfo("SRC_URI is '%s'" % url)

            if '://' not in url:
                output.einfo("Invalid url '%s'" % url)
                continue

            # Try normal scan
            if CONFIG["scan-dir"]:
                try:
                    versions.extend(handlers.scan(cpv, url))
                except Exception as e:
                    output.ewarn("Handler failed: [%s] %s"
                            % (e.__class__.__name__, e.message))

            if versions and CONFIG['oneshot']:
                break

            # Brute Force
            if CONFIG["brute-force"] > 0:
                versions.extend(handlers.brute_force(cpv, url))

            if versions and CONFIG['oneshot']:
                break

    cp, ver, rev = portage.pkgsplit(cpv)

    curval += 1
    if on_progress:
        on_progress(maxval, curval)

    result = filter_versions(cp, versions)

    curval += 1
    if on_progress:
        on_progress(maxval, curval)

    return result
Exemple #36
0
def _do_symlinks(mdb, slots, atom, atom_dir, summary):
    logging.debug("working in {}".format(atom_dir))
    # for each slot, get the best match according to the profile (i.e. newer stable, or newer unstable if the profile whitelist this atom)
    visibles = set()
    for slot in slots:
        for cpv in mdb.match_best_visibles("{}:{}".format(atom, slot)):
            slot, keywords = mdb.aux_get_first(cpv, ["SLOT", "KEYWORDS"])
            visibles.add(os.path.basename(cpv))
            # may use isStable() from portage/package/ebuild/_config/KeywordsManager.py
            logging.debug("found {} slot:{}".format(cpv, slot))
    for root, dirs, files in os.walk(atom_dir):
        for name in files:
            head, tail = os.path.splitext(name)
            # remove invisible ebuilds
            if tail == ".ebuild" and head not in visibles:
                try:
                    _fs_remove(os.path.join(atom_dir, name), mdb.assert_beneath_portdir)
                    summary.removed_ebuilds.add(head)
                except OutsideOfPortageTreeException as exc:
                    logging.warning("skipping file for deletion (duplicate atoms?): %s/%s", atom_dir, name)
                    continue

    # It's more common to remove an old package version than the more
    # up-to-date (i.e. we don't downgrade packages but can keep and old version
    # for compatibility): decrease order to keep the most up to date at first.
    for i, pvr in enumerate(sorted([portage.pkgsplit(x) for x in visibles], key=functools.cmp_to_key(portage.pkgcmp), reverse=True)):
        if pvr[2] == "r0":
            name = "-".join(pvr[:-1])
        else:
            name = "-".join(pvr)
        src = "{}.ebuild".format(name)

        # ignore already equalized ebuilds
        if os.path.islink(os.path.join(atom_dir, src)):
            continue

        # the equalized name should not end with ".ebuild"
        # (cf. dbapi/porttree.py:cp_list "Invalid ebuild name" and
        # versions.py:catpkgsplit)
        dst = ".{}.ebuild.{}".format(pvr[0], i)
        try:
            _fs_move(atom_dir, src, dst, mdb.assert_beneath_portdir)
        except OutsideOfPortageTreeException as exc:
            logging.warning("skipping atom for move (duplicate atoms?): %s/(%s -> %s)", atom_dir, src, dst)
            continue
        try:
            _fs_symlink(atom_dir, src, dst, mdb.assert_beneath_portdir)
            summary.symlinked_ebuilds += 1
        except OutsideOfPortageTreeException as exc:
            logging.warning("skipping atom for symlink (duplicate atoms?): %s/(%s -> %s)", atom_dir, src, dst)
            continue
Exemple #37
0
def revisionMatch(revisionAtom, portdb, match_type="default"):
    """
	handler for the special >~, >=~, <=~ and <~ atoms that are supposed to behave
	as > and < except that they are limited to the same version, the range only
	applies to the revision part.

	@type	revisionAtom: string
	@param	revisionAtom: a <~ or >~ atom that contains the atom to match against
	@type	portdb: portage.dbapi
	@param	portdb:	one of the portage databases to use as information source
	@type	match_type: string
	@param	match_type: if != "default" passed as first argument to portdb.xmatch
				to apply the wanted visibility filters

	@rtype:		list of strings
	@return:	a list with the matching versions
	"""
    if match_type == "default" or not hasattr(portdb, "xmatch"):
        if ":" in revisionAtom:
            mylist = portdb.match(
                re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:]))
        else:
            mylist = portdb.match(re.sub("-r[0-9]+$", "", revisionAtom[2:]))
    else:
        if ":" in revisionAtom:
            mylist = portdb.xmatch(
                match_type,
                re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:]))
        else:
            mylist = portdb.xmatch(match_type,
                                   re.sub("-r[0-9]+$", "", revisionAtom[2:]))
    rValue = []
    for v in mylist:
        r1 = portage.pkgsplit(v)[-1][1:]
        r2 = portage.pkgsplit(revisionAtom[3:])[-1][1:]
        if eval(r1 + " " + revisionAtom[0:2] + " " + r2):
            rValue.append(v)
    return rValue
Exemple #38
0
    def deserialize(cls, value):
        atom = portage.dep.Atom(value)
        operator = portage.dep.get_operator(atom)
        cpv = portage.dep.dep_getcpv(atom)
        category, rest = portage.catsplit(cpv)

        if operator:
            package, version, revision = portage.pkgsplit(rest)
        else:
            package = rest
            version = ""
            operator = ""

        return Dependency(category, package, version, operator)
Exemple #39
0
    def deserialize(cls, value):
        atom = portage.dep.Atom(value)
        operator = portage.dep.get_operator(atom)
        cpv = portage.dep.dep_getcpv(atom)
        category, rest = portage.catsplit(cpv)

        if operator:
            package, version, revision = portage.pkgsplit(rest)
        else:
            package = rest
            version = ""
            operator = ""

        return Dependency(category, package, version, operator)
Exemple #40
0
def scan_directory_recursive(cpv, url, steps):
    if not steps:
        return []

    cp, ver, rev = portage.pkgsplit(cpv)
    url += steps[0][0]
    pattern = steps[0][1]

    steps = steps[1:]

    euscan.output.einfo("Scanning: %s" % url)

    try:
        fp = helpers.urlopen(url)
    except urllib2.URLError:
        return []
    except IOError:
        return []

    if not fp:
        return []

    data = fp.read()

    results = []

    if re.search("<\s*a\s+[^>]*href", data):
        results.extend(scan_html(data, url, pattern))
    elif url.startswith('ftp://'):
        results.extend(scan_ftp(data, url, pattern))

    versions = []

    for version, path in results:
        if helpers.version_filtered(cp, ver, version):
            continue

        if not url.endswith('/') and not path.startswith('/'):
            path = url + '/' + path
        else:
            path = url + path

        versions.append((path, version))

        if steps:
            ret = scan_directory_recursive(cpv, path, steps)
            versions.extend(ret)

    return versions
Exemple #41
0
def guess_package(cp, url):
    match = _cpan_package_name_re.search(url)

    pkg = None

    if match:
        pkg = match.group(1)
        try:
            cp, ver, rev = portage.pkgsplit('fake/' + pkg)
        except:
            pass

    cat, pkg = cp.split("/")

    return pkg
Exemple #42
0
    def _guess_components(cls, my_p):
        """Try to break up raw MY_P into PN and PV"""
        pn, pv = "", ""

        # Ok, we just have one automagical test here.
        # We should look at versionator.eclass for inspiration
        # and then come up with several functions.
        my_p = my_p.replace("_", "-")

        psplit = pkgsplit(my_p)
        if psplit:
            pn = psplit[0]
            pv = psplit[1]
        log.debug("guess_components got: pn(%s), pv(%s)", pn, pv)
        return pn, pv
Exemple #43
0
def guess_package(cp, url):
    match = _cpan_package_name_re.search(url)

    pkg = None

    if match:
        pkg = match.group(1)
        try:
            cp, ver, rev = portage.pkgsplit("fake/" + pkg)
        except:
            pass

    cat, pkg = cp.split("/")

    return pkg
Exemple #44
0
 def getVersion(self, packageName):
     # the portage utilities are almost always going to be in
     # /usr/lib/portage/pym
     import sys
     sys.path.append('/usr/lib/portage/pym')
     import portage
     # FIXME: this takes the first package returned in the list, in the
     # case that there are slotted packages, and removes the leading
     # category such as 'sys-apps'
     gentooPackageName = portage.db["/"]["vartree"].dbapi.match(packageName)[0].split('/')[1]
     # this removes the distribution specific versioning returning only the
     # upstream version
     upstreamVersion = portage.pkgsplit(gentooPackageName)[1]
     # print("Version of package is: " + upstreamVersion)
     return Version.fromString(upstreamVersion)
Exemple #45
0
    def _guess_components(cls, my_p):
        """Try to break up raw MY_P into PN and PV"""
        pn, pv = "", ""

        # Ok, we just have one automagical test here.
        # We should look at versionator.eclass for inspiration
        # and then come up with several functions.
        my_p = my_p.replace("_", "-")

        psplit = pkgsplit(my_p)
        if psplit:
            pn = psplit[0]
            pv = psplit[1]
        log.debug("guess_components got: pn(%s), pv(%s)", pn, pv)
        return pn, pv
Exemple #46
0
 def getVersion(self, packageName):
     # the portage utilities are almost always going to be in
     # /usr/lib/portage/pym
     import sys
     sys.path.append('/usr/lib/portage/pym')
     import portage
     # FIXME: this takes the first package returned in the list, in the
     # case that there are slotted packages, and removes the leading
     # category such as 'sys-apps'
     gentooPackageName = portage.db["/"]["vartree"].dbapi.match(
         packageName)[0].split('/')[1]
     # this removes the distribution specific versioning returning only the
     # upstream version
     upstreamVersion = portage.pkgsplit(gentooPackageName)[1]
     # print("Version of package is: " + upstreamVersion)
     return Version.fromString(upstreamVersion)
Exemple #47
0
def scan_url(pkg, url, options):
    output.einfo("Using SourceForge handler")

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    project, filename = re.search(
        "mirror://sourceforge/([^/]+)/(?:.*/)?([^/]+)", url).groups()

    base_url = "http://qa.debian.org/watch/sf.php/%s" % project
    file_pattern = regex_from_template(filename.replace(ver, "${PV}"))

    result = url_scan(pkg, base_url, file_pattern)

    ret = []
    for url, pv, _, _ in result:
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #48
0
def scan_url(pkg, url, options):
    output.einfo("Using Google Code handler")

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    package_name = re.match(package_name_regex, url).group(1)
    base_url = "http://code.google.com/p/%s/downloads/list" % package_name

    file_pattern = regex_from_template(
        url.split("/")[-1].replace(ver, "${PV}"))

    result = url_scan(pkg, base_url, file_pattern)

    ret = []
    for url, pv, _, _ in result:
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
def equalize(mdb, atoms=None, dry_run=False):
    """Equalize a Gentoo Portage tree. If `atoms` is None (or emtpy list, or
    whatever evaluates to False), then this function will equalize the entire
    Portage tree.

    :param mdb: a set of Portage tree databases from MultiDb
    """

    global DRY_RUN
    DRY_RUN = dry_run

    # get the entire list of atoms provided by the set of Portage tree
    # databases `mdb`
    if not atoms:
        atoms = mdb.cp_all()

    atom_nb = len(atoms)
    summary = EqualizeSummary()
    for i, atom in enumerate(atoms, start=1):
        # find all the slots for this atom
        slots = set()
        for cpv in mdb.match(atom):
            slots.add(mdb.aux_get_first(cpv, ["SLOT"])[0])
        logging.debug("")
        logging.info("equalizing {}/{} {}".format(i, atom_nb, atom))
        # check if some slots are visible to the current profile
        if len(slots) != 0:
            _do_symlinks(mdb, slots, atom, mdb.get_atom_dir_selected(cpv),
                         summary)
            continue
        # remove files which are not usable with the current profile
        cpvs = mdb.match_all(atom)
        if len(cpvs) == 0:
            raise Exception(
                "Missing atom in the cache, you should run `egencache --update` for this Portage tree"
            )
        cpv = cpvs.pop()
        atom_dir = mdb.get_atom_dir_selected(cpv)
        try:
            _fs_remove_tree(atom_dir, mdb.assert_beneath_portdir)
            summary.removed_packages.add("/".join(portage.pkgsplit(cpv)[0:2]))
        except OutsideOfPortageTreeException as exc:
            logging.debug("skipping atom dir for deletion: %s", atom_dir)
            continue
    return summary
Exemple #50
0
    def split_uri(cls, uri):
        """Try to split a URI into PN, PV and REV

        :param uri: SRC_URI
        :type uri: string
        :returns: PN, PV, REV
        :rtype: tuple of strings

        **Example:**

        >>> Enamer.split_uri('http://www.foobar.com/foobar-1.0.tar.gz')
        ('foobar', '1.0', 'r0')
        >>> Enamer.split_uri('http://www.foobar.com/foo-2.3_beta3-r5.tar.gz')
        ('foo', '2.3_beta3', 'r5')

        """
        p = cls.get_filename(uri)
        return pkgsplit(p)
Exemple #51
0
    def split_uri(cls, uri):
        """Try to split a URI into PN, PV and REV

        :param uri: SRC_URI
        :type uri: string
        :returns: PN, PV, REV
        :rtype: tuple of strings

        **Example:**

        >>> Enamer.split_uri('http://www.foobar.com/foobar-1.0.tar.gz')
        ('foobar', '1.0', 'r0')
        >>> Enamer.split_uri('http://www.foobar.com/foo-2.3_beta3-r5.tar.gz')
        ('foo', '2.3_beta3', 'r5')

        """
        p = cls.get_filename(uri)
        return pkgsplit(p)
Exemple #52
0
def scan_url(pkg, url, options):
    output.einfo("Using Google Code handler")

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    package_name = re.match(package_name_regex, url).group(1)
    base_url = "http://code.google.com/p/%s/downloads/list" % package_name

    file_pattern = regex_from_template(
        url.split("/")[-1].replace(ver, "${PV}")
    )

    result = url_scan(pkg, base_url, file_pattern)

    ret = []
    for url, pv, _, _ in result:
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
    return ret
Exemple #53
0
def scan_pkg(pkg, options):
    package = options['data']

    output.einfo("Using Gnome json cache: " + package)

    fp = urllib.request.urlopen('/'.join(
        [GNOME_URL_SOURCE, package, 'cache.json']))
    content = fp.read()
    fp.close()

    cache = json.loads(content, encoding='ascii')

    if cache[0] != 4:
        output.eerror('Unknow cache format detected')
        return []

    versions = cache[2][package]

    if not versions:
        return []

    versions.reverse()

    cp, ver, _rev = portage.pkgsplit(pkg.cpv)

    ret = []
    for up_pv in versions:
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        up_files = cache[1][package][up_pv]
        for tarball_comp in ('tar.xz', 'tar.bz2', 'tar.gz'):
            if tarball_comp in up_files:
                url = '/'.join(
                    [GNOME_URL_SOURCE, package, up_files[tarball_comp]])
                break
        else:
            output.ewarn('No tarball for release %s' % up_pv)
        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))

    return ret
Exemple #54
0
def scan_url(pkg, url, options):
    if CONFIG["scan-dir"]:
        for bu in SCANDIR_BLACKLIST_URLS:
            if re.match(bu, url):
                output.einfo("%s is blacklisted by rule %s" % (url, bu))
                return []

        resolved_url = helpers.parse_mirror(url)
        if not resolved_url:
            return []

        cp, ver, rev = portage.pkgsplit(pkg.cpv)

        # 'Hack' for _beta/_rc versions where _ is used instead of -
        if ver not in resolved_url:
            newver = helpers.version_change_end_sep(ver)
            if newver and newver in resolved_url:
                output.einfo(
                    "Version: using %s instead of %s" % (newver, ver)
                )
                ver = newver

        template = helpers.template_from_url(resolved_url, ver)
        if '${' not in template:
            output.einfo(
                "Url doesn't seems to depend on version: %s not found in %s" %
                (ver, resolved_url)
            )
            return []
        else:
            output.einfo("Scanning: %s" % template)

        steps = helpers.generate_scan_paths(template)
        ret = scan_directory_recursive(cp, ver, rev, "", steps, url, options)

    if not ret:
        ret = brute_force(pkg, url)

    return ret
Exemple #55
0
    def __xmatch(self, pdb, package):
        """xmatch function that searches for all packages over all repos"""
        try:
            mycp = port.dep_expand(package, mydb=pdb, settings=pdb.settings).cp
        except port.exception.AmbiguousPackageName as Arg:
            msg_err = 'Ambiguous package name "%s".\n' % package
            found = 'Possibilities: %s' % Arg
            raise SystemExit('%s%s' % (msg_err, found))
        except port.exception.InvalidAtom:
            msg_err = 'No such package "%s"' % package
            raise SystemExit(msg_err)

        mysplit = mycp.split('/')
        mypkgs = []
        for oroot in pdb.porttrees:
            try:
                file_list = os.listdir(os.path.join(oroot, mycp))
            except OSError:
                continue
            for x in file_list:
                pf = x[:-7] if x[-7:] == '.ebuild' else []
                if pf:
                    ps = port.pkgsplit(pf)
                    if not ps or ps[0] != mysplit[1]:
                        # we got garbage or ebuild with wrong name in the dir
                        continue
                    ver_match = port.versions.ver_regexp.match("-".join(
                        ps[1:]))
                    if ver_match is None or not ver_match.groups():
                        # version is not allowed by portage or unset
                        continue
                    # obtain related data from metadata and append to the pkg list
                    keywords, slot = self.__getMetadata(
                        pdb, mysplit[0] + '/' + pf, oroot)
                    mypkgs.append(
                        [mysplit[0] + '/' + pf, oroot, slot, keywords])

        self.__packages_sort(mypkgs)
        return mypkgs
Exemple #56
0
def scan_pkg(pkg, options):
    package = options['data']

    output.einfo("Using PyPi API: " + package)

    url = 'https://pypi.python.org/pypi/%s/json' % package

    try:
        fp = helpers.urlopen(url)
    except urllib.error.URLError:
        return []
    except IOError:
        return []

    if not fp:
        return []

    data = fp.read()
    data = json.loads(data)

    if 'releases' not in data:
        return []

    ret = []

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    ret = []
    for up_pv in data['releases']:
        pv = mangling.mangle_version(up_pv, options)
        if helpers.version_filtered(cp, ver, pv):
            continue
        urls = [entry['url'] for entry in data['releases'][up_pv]]
        urls = " ".join([mangling.mangle_url(url, options)
                         for url in urls])
        ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
    return ret
def searchEbuilds(path,
                  portdir=True,
                  searchdef="",
                  repo_num="",
                  config=None,
                  data=None):
    pv = ""
    pkgs = []
    nr = len(data['ebuilds']) + 1

    if portdir:
        rep = darkgreen("Portage    ")
    else:
        rep = red("Overlay " + str(repo_num) + "  ")

    if isdir(path):
        filelist = listdir(path)

        for file in filelist:
            if file[-7:] == ".ebuild":
                pv = file[:-7]
                pkgs.append(list(pkgsplit(pv)))
                pkgs[-1].append(path + file)
                if searchdef != "" and pv == searchdef:
                    data['defebuild'] = (searchdef, pkgs[-1][3])
        if not portdir:
            config['found_in_overlay'] = True
        pkgs.sort(mypkgcmp)
        for pkg in pkgs:
            rev = ""
            if pkg[2] != "r0":
                rev = "-" + pkg[2]
            data['output'].append(" " + rep + " [" + bold(str(nr)) + "] " +
                                  pkg[0] + "-" + pkg[1] + rev + "\n")
            data['ebuilds'].append(pkg[len(pkg) - 1])
            nr += 1
Exemple #58
0
def brute_force(pkg, url):
    if CONFIG["brute-force"] == 0:
        return []

    cp, ver, rev = portage.pkgsplit(pkg.cpv)

    url = helpers.parse_mirror(url)
    if not url:
        return []

    for bp in BRUTEFORCE_BLACKLIST_PACKAGES:
        if re.match(bp, cp):
            output.einfo("%s is blacklisted by rule %s" % (cp, bp))
            return []

    for bp in BRUTEFORCE_BLACKLIST_URLS:
        if re.match(bp, url):
            output.einfo("%s is blacklisted by rule %s" % (cp, bp))
            return []

    output.einfo("Generating version from " + ver)

    components = helpers.split_version(ver)
    versions = helpers.gen_versions(components, CONFIG["brute-force"])

    # Remove unwanted versions
    for v in versions:
        if helpers.vercmp(cp, ver, helpers.join_version(v)) >= 0:
            versions.remove(v)

    if not versions:
        output.einfo("Can't generate new versions from " + ver)
        return []

    template = helpers.template_from_url(url, ver)

    if '${PV}' not in template:
        output.einfo(
            "Url doesn't seems to depend on full version: %s not found in %s" %
            (ver, url))
        return []
    else:
        output.einfo("Brute forcing: %s" % template)

    result = []

    i = 0
    done = []

    while i < len(versions):
        components = versions[i]
        i += 1
        if components in done:
            continue
        done.append(tuple(components))

        version = helpers.join_version(components)

        if helpers.version_filtered(cp, ver, version):
            continue

        try_url = helpers.url_from_template(template, version)
        infos = helpers.tryurl(try_url, template)

        if not infos:
            continue
        confidence = confidence_score(try_url,
                                      url,
                                      minimum=BRUTEFORCE_CONFIDENCE)
        result.append([try_url, version, BRUTEFORCE_HANDLER_NAME, confidence])

        if len(result) > CONFIG['brute-force-false-watermark']:
            output.einfo("Broken server detected ! Skipping brute force.")
            return []

        if CONFIG["brute-force-recursive"]:
            for v in helpers.gen_versions(list(components),
                                          CONFIG["brute-force"]):
                if v not in versions and tuple(v) not in done:
                    versions.append(v)

        if CONFIG["oneshot"]:
            break

    return result
Exemple #59
0
def pkgsplit(ebuild):
    """Split ebuild into [category/package, version, revision]"""
    debug.dprint("PORTAGELIB: pkgsplit(); calling portage function")
    return portage.pkgsplit(ebuild)