def get_uncompressed_data_from_url(url, filename, proxies):
filename = myurlgrab(url, filename, proxies)
suffix = None
if filename.endswith(".gz"):
suffix = ".gz"
runner.quiet(['gunzip', "-f", filename])
elif filename.endswith(".bz2"):
suffix = ".bz2"
runner.quiet(['bunzip2', "-f", filename])
if suffix:
filename = filename.replace(suffix, "")
return filename
def _get_uncompressed_data_from_url(url, filename, proxies):
filename = myurlgrab(url, filename, proxies)
suffix = None
if filename.endswith(".gz"):
suffix = ".gz"
runner.quiet(['gunzip', "-f", filename])
elif filename.endswith(".bz2"):
suffix = ".bz2"
runner.quiet(['bunzip2', "-f", filename])
if suffix:
filename = filename.replace(suffix, "")
return filename
def get_package(pkg, repometadata, arch=None):
ver = ""
target_repo = None
for repo in repometadata:
if repo["primary"].endswith(".xml"):
root = xmlparse(repo["primary"])
ns = root.getroot().tag
ns = ns[0:ns.rindex("}") + 1]
for elm in root.getiterator("%spackage" % ns):
if elm.find("%sname" % ns).text == pkg:
if elm.find("%sarch" % ns).text != "src":
version = elm.find("%sversion" % ns)
tmpver = "%s-%s" % (version.attrib['ver'],
version.attrib['rel'])
if tmpver > ver:
ver = tmpver
location = elm.find("%slocation" % ns)
pkgpath = "%s" % location.attrib['href']
target_repo = repo
break
if repo["primary"].endswith(".sqlite"):
con = sqlite.connect(repo["primary"])
if not arch:
for row in con.execute(
"select version, release,location_href from packages where name = \"%s\" and arch != \"src\""
% pkg):
tmpver = "%s-%s" % (row[0], row[1])
if tmpver > ver:
pkgpath = "%s" % row[2]
target_repo = repo
break
else:
for row in con.execute(
"select version, release,location_href from packages where name = \"%s\""
% pkg):
tmpver = "%s-%s" % (row[0], row[1])
if tmpver > ver:
pkgpath = "%s" % row[2]
target_repo = repo
break
con.close()
if target_repo:
makedirs("%s/%s/packages" %
(target_repo["cachedir"], target_repo["name"]))
url = str(target_repo["baseurl"] + "/" + pkgpath)
filename = str("%s/%s/packages/%s" %
(target_repo["cachedir"], target_repo["name"],
os.path.basename(pkgpath)))
pkg = myurlgrab(url, filename, target_repo["proxies"])
return pkg
else:
return None
def get_kickstarts_from_repos(repometadata):
kickstarts = []
for repo in repometadata:
try:
root = xmlparse(repo["repomd"])
except SyntaxError:
raise CreatorError("repomd.xml syntax error.")
ns = root.getroot().tag
ns = ns[0:ns.rindex("}") + 1]
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "image-config":
break
if elm.attrib["type"] != "image-config":
continue
location = elm.find("%slocation" % ns)
image_config = str(repo["baseurl"] + "/" + location.attrib["href"])
filename = str("%s/%s/image-config.xml%s" %
(repo["cachedir"], repo["name"], suffix))
image_config = get_uncompressed_data_from_url(image_config, filename,
repo["proxies"])
try:
root = xmlparse(image_config)
except SyntaxError:
raise CreatorError("image-config.xml syntax error.")
for elm in root.getiterator("config"):
path = elm.find("path").text
path = path.replace("images-config", "image-config")
description = elm.find("description").text
makedirs(
os.path.dirname("%s/%s/%s" %
(repo["cachedir"], repo["name"], path)))
url = path
if "http" not in path:
url = str(repo["baseurl"] + "/" + path)
filename = str("%s/%s/%s" % (repo["cachedir"], repo["name"], path))
path = myurlgrab(url, filename, repo["proxies"])
kickstarts.append({"filename": path, "description": description})
return kickstarts
def get_package(pkg, repometadata, arch = None):
ver = ""
target_repo = None
for repo in repometadata:
if repo["primary"].endswith(".xml"):
root = xmlparse(repo["primary"])
ns = root.getroot().tag
ns = ns[0:ns.rindex("}")+1]
for elm in root.getiterator("%spackage" % ns):
if elm.find("%sname" % ns).text == pkg:
if elm.find("%sarch" % ns).text != "src":
version = elm.find("%sversion" % ns)
tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
if tmpver > ver:
ver = tmpver
location = elm.find("%slocation" % ns)
pkgpath = "%s" % location.attrib['href']
target_repo = repo
break
if repo["primary"].endswith(".sqlite"):
con = sqlite.connect(repo["primary"])
if not arch:
for row in con.execute("select version, release,location_href from packages where name = \"%s\" and arch != \"src\"" % pkg):
tmpver = "%s-%s" % (row[0], row[1])
if tmpver > ver:
pkgpath = "%s" % row[2]
target_repo = repo
break
else:
for row in con.execute("select version, release,location_href from packages where name = \"%s\"" % pkg):
tmpver = "%s-%s" % (row[0], row[1])
if tmpver > ver:
pkgpath = "%s" % row[2]
target_repo = repo
break
con.close()
if target_repo:
makedirs("%s/%s/packages" % (target_repo["cachedir"], target_repo["name"]))
url = str(target_repo["baseurl"] + "/" + pkgpath)
filename = str("%s/%s/packages/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath)))
pkg = myurlgrab(url, filename, target_repo["proxies"])
return pkg
else:
return None
def get_kickstarts_from_repos(repometadata):
kickstarts = []
for repo in repometadata:
try:
root = xmlparse(repo["repomd"])
except SyntaxError:
raise CreatorError("repomd.xml syntax error.")
ns = root.getroot().tag
ns = ns[0:ns.rindex("}")+1]
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "image-config":
break
if elm.attrib["type"] != "image-config":
continue
location = elm.find("%slocation" % ns)
image_config = str(repo["baseurl"] + "/" + location.attrib["href"])
filename = str("%s/%s/image-config.xml%s" % (repo["cachedir"], repo["name"], suffix))
image_config = get_uncompressed_data_from_url(image_config,filename,repo["proxies"])
try:
root = xmlparse(image_config)
except SyntaxError:
raise CreatorError("image-config.xml syntax error.")
for elm in root.getiterator("config"):
path = elm.find("path").text
path = path.replace("images-config", "image-config")
description = elm.find("description").text
makedirs(os.path.dirname("%s/%s/%s" % (repo["cachedir"], repo["name"], path)))
url = path
if "http" not in path:
url = str(repo["baseurl"] + "/" + path)
filename = str("%s/%s/%s" % (repo["cachedir"], repo["name"], path))
path = myurlgrab(url, filename, repo["proxies"])
kickstarts.append({"filename":path,"description":description})
return kickstarts
def get_metadata_from_repos(repostrs, cachedir):
my_repo_metadata = []
for repostr in repostrs:
reponame = None
baseurl = None
proxy = None
items = repostr.split(",")
for item in items:
subitems = item.split(":")
if subitems[0] == "name":
reponame = subitems[1]
if subitems[0] == "baseurl":
baseurl = item[8:]
if subitems[0] == "proxy":
proxy = item[6:]
if subitems[0] in ("http", "https", "ftp", "ftps", "file"):
baseurl = item
if not proxy:
proxy = get_proxy_for(baseurl)
proxies = None
if proxy:
proxies = {str(proxy.split(":")[0]): str(proxy)}
makedirs(cachedir + "/" + reponame)
url = str(baseurl + "/repodata/repomd.xml")
filename = str("%s/%s/repomd.xml" % (cachedir, reponame))
repomd = myurlgrab(url, filename, proxies)
try:
root = xmlparse(repomd)
except SyntaxError:
raise CreatorError("repomd.xml syntax error.")
ns = root.getroot().tag
ns = ns[0:ns.rindex("}") + 1]
patterns = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "patterns":
patterns = elm.find("%slocation" % ns).attrib['href']
break
comps = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "group_gz":
comps = elm.find("%slocation" % ns).attrib['href']
break
if not comps:
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "group":
comps = elm.find("%slocation" % ns).attrib['href']
break
primary_type = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "primary_db":
primary_type = ".sqlite"
break
if not primary_type:
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "primary":
primary_type = ".xml"
break
if not primary_type:
continue
primary = elm.find("%slocation" % ns).attrib['href']
primary = get_metadata_from_repo(baseurl, proxies, cachedir, reponame,
primary)
if patterns:
patterns = get_metadata_from_repo(baseurl, proxies, cachedir,
reponame, patterns)
if comps:
comps = get_metadata_from_repo(baseurl, proxies, cachedir,
reponame, comps)
""" Get repo key """
try:
repokey = get_metadata_from_repo(baseurl, proxies, cachedir,
reponame,
"repodata/repomd.xml.key")
except CreatorError:
repokey = None
msger.warning("\ncan't get %s/%s" %
(baseurl, "repodata/repomd.xml.key"))
my_repo_metadata.append({
"name": reponame,
"baseurl": baseurl,
"repomd": repomd,
"primary": primary,
"cachedir": cachedir,
"proxies": proxies,
"patterns": patterns,
"comps": comps,
"repokey": repokey
})
return my_repo_metadata
def get_metadata_from_repos(repostrs, cachedir):
my_repo_metadata = []
for repostr in repostrs:
reponame = None
baseurl = None
proxy = None
items = repostr.split(",")
for item in items:
subitems = item.split(":")
if subitems[0] == "name":
reponame = subitems[1]
if subitems[0] == "baseurl":
baseurl = item[8:]
if subitems[0] == "proxy":
proxy = item[6:]
if subitems[0] in ("http", "https", "ftp", "ftps", "file"):
baseurl = item
if not proxy:
proxy = get_proxy_for(baseurl)
proxies = None
if proxy:
proxies = {str(proxy.split(":")[0]):str(proxy)}
makedirs(cachedir + "/" + reponame)
url = str(baseurl + "/repodata/repomd.xml")
filename = str("%s/%s/repomd.xml" % (cachedir, reponame))
repomd = myurlgrab(url, filename, proxies)
try:
root = xmlparse(repomd)
except SyntaxError:
raise CreatorError("repomd.xml syntax error.")
ns = root.getroot().tag
ns = ns[0:ns.rindex("}")+1]
patterns = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "patterns":
patterns = elm.find("%slocation" % ns).attrib['href']
break
comps = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "group_gz":
comps = elm.find("%slocation" % ns).attrib['href']
break
if not comps:
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "group":
comps = elm.find("%slocation" % ns).attrib['href']
break
primary_type = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "primary_db":
primary_type=".sqlite"
break
if not primary_type:
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "primary":
primary_type=".xml"
break
if not primary_type:
continue
primary = elm.find("%slocation" % ns).attrib['href']
primary = _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, primary)
if patterns:
patterns = _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, patterns)
if comps:
comps = _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, comps)
""" Get repo key """
try:
repokey = _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, "repodata/repomd.xml.key")
except CreatorError:
repokey = None
msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key"))
my_repo_metadata.append({"name":reponame, "baseurl":baseurl, "repomd":repomd, "primary":primary, "cachedir":cachedir, "proxies":proxies, "patterns":patterns, "comps":comps, "repokey":repokey})
return my_repo_metadata
def get_metadata_from_repos(repos, cachedir):
my_repo_metadata = []
for repo in repos:
reponame = repo['name']
baseurl = repo['baseurl']
if 'proxy' in repo:
proxy = repo['proxy']
else:
proxy = get_proxy_for(baseurl)
proxies = None
if proxy:
proxies = {str(baseurl.split(":")[0]):str(proxy)}
makedirs(os.path.join(cachedir, reponame))
url = os.path.join(baseurl, "repodata/repomd.xml")
filename = os.path.join(cachedir, reponame, 'repomd.xml')
repomd = myurlgrab(url, filename, proxies)
try:
root = xmlparse(repomd)
except SyntaxError:
raise CreatorError("repomd.xml syntax error.")
ns = root.getroot().tag
ns = ns[0:ns.rindex("}")+1]
filepaths = {}
checksums = {}
sumtypes = {}
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "patterns":
filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
break
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] in ("group_gz", "group"):
filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
checksums['comps'] = elm.find("%sopen-checksum" % ns).text
sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
break
primary_type = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] in ("primary_db", "primary"):
primary_type = elm.attrib["type"]
filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
checksums['primary'] = elm.find("%sopen-checksum" % ns).text
sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
break
if not primary_type:
continue
for item in ("primary", "patterns", "comps"):
if item not in filepaths:
filepaths[item] = None
continue
if not filepaths[item]:
continue
filepaths[item] = _get_metadata_from_repo(baseurl,
proxies,
cachedir,
reponame,
filepaths[item],
sumtypes[item],
checksums[item])
""" Get repo key """
try:
repokey = _get_metadata_from_repo(baseurl,
proxies,
cachedir,
reponame,
"repodata/repomd.xml.key")
except CreatorError:
repokey = None
msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key"))
my_repo_metadata.append({"name":reponame,
"baseurl":baseurl,
"repomd":repomd,
"primary":filepaths['primary'],
"cachedir":cachedir,
"proxies":proxies,
"patterns":filepaths['patterns'],
"comps":filepaths['comps'],
"repokey":repokey})
return my_repo_metadata
def get_metadata_from_repos(repos, cachedir):
my_repo_metadata = []
for repo in repos:
reponame = repo['name']
baseurl = repo['baseurl']
if 'proxy' in repo:
proxy = repo['proxy']
else:
proxy = get_proxy_for(baseurl)
proxies = None
if proxy:
proxies = {str(baseurl.split(":")[0]): str(proxy)}
makedirs(os.path.join(cachedir, reponame))
url = os.path.join(baseurl, "repodata/repomd.xml")
filename = os.path.join(cachedir, reponame, 'repomd.xml')
repomd = myurlgrab(url, filename, proxies)
try:
root = xmlparse(repomd)
except SyntaxError:
raise CreatorError("repomd.xml syntax error.")
ns = root.getroot().tag
ns = ns[0:ns.rindex("}") + 1]
filepaths = {}
checksums = {}
sumtypes = {}
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "patterns":
filepaths['patterns'] = elm.find("%slocation" %
ns).attrib['href']
checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
sumtypes['patterns'] = elm.find("%sopen-checksum" %
ns).attrib['type']
break
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] in ("group_gz", "group"):
filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
checksums['comps'] = elm.find("%sopen-checksum" % ns).text
sumtypes['comps'] = elm.find("%sopen-checksum" %
ns).attrib['type']
break
primary_type = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] in ("primary_db", "primary"):
primary_type = elm.attrib["type"]
filepaths['primary'] = elm.find("%slocation" %
ns).attrib['href']
checksums['primary'] = elm.find("%sopen-checksum" % ns).text
sumtypes['primary'] = elm.find("%sopen-checksum" %
ns).attrib['type']
break
if not primary_type:
continue
for item in ("primary", "patterns", "comps"):
if item not in filepaths:
filepaths[item] = None
continue
if not filepaths[item]:
continue
filepaths[item] = _get_metadata_from_repo(baseurl, proxies,
cachedir, reponame,
filepaths[item],
sumtypes[item],
checksums[item])
""" Get repo key """
try:
repokey = _get_metadata_from_repo(baseurl, proxies, cachedir,
reponame,
"repodata/repomd.xml.key")
except CreatorError:
repokey = None
msger.debug("\ncan't get %s/%s" %
(baseurl, "repodata/repomd.xml.key"))
my_repo_metadata.append({
"name": reponame,
"baseurl": baseurl,
"repomd": repomd,
"primary": filepaths['primary'],
"cachedir": cachedir,
"proxies": proxies,
"patterns": filepaths['patterns'],
"comps": filepaths['comps'],
"repokey": repokey
})
return my_repo_metadata
def get_metadata_from_repos(repos, cachedir):
my_repo_metadata = []
for repo in repos:
reponame = repo['name']
baseurl = repo['baseurl']
if 'proxy' in repo:
proxy = repo['proxy']
else:
proxy = get_proxy_for(baseurl)
proxies = None
if proxy:
proxies = {str(baseurl.split(":")[0]):str(proxy)}
makedirs(os.path.join(cachedir, reponame))
url = os.path.join(baseurl, "repodata/repomd.xml")
filename = os.path.join(cachedir, reponame, 'repomd.xml')
repomd = myurlgrab(url, filename, proxies)
try:
root = xmlparse(repomd)
except SyntaxError:
raise CreatorError("repomd.xml syntax error.")
ns = root.getroot().tag
ns = ns[0:ns.rindex("}")+1]
patterns = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "patterns":
patterns = elm.find("%slocation" % ns).attrib['href']
break
comps = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "group_gz":
comps = elm.find("%slocation" % ns).attrib['href']
break
if not comps:
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "group":
comps = elm.find("%slocation" % ns).attrib['href']
break
primary_type = None
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "primary_db":
primary_type=".sqlite"
break
if not primary_type:
for elm in root.getiterator("%sdata" % ns):
if elm.attrib["type"] == "primary":
primary_type=".xml"
break
if not primary_type:
continue
primary = elm.find("%slocation" % ns).attrib['href']
primary = _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, primary)
if patterns:
patterns = _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, patterns)
if comps:
comps = _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, comps)
""" Get repo key """
try:
repokey = _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, "repodata/repomd.xml.key")
except CreatorError:
repokey = None
msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key"))
my_repo_metadata.append({"name":reponame, "baseurl":baseurl, "repomd":repomd, "primary":primary, "cachedir":cachedir, "proxies":proxies, "patterns":patterns, "comps":comps, "repokey":repokey})
return my_repo_metadata