def testParsePackageFileName_OneDashTooMany(self):
file_name = ("boost-jam-3.1.18,REV=2010.12.15-"
"SunOS5.9-sparc-UNCOMMITTED.pkg.gz")
parsed = opencsw.ParsePackageFileName(file_name)
self.assertEqual(parsed["arch"], "sparc")
self.assertEqual(parsed["catalogname"], "boost-jam")
self.assertEqual(parsed["vendortag"], "UNCOMMITTED")
def _RemoveFromCatalog(self, filename, arch, osrel, file_metadata):
print("Removing %s (%s %s) from catalog %s %s %s" %
(file_metadata["catalogname"], file_metadata["arch"],
file_metadata["osrel"], DEFAULT_CATREL, arch, osrel))
md5_sum = self._GetFileMd5sum(filename)
basename = os.path.basename(filename)
parsed_basename = opencsw.ParsePackageFileName(basename)
# TODO: Move this bit to a separate class (RestClient)
url = ("%s%s/catalogs/%s/%s/%s/%s/" %
(self.rest_url, RELEASES_APP, DEFAULT_CATREL, arch, osrel,
md5_sum))
logging.debug("DELETE @ URL: %s %s", type(url), url)
c = pycurl.Curl()
d = StringIO()
h = StringIO()
c.setopt(pycurl.URL, str(url))
c.setopt(pycurl.CUSTOMREQUEST, "DELETE")
c.setopt(pycurl.WRITEFUNCTION, d.write)
c.setopt(pycurl.HEADERFUNCTION, h.write)
c.setopt(pycurl.HTTPHEADER, ["Expect:"]) # Fixes the HTTP 417 error
if self.debug:
c.setopt(c.VERBOSE, 1)
c.perform()
http_code = c.getinfo(pycurl.HTTP_CODE)
logging.debug("DELETE curl getinfo: %s %s %s", type(http_code),
http_code, c.getinfo(pycurl.EFFECTIVE_URL))
c.close()
if http_code >= 400 and http_code <= 499:
raise RestCommunicationError("%s - HTTP code: %s" %
(url, http_code))
def __init__(self, pkg_path, debug):
self.debug = debug
self.pkg_path = pkg_path
self.srv4 = CswSrv4File(pkg_path)
self.dir_pkg = None
self.exported_dir = None
self.parsed_filename = opencsw.ParsePackageFileName(self.pkg_path)
def _InsertIntoCatalog(self, filename, arch, osrel, file_metadata):
logging.debug(
"_InsertIntoCatalog(%s, %s, %s)",
repr(arch), repr(osrel), repr(filename))
print("Inserting %s (%s %s) into catalog %s %s %s"
% (file_metadata["catalogname"],
file_metadata["arch"],
file_metadata["osrel"],
DEFAULT_CATREL, arch, osrel))
md5_sum = self._GetFileMd5sum(filename)
basename = os.path.basename(filename)
parsed_basename = opencsw.ParsePackageFileName(basename)
logging.debug("parsed_basename: %s", parsed_basename)
url = (
"%s%s/catalogs/%s/%s/%s/%s/"
% (self.rest_url,
RELEASES_APP,
DEFAULT_CATREL,
arch,
osrel,
md5_sum))
logging.debug("URL: %s %s", type(url), url)
c = pycurl.Curl()
d = StringIO()
h = StringIO()
# Bogus data to upload
s = StringIO()
c.setopt(pycurl.URL, str(url))
c.setopt(pycurl.PUT, 1)
c.setopt(pycurl.UPLOAD, 1)
c.setopt(pycurl.INFILESIZE_LARGE, s.len)
c.setopt(pycurl.READFUNCTION, s.read)
c.setopt(pycurl.WRITEFUNCTION, d.write)
c.setopt(pycurl.HEADERFUNCTION, h.write)
c.setopt(pycurl.HTTPHEADER, ["Expect:"]) # Fixes the HTTP 417 error
c = self._SetAuth(c)
if self.debug:
c.setopt(c.VERBOSE, 1)
c.perform()
http_code = c.getinfo(pycurl.HTTP_CODE)
logging.debug(
"curl getinfo: %s %s %s",
type(http_code),
http_code,
c.getinfo(pycurl.EFFECTIVE_URL))
c.close()
# if self.debug:
# logging.debug("*** Headers")
# logging.debug(h.getvalue())
# logging.debug("*** Data")
if http_code >= 400 and http_code <= 599:
if not self.debug:
# In debug mode, all headers are printed to screen, and we aren't
# interested in the response body.
logging.fatal("Response: %s %s", http_code, d.getvalue())
raise RestCommunicationError("%s - HTTP code: %s" % (url, http_code))
else:
logging.debug("Response: %s %s", http_code, d.getvalue())
return http_code
def testParsePackageFileName_RichpSe(self):
file_name = "RICHPse-3.5.1.pkg.gz"
parsed = opencsw.ParsePackageFileName(file_name)
self.assertEqual(parsed["version"], "3.5.1")
self.assertEqual(parsed["vendortag"], "UNKN")
self.assertEqual(parsed["arch"], "unknown")
self.assertEqual(parsed["osrel"], "unspecified")
self.assertEqual(parsed["catalogname"], "RICHPse")
def _InsertIntoCatalog(self, filename, arch, osrel, file_metadata):
logging.debug("_InsertIntoCatalog(%s, %s, %s)", repr(arch),
repr(osrel), repr(filename))
print("Inserting %s (%s %s) into catalog %s %s %s" %
(file_metadata["catalogname"], file_metadata["arch"],
file_metadata["osrel"], self.catrel, arch, osrel))
md5_sum = self._GetFileMd5sum(filename)
basename = os.path.basename(filename)
parsed_basename = opencsw.ParsePackageFileName(basename)
logging.debug("parsed_basename: %s", parsed_basename)
return self._rest_client.AddSvr4ToCatalog(self.catrel, arch, osrel,
md5_sum)
def GetBasicStats(self):
dir_pkg = self.GetInspectivePkg()
basic_stats = {}
basic_stats["stats_version"] = PACKAGE_STATS_VERSION
basic_stats["pkg_path"] = self.srv4_pkg.pkg_path
basic_stats["pkg_basename"] = os.path.basename(self.srv4_pkg.pkg_path)
basic_stats["parsed_basename"] = opencsw.ParsePackageFileName(
basic_stats["pkg_basename"])
basic_stats["pkgname"] = dir_pkg.pkgname
basic_stats["catalogname"] = dir_pkg.GetCatalogname()
basic_stats["md5_sum"] = self.GetMd5sum()
basic_stats["size"] = self.GetSize()
return basic_stats
def _FilesWithMetadata(self, file_list):
files_with_metadata = []
for file_path in file_list:
pkg_path, basename = os.path.split(file_path)
parsed = opencsw.ParsePackageFileName(basename)
catalogname = parsed["catalogname"]
files_with_metadata.append((basename, parsed))
if parsed["arch"] == "all":
archs = common_constants.PHYSICAL_ARCHITECTURES
else:
archs = [parsed["arch"]]
for arch in archs:
for osrel in common_constants.OS_RELS:
key = arch, osrel
return files_with_metadata
def testParsePackageFileName1(self):
test_data = open(os.path.join(os.path.split(__file__)[0],
"testdata/example-catalog.txt"))
split_re = re.compile(r"\s+")
for line in test_data:
fields = re.split(split_re, line)
catalogname = fields[0]
pkg_version = fields[1]
pkgname = fields[2]
file_name = fields[3]
pkg_md5 = fields[4]
pkg_size = fields[5]
depends_on = fields[6]
compiled = opencsw.ParsePackageFileName(file_name)
self.assertTrue(compiled, "File name %s did not compile" % repr(file_name))
self.assertEqual(catalogname, compiled["catalogname"])
self.assertEqual(pkg_version, compiled["full_version_string"])
def CheckFiles(self, file_list):
"""Checks a set of files. Returns error tags."""
files_with_metadata = []
for file_path in file_list:
pkg_path, basename = os.path.split(file_path)
parsed = opencsw.ParsePackageFileName(basename)
catalogname = parsed["catalogname"]
files_with_metadata.append((basename, parsed))
if parsed["arch"] == "all":
archs = common_constants.PHYSICAL_ARCHITECTURES
else:
archs = [parsed["arch"]]
for arch in archs:
for osrel in common_constants.OS_RELS:
key = arch, osrel
# catalognames_by_arch.setdefault(key, set()).add(catalogname)
tags = []
tags.extend(self._CheckMissingArchs(files_with_metadata))
tags.extend(self._CheckUncommitted(files_with_metadata))
return tags
def SortFilenames(self, filenames):
"""Sorts filenames according to OS release.
The idea is that in lexicographic sorting, SunOS5.9 is after
SunOS5.10, while we want 5.9 to be first.
"""
by_osrel = {}
sorted_filenames = []
for filename in filenames:
basename = os.path.basename(filename)
parsed_basename = opencsw.ParsePackageFileName(basename)
by_osrel.setdefault(parsed_basename["osrel"], []).append(filename)
for osrel in common_constants.OS_RELS:
if osrel in by_osrel:
for filename in by_osrel.pop(osrel):
sorted_filenames.append(filename)
if by_osrel:
raise DataError("Unexpected architecture found in file list: %s." %
(repr(by_osrel), ))
return sorted_filenames
def testParsePackageFileName_OldFormat(self):
"""Old srv4 file name."""
file_name = "achievo-0.8.4-all-CSW.pkg.gz"
parsed = opencsw.ParsePackageFileName(file_name)
self.assertEqual("unspecified", parsed["osrel"])
def setUp(self):
self.file_name = 'mysql5client-5.0.87,REV=2010.02.28-SunOS5.8-i386-CSW.pkg.gz'
self.parsed = opencsw.ParsePackageFileName(self.file_name)
def testParsePackageFileName_Nonsense(self):
"""Checks if the function can sustain a non-conformant string."""
file_name = "What if I wrote a letter to my grandma here?"
parsed = opencsw.ParsePackageFileName(file_name)
def _MatchSrv4ToCatalogs(self, filename, catrel, srv4_arch, srv4_osrel,
md5_sum):
"""Compile a list of catalogs affected by the given file.
If it's a 5.9 package, it can be matched to 5.9, 5.10 and 5.11. However,
if there already are specific 5.10 and/or 5.11 versions of the package,
don't overwrite them.
Args:
filename: string, base file name of the srv4 stream file
catrel: string denoting catalog release, usually 'unstable'
srv4_arch: string, denoting srv4 file architecture (sparc, i386 or all)
srv4_osrel: string, OS release of the package, e.g. 'SunOS5.9'
md5_sum: string, md5 sum of the srv4 file
Returns:
A tuple of tuples, where each tuple is: (catrel, arch, osrel)
"""
basename = os.path.basename(filename)
parsed_basename = opencsw.ParsePackageFileName(basename)
osrels = None
for idx, known_osrel in enumerate(common_constants.OS_RELS):
if srv4_osrel == known_osrel:
osrels = common_constants.OS_RELS[idx:]
assert osrels, "OS releases not found"
if srv4_arch == 'all':
archs = ('sparc', 'i386')
else:
archs = (srv4_arch, )
catalogname = parsed_basename["catalogname"]
# This part of code quickly became complicated and should probably be
# rewritten. However, it is unit tested, so all the known cases are
# handled as intended.
catalogs = []
first_cat_osrel_seen = None
for arch in archs:
for osrel in osrels:
logging.debug("%s %s %s", catrel, arch, osrel)
cat_key = (catrel, arch, osrel)
try:
srv4_in_catalog = self._rest_client.Srv4ByCatalogAndCatalogname(
catrel, arch, osrel, catalogname)
except urllib2.HTTPError, e:
srv4_in_catalog = None
if srv4_in_catalog:
logging.debug(
"Catalog %s %s contains version %s of the %s package",
arch, osrel, srv4_in_catalog["osrel"], catalogname)
else:
logging.debug(
"Catalog %s %s does not contain any version of the %s package.",
arch, osrel, catalogname)
if not first_cat_osrel_seen:
if srv4_in_catalog:
first_cat_osrel_seen = srv4_in_catalog["osrel"]
else:
first_cat_osrel_seen = srv4_osrel
logging.debug("Considering %s the base OS to match",
first_cat_osrel_seen)
if (not srv4_in_catalog
or srv4_in_catalog["osrel"] == srv4_osrel
or srv4_in_catalog["osrel"] == first_cat_osrel_seen):
# The same architecture as our package, meaning that we can insert
# the same architecture into the catalog.
if (not self.os_release
or (self.os_release and osrel == self.os_release)):
catalogs.append(cat_key)
else:
if self.os_release and osrel == self.os_release:
logging.debug("OS release specified and matches %s.",
osrel)
catalogs.append(cat_key)
else:
logging.debug(
"Not matching %s %s package with %s containing a %s package",
catalogname, srv4_osrel, osrel,
srv4_in_catalog["osrel"])
logging.debug(
"Catalog %s %s %s has another version of %s.", catrel,
arch, osrel, catalogname)
def testParsePackageFileName_Achievo(self):
file_name = ("achievo-0.8.4-all-CSW.pkg.gz")
parsed = opencsw.ParsePackageFileName(file_name)
self.assertEqual(parsed["arch"], "all")
self.assertEqual(parsed["catalogname"], "achievo")
self.assertEqual(parsed["osrel"], "unspecified")
sys.exit(1)
staging_dir = opencsw.StagingDir(config.get(CONFIG_RELEASE_SECTION,
"package dir"))
if options.catalognames:
catalognames = options.catalognames.split(",")
else:
catalognames = []
if file_names:
for file_name in file_names:
base_name = os.path.basename(file_name)
if base_name != file_name:
logging.warn("Removing %s, using only %s"
% (repr(os.path.dirname(file_name)), repr(base_name)))
logging.warn("Only %s will be searched for packages."
% repr(staging_dir))
parsed_file_name = opencsw.ParsePackageFileName(base_name)
catalognames.append(parsed_file_name["catalogname"])
catalognames = sorted(set(catalognames))
if not catalognames:
parser.print_help()
raise ConfigurationError("You need to specify a package name or names.")
package_files = []
for p in catalognames:
package_files.extend(staging_dir.GetLatest(p))
logging.debug("Copying files to the target host:dir")
remote_package_files = []
remote_package_references = []
files_to_rsync = []
target_host = config.get(CONFIG_RELEASE_SECTION, "target host")
target_dir = config.get(CONFIG_RELEASE_SECTION, "target dir")
