def testParsePackageFileName_OneDashTooMany(self): file_name = ("boost-jam-3.1.18,REV=2010.12.15-" "SunOS5.9-sparc-UNCOMMITTED.pkg.gz") parsed = opencsw.ParsePackageFileName(file_name) self.assertEqual(parsed["arch"], "sparc") self.assertEqual(parsed["catalogname"], "boost-jam") self.assertEqual(parsed["vendortag"], "UNCOMMITTED")
def _RemoveFromCatalog(self, filename, arch, osrel, file_metadata): print("Removing %s (%s %s) from catalog %s %s %s" % (file_metadata["catalogname"], file_metadata["arch"], file_metadata["osrel"], DEFAULT_CATREL, arch, osrel)) md5_sum = self._GetFileMd5sum(filename) basename = os.path.basename(filename) parsed_basename = opencsw.ParsePackageFileName(basename) # TODO: Move this bit to a separate class (RestClient) url = ("%s%s/catalogs/%s/%s/%s/%s/" % (self.rest_url, RELEASES_APP, DEFAULT_CATREL, arch, osrel, md5_sum)) logging.debug("DELETE @ URL: %s %s", type(url), url) c = pycurl.Curl() d = StringIO() h = StringIO() c.setopt(pycurl.URL, str(url)) c.setopt(pycurl.CUSTOMREQUEST, "DELETE") c.setopt(pycurl.WRITEFUNCTION, d.write) c.setopt(pycurl.HEADERFUNCTION, h.write) c.setopt(pycurl.HTTPHEADER, ["Expect:"]) # Fixes the HTTP 417 error if self.debug: c.setopt(c.VERBOSE, 1) c.perform() http_code = c.getinfo(pycurl.HTTP_CODE) logging.debug("DELETE curl getinfo: %s %s %s", type(http_code), http_code, c.getinfo(pycurl.EFFECTIVE_URL)) c.close() if http_code >= 400 and http_code <= 499: raise RestCommunicationError("%s - HTTP code: %s" % (url, http_code))
def __init__(self, pkg_path, debug): self.debug = debug self.pkg_path = pkg_path self.srv4 = CswSrv4File(pkg_path) self.dir_pkg = None self.exported_dir = None self.parsed_filename = opencsw.ParsePackageFileName(self.pkg_path)
def _InsertIntoCatalog(self, filename, arch, osrel, file_metadata): logging.debug( "_InsertIntoCatalog(%s, %s, %s)", repr(arch), repr(osrel), repr(filename)) print("Inserting %s (%s %s) into catalog %s %s %s" % (file_metadata["catalogname"], file_metadata["arch"], file_metadata["osrel"], DEFAULT_CATREL, arch, osrel)) md5_sum = self._GetFileMd5sum(filename) basename = os.path.basename(filename) parsed_basename = opencsw.ParsePackageFileName(basename) logging.debug("parsed_basename: %s", parsed_basename) url = ( "%s%s/catalogs/%s/%s/%s/%s/" % (self.rest_url, RELEASES_APP, DEFAULT_CATREL, arch, osrel, md5_sum)) logging.debug("URL: %s %s", type(url), url) c = pycurl.Curl() d = StringIO() h = StringIO() # Bogus data to upload s = StringIO() c.setopt(pycurl.URL, str(url)) c.setopt(pycurl.PUT, 1) c.setopt(pycurl.UPLOAD, 1) c.setopt(pycurl.INFILESIZE_LARGE, s.len) c.setopt(pycurl.READFUNCTION, s.read) c.setopt(pycurl.WRITEFUNCTION, d.write) c.setopt(pycurl.HEADERFUNCTION, h.write) c.setopt(pycurl.HTTPHEADER, ["Expect:"]) # Fixes the HTTP 417 error c = self._SetAuth(c) if self.debug: c.setopt(c.VERBOSE, 1) c.perform() http_code = c.getinfo(pycurl.HTTP_CODE) logging.debug( "curl getinfo: %s %s %s", type(http_code), http_code, c.getinfo(pycurl.EFFECTIVE_URL)) c.close() # if self.debug: # logging.debug("*** Headers") # logging.debug(h.getvalue()) # logging.debug("*** Data") if http_code >= 400 and http_code <= 599: if not self.debug: # In debug mode, all headers are printed to screen, and we aren't # interested in the response body. logging.fatal("Response: %s %s", http_code, d.getvalue()) raise RestCommunicationError("%s - HTTP code: %s" % (url, http_code)) else: logging.debug("Response: %s %s", http_code, d.getvalue()) return http_code
def testParsePackageFileName_RichpSe(self): file_name = "RICHPse-3.5.1.pkg.gz" parsed = opencsw.ParsePackageFileName(file_name) self.assertEqual(parsed["version"], "3.5.1") self.assertEqual(parsed["vendortag"], "UNKN") self.assertEqual(parsed["arch"], "unknown") self.assertEqual(parsed["osrel"], "unspecified") self.assertEqual(parsed["catalogname"], "RICHPse")
def _InsertIntoCatalog(self, filename, arch, osrel, file_metadata): logging.debug("_InsertIntoCatalog(%s, %s, %s)", repr(arch), repr(osrel), repr(filename)) print("Inserting %s (%s %s) into catalog %s %s %s" % (file_metadata["catalogname"], file_metadata["arch"], file_metadata["osrel"], self.catrel, arch, osrel)) md5_sum = self._GetFileMd5sum(filename) basename = os.path.basename(filename) parsed_basename = opencsw.ParsePackageFileName(basename) logging.debug("parsed_basename: %s", parsed_basename) return self._rest_client.AddSvr4ToCatalog(self.catrel, arch, osrel, md5_sum)
def GetBasicStats(self): dir_pkg = self.GetInspectivePkg() basic_stats = {} basic_stats["stats_version"] = PACKAGE_STATS_VERSION basic_stats["pkg_path"] = self.srv4_pkg.pkg_path basic_stats["pkg_basename"] = os.path.basename(self.srv4_pkg.pkg_path) basic_stats["parsed_basename"] = opencsw.ParsePackageFileName( basic_stats["pkg_basename"]) basic_stats["pkgname"] = dir_pkg.pkgname basic_stats["catalogname"] = dir_pkg.GetCatalogname() basic_stats["md5_sum"] = self.GetMd5sum() basic_stats["size"] = self.GetSize() return basic_stats
def _FilesWithMetadata(self, file_list): files_with_metadata = [] for file_path in file_list: pkg_path, basename = os.path.split(file_path) parsed = opencsw.ParsePackageFileName(basename) catalogname = parsed["catalogname"] files_with_metadata.append((basename, parsed)) if parsed["arch"] == "all": archs = common_constants.PHYSICAL_ARCHITECTURES else: archs = [parsed["arch"]] for arch in archs: for osrel in common_constants.OS_RELS: key = arch, osrel return files_with_metadata
def testParsePackageFileName1(self): test_data = open(os.path.join(os.path.split(__file__)[0], "testdata/example-catalog.txt")) split_re = re.compile(r"\s+") for line in test_data: fields = re.split(split_re, line) catalogname = fields[0] pkg_version = fields[1] pkgname = fields[2] file_name = fields[3] pkg_md5 = fields[4] pkg_size = fields[5] depends_on = fields[6] compiled = opencsw.ParsePackageFileName(file_name) self.assertTrue(compiled, "File name %s did not compile" % repr(file_name)) self.assertEqual(catalogname, compiled["catalogname"]) self.assertEqual(pkg_version, compiled["full_version_string"])
def CheckFiles(self, file_list): """Checks a set of files. Returns error tags.""" files_with_metadata = [] for file_path in file_list: pkg_path, basename = os.path.split(file_path) parsed = opencsw.ParsePackageFileName(basename) catalogname = parsed["catalogname"] files_with_metadata.append((basename, parsed)) if parsed["arch"] == "all": archs = common_constants.PHYSICAL_ARCHITECTURES else: archs = [parsed["arch"]] for arch in archs: for osrel in common_constants.OS_RELS: key = arch, osrel # catalognames_by_arch.setdefault(key, set()).add(catalogname) tags = [] tags.extend(self._CheckMissingArchs(files_with_metadata)) tags.extend(self._CheckUncommitted(files_with_metadata)) return tags
def SortFilenames(self, filenames): """Sorts filenames according to OS release. The idea is that in lexicographic sorting, SunOS5.9 is after SunOS5.10, while we want 5.9 to be first. """ by_osrel = {} sorted_filenames = [] for filename in filenames: basename = os.path.basename(filename) parsed_basename = opencsw.ParsePackageFileName(basename) by_osrel.setdefault(parsed_basename["osrel"], []).append(filename) for osrel in common_constants.OS_RELS: if osrel in by_osrel: for filename in by_osrel.pop(osrel): sorted_filenames.append(filename) if by_osrel: raise DataError("Unexpected architecture found in file list: %s." % (repr(by_osrel), )) return sorted_filenames
def testParsePackageFileName_OldFormat(self): """Old srv4 file name.""" file_name = "achievo-0.8.4-all-CSW.pkg.gz" parsed = opencsw.ParsePackageFileName(file_name) self.assertEqual("unspecified", parsed["osrel"])
def setUp(self): self.file_name = 'mysql5client-5.0.87,REV=2010.02.28-SunOS5.8-i386-CSW.pkg.gz' self.parsed = opencsw.ParsePackageFileName(self.file_name)
def testParsePackageFileName_Nonsense(self): """Checks if the function can sustain a non-conformant string.""" file_name = "What if I wrote a letter to my grandma here?" parsed = opencsw.ParsePackageFileName(file_name)
def _MatchSrv4ToCatalogs(self, filename, catrel, srv4_arch, srv4_osrel, md5_sum): """Compile a list of catalogs affected by the given file. If it's a 5.9 package, it can be matched to 5.9, 5.10 and 5.11. However, if there already are specific 5.10 and/or 5.11 versions of the package, don't overwrite them. Args: filename: string, base file name of the srv4 stream file catrel: string denoting catalog release, usually 'unstable' srv4_arch: string, denoting srv4 file architecture (sparc, i386 or all) srv4_osrel: string, OS release of the package, e.g. 'SunOS5.9' md5_sum: string, md5 sum of the srv4 file Returns: A tuple of tuples, where each tuple is: (catrel, arch, osrel) """ basename = os.path.basename(filename) parsed_basename = opencsw.ParsePackageFileName(basename) osrels = None for idx, known_osrel in enumerate(common_constants.OS_RELS): if srv4_osrel == known_osrel: osrels = common_constants.OS_RELS[idx:] assert osrels, "OS releases not found" if srv4_arch == 'all': archs = ('sparc', 'i386') else: archs = (srv4_arch, ) catalogname = parsed_basename["catalogname"] # This part of code quickly became complicated and should probably be # rewritten. However, it is unit tested, so all the known cases are # handled as intended. catalogs = [] first_cat_osrel_seen = None for arch in archs: for osrel in osrels: logging.debug("%s %s %s", catrel, arch, osrel) cat_key = (catrel, arch, osrel) try: srv4_in_catalog = self._rest_client.Srv4ByCatalogAndCatalogname( catrel, arch, osrel, catalogname) except urllib2.HTTPError, e: srv4_in_catalog = None if srv4_in_catalog: logging.debug( "Catalog %s %s contains version %s of the %s package", arch, osrel, srv4_in_catalog["osrel"], catalogname) else: logging.debug( "Catalog %s %s does not contain any version of the %s package.", arch, osrel, catalogname) if not first_cat_osrel_seen: if srv4_in_catalog: first_cat_osrel_seen = srv4_in_catalog["osrel"] else: first_cat_osrel_seen = srv4_osrel logging.debug("Considering %s the base OS to match", first_cat_osrel_seen) if (not srv4_in_catalog or srv4_in_catalog["osrel"] == srv4_osrel or srv4_in_catalog["osrel"] == first_cat_osrel_seen): # The same architecture as our package, meaning that we can insert # the same architecture into the catalog. if (not self.os_release or (self.os_release and osrel == self.os_release)): catalogs.append(cat_key) else: if self.os_release and osrel == self.os_release: logging.debug("OS release specified and matches %s.", osrel) catalogs.append(cat_key) else: logging.debug( "Not matching %s %s package with %s containing a %s package", catalogname, srv4_osrel, osrel, srv4_in_catalog["osrel"]) logging.debug( "Catalog %s %s %s has another version of %s.", catrel, arch, osrel, catalogname)
def testParsePackageFileName_Achievo(self): file_name = ("achievo-0.8.4-all-CSW.pkg.gz") parsed = opencsw.ParsePackageFileName(file_name) self.assertEqual(parsed["arch"], "all") self.assertEqual(parsed["catalogname"], "achievo") self.assertEqual(parsed["osrel"], "unspecified")
sys.exit(1) staging_dir = opencsw.StagingDir(config.get(CONFIG_RELEASE_SECTION, "package dir")) if options.catalognames: catalognames = options.catalognames.split(",") else: catalognames = [] if file_names: for file_name in file_names: base_name = os.path.basename(file_name) if base_name != file_name: logging.warn("Removing %s, using only %s" % (repr(os.path.dirname(file_name)), repr(base_name))) logging.warn("Only %s will be searched for packages." % repr(staging_dir)) parsed_file_name = opencsw.ParsePackageFileName(base_name) catalognames.append(parsed_file_name["catalogname"]) catalognames = sorted(set(catalognames)) if not catalognames: parser.print_help() raise ConfigurationError("You need to specify a package name or names.") package_files = [] for p in catalognames: package_files.extend(staging_dir.GetLatest(p)) logging.debug("Copying files to the target host:dir") remote_package_files = [] remote_package_references = [] files_to_rsync = [] target_host = config.get(CONFIG_RELEASE_SECTION, "target host") target_dir = config.get(CONFIG_RELEASE_SECTION, "target dir")