def PUT(self, catrel_name, arch_name, osrel_name, md5_sum): """Adds package to a catalog. When pycurl calls this function, it often hangs, waiting. A fix for that is to add the 'Content-Length' header. However, it sometimes still gets stuck and I don't know why. """ configuration.SetUpSqlobjectConnection() if catrel_name != 'unstable': # Updates via web are allowed only for the unstable catalog. # We should return an error message instead. raise web.notfound() try: if arch_name == 'all': raise checkpkg_lib.CatalogDatabaseError( "Cannot add to 'all' catalog.") srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne() parsed_basename = opencsw.ParsePackageFileName(srv4.basename) if parsed_basename["vendortag"] != "CSW": raise checkpkg_lib.CatalogDatabaseError( "Package vendor tag is %s instead of CSW." % parsed_basename["vendortag"]) if not srv4.registered: # Package needs to be registered for releases stats = srv4.GetStatsStruct() # This can throw CatalogDatabaseError if the db user doesn't have # enough permissions. package_stats.PackageStats.ImportPkg(stats, True) srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne() c = checkpkg_lib.Catalog() # See if there already is a package with that catalogname. sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad( osrel_name, arch_name, catrel_name) res = c.GetConflictingSrv4ByCatalognameResult( srv4, srv4.catalogname, sqo_osrel, sqo_arch, sqo_catrel) if res.count() == 1: # Removing old version of the package from the catalog for pkg_in_catalog in res: srv4_to_remove = pkg_in_catalog.srv4file c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name) c.AddSrv4ToCatalog(srv4, osrel_name, arch_name, catrel_name) web.header('Content-type', 'application/x-vnd.opencsw.pkg;type=catalog-update') response = json.dumps([ u"Added to catalog %s %s %s" % (catrel_name, arch_name, osrel_name), u"%s" % srv4.basename, ]) web.header('Content-Length', len(response)) return response except (checkpkg_lib.CatalogDatabaseError, sqlobject.dberrors.OperationalError), e: web.header('Content-type', 'application/x-vnd.opencsw.pkg;type=error-message') response = json.dumps({ "error_message": unicode(e), }) web.header('Content-Length', len(response)) return response
def Maintainers(pkgs): """Process a catalog and compile data structures with activity stats. Args: pkgs: a list of packages as returned by the catalog timing REST endpoint Returns: maintainers, bad_dates """ bad_dates = [] maintainers = {} for entry in pkgs: entry['maintainer'] = entry['maintainer'].split('@')[0] if entry['maintainer'] in MAINTAINER_STOPLIST: continue parsed_fn = opencsw.ParsePackageFileName(entry['basename']) dates_to_try = [] if 'REV' in parsed_fn['revision_info']: dates_to_try.append(parsed_fn['revision_info']['REV']) else: logging.warning('{catalogname} did not have a REV=. ' 'Falling back to mtime.'.format(**entry)) dates_to_try.append(entry['mtime']) for date_str in dates_to_try: try: date = dateutil.parser.parse(date_str) break except ValueError as exc: logging.warning(exc) logging.warning( "WTF is {date} in {catalogname}? " "Go home {maintainer}, you're drunk.".format( catalogname=entry['catalogname'], maintainer=entry['maintainer'], date=date_str)) bad_dates.append(date_str) continue entry['date'] = date maintainer = maintainers.setdefault(entry['maintainer'], Maintainer(username=entry['maintainer'], pkgs={}, last_activity=datetime.datetime(1970, 1, 1, 0, 0), last_activity_pkg=None, active=True, csw_db_status=None, fullname=None, date_created=None)) if entry['catalogname'] not in maintainer.pkgs: maintainer.pkgs[entry['catalogname']] = entry if maintainer.last_activity < date: maintainer = maintainer._replace(last_activity=date) maintainer = maintainer._replace(last_activity_pkg=entry) maintainers[maintainer.username] = maintainer maintainers = ComputeMaintainerActivity(maintainers) return maintainers, bad_dates
def GetBasicStats(self): basic_stats = {} basic_stats["stats_version"] = self.STATS_VERSION basic_stats["pkg_path"] = self.pkg_path basic_stats["pkg_basename"] = os.path.basename(self.pkg_path) basic_stats["parsed_basename"] = opencsw.ParsePackageFileName( basic_stats["pkg_basename"]) basic_stats["pkgname"] = self.pkgname basic_stats["catalogname"] = self.GetCatalogname() basic_stats["md5_sum"] = self.md5_sum basic_stats["size"] = self.size return basic_stats
def main(): parser = argparse.ArgumentParser() parser.add_argument('output', help='Output file') args = parser.parse_args() url = ('http://buildfarm.opencsw.org/pkgdb/rest/catalogs/' 'unstable/i386/SunOS5.10/timing/') data = requests.get(url).json() with open(args.output, 'wb') as fd: fd.write('catalogname maintainer date\n') bad_dates = [] for entry in data: entry['maintainer'] = entry['maintainer'].split('@')[0] parsed_fn = opencsw.ParsePackageFileName(entry['basename']) dates_to_try = [] if 'REV' in parsed_fn['revision_info']: dates_to_try.append(parsed_fn['revision_info']['REV']) else: logging.warning('{catalogname} did not have a REV=. ' 'Falling back to mtime.'.format(**entry)) dates_to_try.append(entry['mtime']) for date_str in dates_to_try: try: date = dateutil.parser.parse(date_str) break except ValueError as exc: logging.warning(exc) logging.warning( "WTF is {date} in {catalogname}? " "Go home {maintainer}, you're drunk.".format( catalogname=entry['catalogname'], maintainer=entry['maintainer'], date=date_str)) bad_dates.append(date_str) continue entry['date'] = date.strftime('%Y-%m-%d') line = '{catalogname} {maintainer} {date}\n'.format(**entry) fd.write(line) if bad_dates: logging.warning('Bad dates encountered. mtime used as fallback.')