def PUT(self, catrel_name, arch_name, osrel_name, md5_sum):
"""Adds package to a catalog.
When pycurl calls this function, it often hangs, waiting. A fix for that
is to add the 'Content-Length' header. However, it sometimes still gets
stuck and I don't know why.
"""
configuration.SetUpSqlobjectConnection()
if catrel_name != 'unstable':
# Updates via web are allowed only for the unstable catalog.
# We should return an error message instead.
raise web.notfound()
try:
if arch_name == 'all':
raise checkpkg_lib.CatalogDatabaseError(
"Cannot add to 'all' catalog.")
srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
parsed_basename = opencsw.ParsePackageFileName(srv4.basename)
if parsed_basename["vendortag"] != "CSW":
raise checkpkg_lib.CatalogDatabaseError(
"Package vendor tag is %s instead of CSW." %
parsed_basename["vendortag"])
if not srv4.registered:
# Package needs to be registered for releases
stats = srv4.GetStatsStruct()
# This can throw CatalogDatabaseError if the db user doesn't have
# enough permissions.
package_stats.PackageStats.ImportPkg(stats, True)
srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
c = checkpkg_lib.Catalog()
# See if there already is a package with that catalogname.
sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
osrel_name, arch_name, catrel_name)
res = c.GetConflictingSrv4ByCatalognameResult(
srv4, srv4.catalogname, sqo_osrel, sqo_arch, sqo_catrel)
if res.count() == 1:
# Removing old version of the package from the catalog
for pkg_in_catalog in res:
srv4_to_remove = pkg_in_catalog.srv4file
c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name,
catrel_name)
c.AddSrv4ToCatalog(srv4, osrel_name, arch_name, catrel_name)
web.header('Content-type',
'application/x-vnd.opencsw.pkg;type=catalog-update')
response = json.dumps([
u"Added to catalog %s %s %s" %
(catrel_name, arch_name, osrel_name),
u"%s" % srv4.basename,
])
web.header('Content-Length', len(response))
return response
except (checkpkg_lib.CatalogDatabaseError,
sqlobject.dberrors.OperationalError), e:
web.header('Content-type',
'application/x-vnd.opencsw.pkg;type=error-message')
response = json.dumps({
"error_message": unicode(e),
})
web.header('Content-Length', len(response))
return response
def Maintainers(pkgs):
"""Process a catalog and compile data structures with activity stats.
Args:
pkgs: a list of packages as returned by the catalog timing REST endpoint
Returns:
maintainers, bad_dates
"""
bad_dates = []
maintainers = {}
for entry in pkgs:
entry['maintainer'] = entry['maintainer'].split('@')[0]
if entry['maintainer'] in MAINTAINER_STOPLIST:
continue
parsed_fn = opencsw.ParsePackageFileName(entry['basename'])
dates_to_try = []
if 'REV' in parsed_fn['revision_info']:
dates_to_try.append(parsed_fn['revision_info']['REV'])
else:
logging.warning('{catalogname} did not have a REV=. '
'Falling back to mtime.'.format(**entry))
dates_to_try.append(entry['mtime'])
for date_str in dates_to_try:
try:
date = dateutil.parser.parse(date_str)
break
except ValueError as exc:
logging.warning(exc)
logging.warning(
"WTF is {date} in {catalogname}? "
"Go home {maintainer}, you're drunk.".format(
catalogname=entry['catalogname'],
maintainer=entry['maintainer'],
date=date_str))
bad_dates.append(date_str)
continue
entry['date'] = date
maintainer = maintainers.setdefault(entry['maintainer'],
Maintainer(username=entry['maintainer'], pkgs={},
last_activity=datetime.datetime(1970, 1, 1, 0, 0),
last_activity_pkg=None,
active=True,
csw_db_status=None,
fullname=None,
date_created=None))
if entry['catalogname'] not in maintainer.pkgs:
maintainer.pkgs[entry['catalogname']] = entry
if maintainer.last_activity < date:
maintainer = maintainer._replace(last_activity=date)
maintainer = maintainer._replace(last_activity_pkg=entry)
maintainers[maintainer.username] = maintainer
maintainers = ComputeMaintainerActivity(maintainers)
return maintainers, bad_dates
def GetBasicStats(self):
basic_stats = {}
basic_stats["stats_version"] = self.STATS_VERSION
basic_stats["pkg_path"] = self.pkg_path
basic_stats["pkg_basename"] = os.path.basename(self.pkg_path)
basic_stats["parsed_basename"] = opencsw.ParsePackageFileName(
basic_stats["pkg_basename"])
basic_stats["pkgname"] = self.pkgname
basic_stats["catalogname"] = self.GetCatalogname()
basic_stats["md5_sum"] = self.md5_sum
basic_stats["size"] = self.size
return basic_stats
def main():
parser = argparse.ArgumentParser()
parser.add_argument('output', help='Output file')
args = parser.parse_args()
url = ('http://buildfarm.opencsw.org/pkgdb/rest/catalogs/'
'unstable/i386/SunOS5.10/timing/')
data = requests.get(url).json()
with open(args.output, 'wb') as fd:
fd.write('catalogname maintainer date\n')
bad_dates = []
for entry in data:
entry['maintainer'] = entry['maintainer'].split('@')[0]
parsed_fn = opencsw.ParsePackageFileName(entry['basename'])
dates_to_try = []
if 'REV' in parsed_fn['revision_info']:
dates_to_try.append(parsed_fn['revision_info']['REV'])
else:
logging.warning('{catalogname} did not have a REV=. '
'Falling back to mtime.'.format(**entry))
dates_to_try.append(entry['mtime'])
for date_str in dates_to_try:
try:
date = dateutil.parser.parse(date_str)
break
except ValueError as exc:
logging.warning(exc)
logging.warning(
"WTF is {date} in {catalogname}? "
"Go home {maintainer}, you're drunk.".format(
catalogname=entry['catalogname'],
maintainer=entry['maintainer'],
date=date_str))
bad_dates.append(date_str)
continue
entry['date'] = date.strftime('%Y-%m-%d')
line = '{catalogname} {maintainer} {date}\n'.format(**entry)
fd.write(line)
if bad_dates:
logging.warning('Bad dates encountered. mtime used as fallback.')