def index(self):
db = None
try:
db = DBInstance(app_globals.pool)
c.names = link_quote_array(app_globals.shm.archives_get_list(db))
c.srcstarts = link_quote_array(app_globals.shm.packages_get_name_starts(db))
c.binstarts = link_quote_array(app_globals.shm.packages_get_name_starts(db, get_binary=True))
set_expires(int(config['app_conf']['expires.root']))
return render('/root.mako')
finally:
if not db is None: db.close()
def trace(self):
db = None
try:
db = DBInstance(g.pool)
set_expires(int(config['app_conf']['expires.root']))
last = g.shm.get_last_mirrorrun(db)
content = []
content.append("%s\n"%(last.ctime()))
content.append("# Above timestamp is timestamp of latest mirrrorrun\n")
response.content_type = 'text/plain'
return content
finally:
if not db is None: db.close()
def _db(self):
if self.db is None:
self.db = DBInstance(g.pool)
return self.db
class RemovalController(BaseController):
db = None
def _db(self):
if self.db is None:
self.db = DBInstance(g.pool)
return self.db
def _db_close(self):
if not self.db is None:
self.db.close()
def _build_crumbs(self, entry=None):
crumbs = []
url = urllib.quote(request.environ.get('SCRIPT_NAME')) + "/"
crumbs.append( { 'url': url, 'name': g.domain, 'sep': '|' });
url += 'removal/'
crumbs.append( { 'url': url, 'name': 'removal' });
if not entry is None:
entry=str(entry)
url += urllib.quote(entry)
crumbs.append( { 'url': url, 'name': entry, 'sep': '' });
crumbs[-1]['url'] = None
return crumbs
def root(self):
try:
set_expires(int(config['app_conf']['expires.removal']))
removals = g.shm.removal_get_list(self._db())
c.removals = removals
c.breadcrumbs = self._build_crumbs()
return render('/removal-list.mako')
finally:
self._db_close()
def one(self, id):
try:
try:
id = int(id)
except ValueError:
abort(404, 'No such log')
set_expires(int(config['app_conf']['expires.removal.one']))
removal = g.shm.removal_get_one(self._db(), id)
files = g.shm.removal_get_affected(self._db(), id)
fileinfo = {}
for hash in files:
fileinfo[hash] = g.shm.packages_get_file_info(self._db(), hash)
for hash in fileinfo:
fileinfo[hash] = map(lambda fi: dict(fi), fileinfo[hash])
for fi in fileinfo[hash]:
fi['dirlink'] = build_url_archive(fi['archive_name'], fi['run'], fi['path'])
fi['link'] = build_url_archive(fi['archive_name'], fi['run'], os.path.join(fi['path'], fi['name']), isadir=False )
files.sort(key=lambda a: (fileinfo[a][0]['name'], a)) # reproducible file order
c.removal = removal
c.files = files
c.fileinfo = fileinfo
c.breadcrumbs = self._build_crumbs(id)
c.title = 'Removal log #%d'%(id)
return render('/removal-list-one.mako')
finally:
self._db_close()
def _db(self):
if self.db is None:
self.db = DBInstance(app_globals.pool)
return self.db
class ArchiveController(BaseController):
db = None
def _db(self):
if self.db is None:
self.db = DBInstance(app_globals.pool)
return self.db
def _db_close(self):
if not self.db is None:
self.db.close()
self.db = None
def root(self):
set_expires(int(config['app_conf']['expires.archive.index']))
return redirect("../")
def archive_base(self, archive):
try:
#etag_cache( app_globals.shm.mirrorruns_get_etag(self._db(), archive) )
set_expires(int(config['app_conf']['expires.archive.index']))
if 'year' in request.params and 'month' in request.params:
y = request.params['year']
m = request.params['month']
return self._archive_ym(archive, y, m)
yearmonths = app_globals.shm.mirrorruns_get_yearmonths_from_archive(self._db(), archive)
if yearmonths is None:
abort(404, 'Archive "%s" does not exist'%(archive))
c.yearmonths = yearmonths
c.archive = archive
c.breadcrumbs = self._build_crumbs(archive)
c.title = archive
return render('/archive.mako')
finally:
self._db_close()
def _archive_ym(self, archive, year, month):
if not re.match('\d{4}$', year): # match matches only at start of string
abort(404, 'Year "%s" is not valid.'%(year))
if not re.match('\d{1,2}$', month): # match matches only at start of string
abort(404, 'Month "%s" is not valid.'%(month))
runs = app_globals.shm.mirrorruns_get_runs_from_archive_ym(self._db(), archive, year, month)
if runs is None:
abort(404, 'Archive "%s" does not exist'%(archive))
if len(runs) == 0:
abort(404, 'Found no mirrorruns for archive %s in %s-%s.'%(archive, year, month))
c.archive = archive
c.year = year
c.month = "%02d"%(int(month))
c.breadcrumbs = self._build_crumbs(archive, year=int(year), month=int(month))
c.runs = map(lambda r:
{ 'run' : r['run'],
# make a machine readable version of a timestamp
'run_mr': rfc3339_timestamp(r['run'])
}, runs)
c.title = '%s:%s-%02d'%(archive, year, int(month))
return render('/archive-runs.mako')
def _regular_file(self, digest, visiblepath=None):
try:
realpath = app_globals.shm.get_filepath(digest)
fa = SnapshotFileApp(realpath, digest, visiblepath)
return fa(request.environ, self.start_response)
except OSError, error:
if (error.errno == errno.ENOENT):
abort(404, "Ooops, we do not have a file with digest %s even tho we should. You might want to report this."%(digest))
else:
raise
except IOError, error:
if (error.errno == errno.EACCES):
abort(403, "Ooops, cannot read file with digest %s. Maybe this file is not redistributable and this was done on purpose. If in doubt report this."%(digest))
else:
raise
class PackageController(BaseController):
db = None
def _db(self):
if self.db is None:
self.db = DBInstance(app_globals.pool)
return self.db
def _db_close(self):
if not self.db is None:
self.db.close()
def _build_crumbs(self, pkg=None, version=None, start=None, is_binary=False):
crumbs = []
url = urllib.quote(request.environ.get('SCRIPT_NAME')) + "/"
crumbs.append( { 'url': url, 'name': app_globals.domain, 'sep': '|' });
if is_binary:
crumbs.append( { 'url': None, 'name': 'binary package:', 'sep': '' });
else:
crumbs.append( { 'url': None, 'name': 'source package:', 'sep': '' });
if not start:
if pkg.startswith('lib') and len(pkg) >= 4:
start = pkg[0:4]
else:
start = pkg[0:1]
if is_binary:
url += 'binary/'
else:
url += 'package/'
crumbs.append( { 'url': url + '?cat=%s'%urllib.quote(start), 'name': start+'*' } )
if not pkg is None:
url += urllib.quote(pkg) + '/'
crumbs.append( { 'url': url, 'name': pkg });
if version:
url += urllib.quote(version) + '/'
crumbs.append( { 'url': url, 'name': version });
crumbs[-1]['url'] = None
return crumbs
def _ensure_ascii(self, string):
# Package names are ascii.
# Check that before passing it on to postgres since the DB
# will just whine about not being able to convert the string
# anyway.
# If the passed string is not ascii, then the package name
# simply does not exist.
try:
string.encode('ascii')
except UnicodeEncodeError:
abort(404, 'No such package')
def root(self):
if 'src' in request.params:
set_expires(int(config['app_conf']['expires.package.root_cat']))
url = url_quote(request.params['src'] + "/")
return redirect(url)
elif 'cat' in request.params:
try:
#etag_cache( app_globals.shm.packages_get_etag(self._db()) )
set_expires(int(config['app_conf']['expires.package.root_cat']))
start = request.params['cat']
pkgs = app_globals.shm.packages_get_name_starts_with(self._db(), start)
if pkgs is None:
abort(404, 'No source packages in this category.')
c.start = start
c.packages = link_quote_array(pkgs)
c.breadcrumbs = self._build_crumbs(start=start)
c.title = '%s*'%(start)
return render('/package-list-packages.mako')
finally:
self._db_close()
else:
set_expires(int(config['app_conf']['expires.package.root_cat']))
return redirect("../")
def source(self, source):
self._ensure_ascii(source)
try:
#etag_cache( app_globals.shm.packages_get_etag(self._db()) )
set_expires(int(config['app_conf']['expires.package.source']))
sourceversions = app_globals.shm.packages_get_source_versions(self._db(), source)
if len(sourceversions) == 0:
abort(404, 'No such source package')
c.src = source
c.sourceversions = link_quote_array(sourceversions)
c.breadcrumbs = self._build_crumbs(source)
c.title = source
return render('/package-source.mako')
finally:
self._db_close()
def _attribute_escape(self, a):
return re.sub('[^a-zA-Z0-9_.-]', lambda m: ':%x:'%(ord(m.group())), a)
def source_version(self, source, version):
self._ensure_ascii(source)
self._ensure_ascii(version)
try:
#etag_cache( app_globals.shm.packages_get_etag(self._db()) )
set_expires(int(config['app_conf']['expires.package.source_version']))
sourcefiles = app_globals.shm.packages_get_source_files(self._db(), source, version)
binpkgs = app_globals.shm.packages_get_binpkgs_from_source(self._db(), source, version)
# we may have binaries without sources.
if len(sourcefiles) == 0 and len(binpkgs) == 0:
abort(404, 'No source or binary packages found')
binpkgs = map(lambda b: dict(b), binpkgs)
binhashes = []
for binpkg in binpkgs:
binpkg['escaped_name'] = self._attribute_escape(binpkg['name'])
binpkg['escaped_version'] = self._attribute_escape(binpkg['version'])
binpkg['files'] = map(lambda x: x['hash'], app_globals.shm.packages_get_binary_files_from_id(self._db(), binpkg['binpkg_id']))
binhashes += binpkg['files']
fileinfo = {}
for hash in sourcefiles + binhashes:
fileinfo[hash] = app_globals.shm.packages_get_file_info(self._db(), hash)
for hash in fileinfo:
fileinfo[hash] = map(lambda fi: dict(fi), fileinfo[hash])
for fi in fileinfo[hash]:
fi['dirlink'] = build_url_archive(fi['archive_name'], fi['run'], fi['path'])
fi['link'] = build_url_archive(fi['archive_name'], fi['run'], os.path.join(fi['path'], fi['name']), isadir=False )
sourcefiles.sort(key=lambda a: (fileinfo[a][0]['name'], a) if len(fileinfo[a]) > 0 else (None,a)) # reproducible file order
c.src = source
c.version = version
c.sourcefiles = sourcefiles
c.binpkgs = binpkgs
c.fileinfo = fileinfo
c.breadcrumbs = self._build_crumbs(source, version)
c.title = '%s (%s)'%(source, version)
return render('/package-source-one.mako')
finally:
self._db_close()
def binary_root(self):
if 'bin' in request.params:
set_expires(int(config['app_conf']['expires.package.root_cat']))
url = url_quote(request.params['bin'] + "/")
return redirect(url)
elif 'cat' in request.params:
try:
#etag_cache( app_globals.shm.packages_get_etag(self._db()) )
set_expires(int(config['app_conf']['expires.package.root_cat']))
start = request.params['cat']
pkgs = app_globals.shm.packages_get_name_starts_with(self._db(), start, get_binary=True)
if pkgs is None:
abort(404, 'No binary packages in this category.')
c.start = start
c.packages = link_quote_array(pkgs)
c.breadcrumbs = self._build_crumbs(start=start, is_binary=True)
c.title = '%s*'%(start)
return render('/package-binary-list-packages.mako')
finally:
self._db_close()
else:
set_expires(int(config['app_conf']['expires.package.root_cat']))
return redirect("../")
def binary(self, binary):
self._ensure_ascii(binary)
try:
#etag_cache( app_globals.shm.packages_get_etag(self._db()) )
set_expires(int(config['app_conf']['expires.package.source']))
binaryversions = app_globals.shm.packages_get_binary_versions_by_name(self._db(), binary)
if len(binaryversions) == 0:
abort(404, 'No such binary package')
binaryversions = map(lambda b: dict(b), binaryversions)
for b in binaryversions:
b['link'] = url_quote('../../package/%s/%s/'%(b['source'], b['version']))
b['escaped_name'] = self._attribute_escape(b['name'])
b['escaped_binary_version'] = self._attribute_escape(b['binary_version'])
c.binary = binary
c.binaryversions = binaryversions
c.breadcrumbs = self._build_crumbs(binary, is_binary=True)
c.title = binary
return render('/package-binary.mako')
finally:
self._db_close()
def _get_fileinfo_for_mr_one(self, hash):
fileinfo = map(lambda x: dict(x), app_globals.shm.packages_get_file_info(self._db(), hash))
for fi in fileinfo:
fi['first_seen'] = rfc3339_timestamp(fi['run'])
del fi['run']
return fileinfo
def _get_fileinfo_for_mr(self, hashes):
r = {}
for hash in hashes:
r[hash] = self._get_fileinfo_for_mr_one(hash)
return r
@jsonify
def mr_list(self):
try:
set_expires(int(config['app_conf']['expires.package.mr.list']))
pkgs = app_globals.shm.packages_get_all(self._db())
return { '_comment': "foo",
'result': map(lambda x: { 'package': x }, pkgs) }
finally:
self._db_close()
@jsonify
def mr_source(self, source):
self._ensure_ascii(source)
try:
set_expires(int(config['app_conf']['expires.package.mr.source']))
sourceversions = app_globals.shm.packages_get_source_versions(self._db(), source)
if len(sourceversions) == 0: abort(404, 'No such source package')
return { '_comment': "foo",
'package': source,
'result': map(lambda x: { 'version': x }, sourceversions) }
finally:
self._db_close()
@jsonify
def mr_source_version_srcfiles(self, source, version):
self._ensure_ascii(source)
self._ensure_ascii(version)
try:
set_expires(int(config['app_conf']['expires.package.mr.source_version']))
sourcefiles = app_globals.shm.packages_get_source_files(self._db(), source, version)
if len(sourcefiles) == 0: abort(404, 'No such source package or no sources found')
r = { '_comment': "foo",
'package': source,
'version': version,
'result': map(lambda x: { 'hash': x }, sourcefiles) }
if ('fileinfo' in request.params) and (request.params['fileinfo'] == '1'):
r['fileinfo'] = self._get_fileinfo_for_mr(sourcefiles)
return r
finally:
self._db_close()
@jsonify
def mr_source_version_binpackages(self, source, version):
self._ensure_ascii(source)
self._ensure_ascii(version)
try:
set_expires(int(config['app_conf']['expires.package.mr.source_version']))
binpkgs = app_globals.shm.packages_get_binpkgs_from_source(self._db(), source, version)
if len(binpkgs) == 0: abort(404, 'No such source package or no binary packages found')
binpkgs = map(lambda b: { 'name': b['name'],
'version': b['version'] }, binpkgs)
return { '_comment': "foo",
'package': source,
'version': version,
'result': binpkgs }
finally:
self._db_close()
@jsonify
def mr_source_version_binfiles(self, source, version, binary, binary_version):
self._ensure_ascii(source)
self._ensure_ascii(version)
self._ensure_ascii(binary)
self._ensure_ascii(binary_version)
try:
set_expires(int(config['app_conf']['expires.package.mr.source_version']))
binfiles = app_globals.shm.packages_get_binary_files_from_packagenames(self._db(), source, version, binary, binary_version)
if len(binfiles) == 0: abort(404, 'No such package or no binary files found')
binfiles = map(lambda b: dict(b), binfiles)
r = { '_comment': "foo",
'package': source,
'version': version,
'binary': binary,
'binary_version': binary_version,
'result': binfiles }
if ('fileinfo' in request.params) and (request.params['fileinfo'] == '1'):
r['fileinfo'] = self._get_fileinfo_for_mr(map(lambda x: x['hash'], binfiles))
return r
finally:
self._db_close()
@jsonify
def mr_source_version_allfiles(self, source, version):
self._ensure_ascii(source)
self._ensure_ascii(version)
try:
set_expires(int(config['app_conf']['expires.package.mr.source_version']))
sourcefiles = app_globals.shm.packages_get_source_files(self._db(), source, version)
binpkgs = app_globals.shm.packages_get_binpkgs_from_source(self._db(), source, version)
# we may have binaries without sources.
if len(sourcefiles) == 0 and len(binpkgs) == 0:
abort(404, 'No source or binary packages found')
binpkgs = map(lambda b: dict(b), binpkgs)
binhashes = []
for binpkg in binpkgs:
binpkg['files'] = map(lambda x: dict(x), app_globals.shm.packages_get_binary_files_from_id(self._db(), binpkg['binpkg_id']))
del binpkg['binpkg_id']
binhashes += map(lambda x: x['hash'], binpkg['files'])
r = { '_comment': "foo",
'package': source,
'version': version,
'result': { 'source': map(lambda x: { 'hash': x }, sourcefiles), 'binaries': binpkgs }
}
if ('fileinfo' in request.params) and (request.params['fileinfo'] == '1'):
r['fileinfo'] = self._get_fileinfo_for_mr(sourcefiles + binhashes)
return r
finally:
self._db_close()
@jsonify
def mr_binary(self, binary):
self._ensure_ascii(binary)
try:
set_expires(int(config['app_conf']['expires.package.mr.source']))
binaryversions = app_globals.shm.packages_get_binary_versions_by_name(self._db(), binary)
binaryversions = map(lambda b: dict(b), binaryversions)
if len(binaryversions) == 0: abort(404, 'No such binary package')
r = { '_comment': "foo",
'binary': binary,
'result': binaryversions }
return r
finally:
self._db_close()
@jsonify
def mr_binary_version_binfiles(self, binary, binary_version):
self._ensure_ascii(binary)
self._ensure_ascii(binary_version)
try:
binfiles = app_globals.shm.packages_get_binary_files(self._db(), binary, binary_version)
if len(binfiles) == 0: abort(404, 'No such package or no binary files found')
binfiles = map(lambda b: dict(b), binfiles)
r = { '_comment': "foo",
'binary': binary,
'binary_version': binary_version,
'result': binfiles }
if ('fileinfo' in request.params) and (request.params['fileinfo'] == '1'):
r['fileinfo'] = self._get_fileinfo_for_mr(map(lambda x: x['hash'], binfiles))
return r
finally:
self._db_close()
@jsonify
def mr_fileinfo(self, hash):
if not re.match('[0-9a-f]{40}$', hash): # match matches only at start of string
abort(404, 'Invalid hash format.')
try:
return { '_comment': "foo",
'hash': hash,
'result': self._get_fileinfo_for_mr_one(hash) }
return r
finally:
self._db_close()
