def index(self): db = None try: db = DBInstance(app_globals.pool) c.names = link_quote_array(app_globals.shm.archives_get_list(db)) c.srcstarts = link_quote_array(app_globals.shm.packages_get_name_starts(db)) c.binstarts = link_quote_array(app_globals.shm.packages_get_name_starts(db, get_binary=True)) set_expires(int(config['app_conf']['expires.root'])) return render('/root.mako') finally: if not db is None: db.close()
def trace(self): db = None try: db = DBInstance(g.pool) set_expires(int(config['app_conf']['expires.root'])) last = g.shm.get_last_mirrorrun(db) content = [] content.append("%s\n"%(last.ctime())) content.append("# Above timestamp is timestamp of latest mirrrorrun\n") response.content_type = 'text/plain' return content finally: if not db is None: db.close()
class RemovalController(BaseController): db = None def _db(self): if self.db is None: self.db = DBInstance(g.pool) return self.db def _db_close(self): if not self.db is None: self.db.close() def _build_crumbs(self, entry=None): crumbs = [] url = urllib.quote(request.environ.get('SCRIPT_NAME')) + "/" crumbs.append( { 'url': url, 'name': g.domain, 'sep': '|' }); url += 'removal/' crumbs.append( { 'url': url, 'name': 'removal' }); if not entry is None: entry=str(entry) url += urllib.quote(entry) crumbs.append( { 'url': url, 'name': entry, 'sep': '' }); crumbs[-1]['url'] = None return crumbs def root(self): try: set_expires(int(config['app_conf']['expires.removal'])) removals = g.shm.removal_get_list(self._db()) c.removals = removals c.breadcrumbs = self._build_crumbs() return render('/removal-list.mako') finally: self._db_close() def one(self, id): try: try: id = int(id) except ValueError: abort(404, 'No such log') set_expires(int(config['app_conf']['expires.removal.one'])) removal = g.shm.removal_get_one(self._db(), id) files = g.shm.removal_get_affected(self._db(), id) fileinfo = {} for hash in files: fileinfo[hash] = g.shm.packages_get_file_info(self._db(), hash) for hash in fileinfo: fileinfo[hash] = map(lambda fi: dict(fi), fileinfo[hash]) for fi in fileinfo[hash]: fi['dirlink'] = build_url_archive(fi['archive_name'], fi['run'], fi['path']) fi['link'] = build_url_archive(fi['archive_name'], fi['run'], os.path.join(fi['path'], fi['name']), isadir=False ) files.sort(key=lambda a: (fileinfo[a][0]['name'], a)) # reproducible file order c.removal = removal c.files = files c.fileinfo = fileinfo c.breadcrumbs = self._build_crumbs(id) c.title = 'Removal log #%d'%(id) return render('/removal-list-one.mako') finally: self._db_close()
class ArchiveController(BaseController): db = None def _db(self): if self.db is None: self.db = DBInstance(app_globals.pool) return self.db def _db_close(self): if not self.db is None: self.db.close() self.db = None def root(self): set_expires(int(config['app_conf']['expires.archive.index'])) return redirect("../") def archive_base(self, archive): try: #etag_cache( app_globals.shm.mirrorruns_get_etag(self._db(), archive) ) set_expires(int(config['app_conf']['expires.archive.index'])) if 'year' in request.params and 'month' in request.params: y = request.params['year'] m = request.params['month'] return self._archive_ym(archive, y, m) yearmonths = app_globals.shm.mirrorruns_get_yearmonths_from_archive(self._db(), archive) if yearmonths is None: abort(404, 'Archive "%s" does not exist'%(archive)) c.yearmonths = yearmonths c.archive = archive c.breadcrumbs = self._build_crumbs(archive) c.title = archive return render('/archive.mako') finally: self._db_close() def _archive_ym(self, archive, year, month): if not re.match('\d{4}$', year): # match matches only at start of string abort(404, 'Year "%s" is not valid.'%(year)) if not re.match('\d{1,2}$', month): # match matches only at start of string abort(404, 'Month "%s" is not valid.'%(month)) runs = app_globals.shm.mirrorruns_get_runs_from_archive_ym(self._db(), archive, year, month) if runs is None: abort(404, 'Archive "%s" does not exist'%(archive)) if len(runs) == 0: abort(404, 'Found no mirrorruns for archive %s in %s-%s.'%(archive, year, month)) c.archive = archive c.year = year c.month = "%02d"%(int(month)) c.breadcrumbs = self._build_crumbs(archive, year=int(year), month=int(month)) c.runs = map(lambda r: { 'run' : r['run'], # make a machine readable version of a timestamp 'run_mr': rfc3339_timestamp(r['run']) }, runs) c.title = '%s:%s-%02d'%(archive, year, int(month)) return render('/archive-runs.mako') def _regular_file(self, digest, visiblepath=None): try: realpath = app_globals.shm.get_filepath(digest) fa = SnapshotFileApp(realpath, digest, visiblepath) return fa(request.environ, self.start_response) except OSError, error: if (error.errno == errno.ENOENT): abort(404, "Ooops, we do not have a file with digest %s even tho we should. You might want to report this."%(digest)) else: raise except IOError, error: if (error.errno == errno.EACCES): abort(403, "Ooops, cannot read file with digest %s. Maybe this file is not redistributable and this was done on purpose. If in doubt report this."%(digest)) else: raise
class PackageController(BaseController): db = None def _db(self): if self.db is None: self.db = DBInstance(app_globals.pool) return self.db def _db_close(self): if not self.db is None: self.db.close() def _build_crumbs(self, pkg=None, version=None, start=None, is_binary=False): crumbs = [] url = urllib.quote(request.environ.get('SCRIPT_NAME')) + "/" crumbs.append( { 'url': url, 'name': app_globals.domain, 'sep': '|' }); if is_binary: crumbs.append( { 'url': None, 'name': 'binary package:', 'sep': '' }); else: crumbs.append( { 'url': None, 'name': 'source package:', 'sep': '' }); if not start: if pkg.startswith('lib') and len(pkg) >= 4: start = pkg[0:4] else: start = pkg[0:1] if is_binary: url += 'binary/' else: url += 'package/' crumbs.append( { 'url': url + '?cat=%s'%urllib.quote(start), 'name': start+'*' } ) if not pkg is None: url += urllib.quote(pkg) + '/' crumbs.append( { 'url': url, 'name': pkg }); if version: url += urllib.quote(version) + '/' crumbs.append( { 'url': url, 'name': version }); crumbs[-1]['url'] = None return crumbs def _ensure_ascii(self, string): # Package names are ascii. # Check that before passing it on to postgres since the DB # will just whine about not being able to convert the string # anyway. # If the passed string is not ascii, then the package name # simply does not exist. try: string.encode('ascii') except UnicodeEncodeError: abort(404, 'No such package') def root(self): if 'src' in request.params: set_expires(int(config['app_conf']['expires.package.root_cat'])) url = url_quote(request.params['src'] + "/") return redirect(url) elif 'cat' in request.params: try: #etag_cache( app_globals.shm.packages_get_etag(self._db()) ) set_expires(int(config['app_conf']['expires.package.root_cat'])) start = request.params['cat'] pkgs = app_globals.shm.packages_get_name_starts_with(self._db(), start) if pkgs is None: abort(404, 'No source packages in this category.') c.start = start c.packages = link_quote_array(pkgs) c.breadcrumbs = self._build_crumbs(start=start) c.title = '%s*'%(start) return render('/package-list-packages.mako') finally: self._db_close() else: set_expires(int(config['app_conf']['expires.package.root_cat'])) return redirect("../") def source(self, source): self._ensure_ascii(source) try: #etag_cache( app_globals.shm.packages_get_etag(self._db()) ) set_expires(int(config['app_conf']['expires.package.source'])) sourceversions = app_globals.shm.packages_get_source_versions(self._db(), source) if len(sourceversions) == 0: abort(404, 'No such source package') c.src = source c.sourceversions = link_quote_array(sourceversions) c.breadcrumbs = self._build_crumbs(source) c.title = source return render('/package-source.mako') finally: self._db_close() def _attribute_escape(self, a): return re.sub('[^a-zA-Z0-9_.-]', lambda m: ':%x:'%(ord(m.group())), a) def source_version(self, source, version): self._ensure_ascii(source) self._ensure_ascii(version) try: #etag_cache( app_globals.shm.packages_get_etag(self._db()) ) set_expires(int(config['app_conf']['expires.package.source_version'])) sourcefiles = app_globals.shm.packages_get_source_files(self._db(), source, version) binpkgs = app_globals.shm.packages_get_binpkgs_from_source(self._db(), source, version) # we may have binaries without sources. if len(sourcefiles) == 0 and len(binpkgs) == 0: abort(404, 'No source or binary packages found') binpkgs = map(lambda b: dict(b), binpkgs) binhashes = [] for binpkg in binpkgs: binpkg['escaped_name'] = self._attribute_escape(binpkg['name']) binpkg['escaped_version'] = self._attribute_escape(binpkg['version']) binpkg['files'] = map(lambda x: x['hash'], app_globals.shm.packages_get_binary_files_from_id(self._db(), binpkg['binpkg_id'])) binhashes += binpkg['files'] fileinfo = {} for hash in sourcefiles + binhashes: fileinfo[hash] = app_globals.shm.packages_get_file_info(self._db(), hash) for hash in fileinfo: fileinfo[hash] = map(lambda fi: dict(fi), fileinfo[hash]) for fi in fileinfo[hash]: fi['dirlink'] = build_url_archive(fi['archive_name'], fi['run'], fi['path']) fi['link'] = build_url_archive(fi['archive_name'], fi['run'], os.path.join(fi['path'], fi['name']), isadir=False ) sourcefiles.sort(key=lambda a: (fileinfo[a][0]['name'], a) if len(fileinfo[a]) > 0 else (None,a)) # reproducible file order c.src = source c.version = version c.sourcefiles = sourcefiles c.binpkgs = binpkgs c.fileinfo = fileinfo c.breadcrumbs = self._build_crumbs(source, version) c.title = '%s (%s)'%(source, version) return render('/package-source-one.mako') finally: self._db_close() def binary_root(self): if 'bin' in request.params: set_expires(int(config['app_conf']['expires.package.root_cat'])) url = url_quote(request.params['bin'] + "/") return redirect(url) elif 'cat' in request.params: try: #etag_cache( app_globals.shm.packages_get_etag(self._db()) ) set_expires(int(config['app_conf']['expires.package.root_cat'])) start = request.params['cat'] pkgs = app_globals.shm.packages_get_name_starts_with(self._db(), start, get_binary=True) if pkgs is None: abort(404, 'No binary packages in this category.') c.start = start c.packages = link_quote_array(pkgs) c.breadcrumbs = self._build_crumbs(start=start, is_binary=True) c.title = '%s*'%(start) return render('/package-binary-list-packages.mako') finally: self._db_close() else: set_expires(int(config['app_conf']['expires.package.root_cat'])) return redirect("../") def binary(self, binary): self._ensure_ascii(binary) try: #etag_cache( app_globals.shm.packages_get_etag(self._db()) ) set_expires(int(config['app_conf']['expires.package.source'])) binaryversions = app_globals.shm.packages_get_binary_versions_by_name(self._db(), binary) if len(binaryversions) == 0: abort(404, 'No such binary package') binaryversions = map(lambda b: dict(b), binaryversions) for b in binaryversions: b['link'] = url_quote('../../package/%s/%s/'%(b['source'], b['version'])) b['escaped_name'] = self._attribute_escape(b['name']) b['escaped_binary_version'] = self._attribute_escape(b['binary_version']) c.binary = binary c.binaryversions = binaryversions c.breadcrumbs = self._build_crumbs(binary, is_binary=True) c.title = binary return render('/package-binary.mako') finally: self._db_close() def _get_fileinfo_for_mr_one(self, hash): fileinfo = map(lambda x: dict(x), app_globals.shm.packages_get_file_info(self._db(), hash)) for fi in fileinfo: fi['first_seen'] = rfc3339_timestamp(fi['run']) del fi['run'] return fileinfo def _get_fileinfo_for_mr(self, hashes): r = {} for hash in hashes: r[hash] = self._get_fileinfo_for_mr_one(hash) return r @jsonify def mr_list(self): try: set_expires(int(config['app_conf']['expires.package.mr.list'])) pkgs = app_globals.shm.packages_get_all(self._db()) return { '_comment': "foo", 'result': map(lambda x: { 'package': x }, pkgs) } finally: self._db_close() @jsonify def mr_source(self, source): self._ensure_ascii(source) try: set_expires(int(config['app_conf']['expires.package.mr.source'])) sourceversions = app_globals.shm.packages_get_source_versions(self._db(), source) if len(sourceversions) == 0: abort(404, 'No such source package') return { '_comment': "foo", 'package': source, 'result': map(lambda x: { 'version': x }, sourceversions) } finally: self._db_close() @jsonify def mr_source_version_srcfiles(self, source, version): self._ensure_ascii(source) self._ensure_ascii(version) try: set_expires(int(config['app_conf']['expires.package.mr.source_version'])) sourcefiles = app_globals.shm.packages_get_source_files(self._db(), source, version) if len(sourcefiles) == 0: abort(404, 'No such source package or no sources found') r = { '_comment': "foo", 'package': source, 'version': version, 'result': map(lambda x: { 'hash': x }, sourcefiles) } if ('fileinfo' in request.params) and (request.params['fileinfo'] == '1'): r['fileinfo'] = self._get_fileinfo_for_mr(sourcefiles) return r finally: self._db_close() @jsonify def mr_source_version_binpackages(self, source, version): self._ensure_ascii(source) self._ensure_ascii(version) try: set_expires(int(config['app_conf']['expires.package.mr.source_version'])) binpkgs = app_globals.shm.packages_get_binpkgs_from_source(self._db(), source, version) if len(binpkgs) == 0: abort(404, 'No such source package or no binary packages found') binpkgs = map(lambda b: { 'name': b['name'], 'version': b['version'] }, binpkgs) return { '_comment': "foo", 'package': source, 'version': version, 'result': binpkgs } finally: self._db_close() @jsonify def mr_source_version_binfiles(self, source, version, binary, binary_version): self._ensure_ascii(source) self._ensure_ascii(version) self._ensure_ascii(binary) self._ensure_ascii(binary_version) try: set_expires(int(config['app_conf']['expires.package.mr.source_version'])) binfiles = app_globals.shm.packages_get_binary_files_from_packagenames(self._db(), source, version, binary, binary_version) if len(binfiles) == 0: abort(404, 'No such package or no binary files found') binfiles = map(lambda b: dict(b), binfiles) r = { '_comment': "foo", 'package': source, 'version': version, 'binary': binary, 'binary_version': binary_version, 'result': binfiles } if ('fileinfo' in request.params) and (request.params['fileinfo'] == '1'): r['fileinfo'] = self._get_fileinfo_for_mr(map(lambda x: x['hash'], binfiles)) return r finally: self._db_close() @jsonify def mr_source_version_allfiles(self, source, version): self._ensure_ascii(source) self._ensure_ascii(version) try: set_expires(int(config['app_conf']['expires.package.mr.source_version'])) sourcefiles = app_globals.shm.packages_get_source_files(self._db(), source, version) binpkgs = app_globals.shm.packages_get_binpkgs_from_source(self._db(), source, version) # we may have binaries without sources. if len(sourcefiles) == 0 and len(binpkgs) == 0: abort(404, 'No source or binary packages found') binpkgs = map(lambda b: dict(b), binpkgs) binhashes = [] for binpkg in binpkgs: binpkg['files'] = map(lambda x: dict(x), app_globals.shm.packages_get_binary_files_from_id(self._db(), binpkg['binpkg_id'])) del binpkg['binpkg_id'] binhashes += map(lambda x: x['hash'], binpkg['files']) r = { '_comment': "foo", 'package': source, 'version': version, 'result': { 'source': map(lambda x: { 'hash': x }, sourcefiles), 'binaries': binpkgs } } if ('fileinfo' in request.params) and (request.params['fileinfo'] == '1'): r['fileinfo'] = self._get_fileinfo_for_mr(sourcefiles + binhashes) return r finally: self._db_close() @jsonify def mr_binary(self, binary): self._ensure_ascii(binary) try: set_expires(int(config['app_conf']['expires.package.mr.source'])) binaryversions = app_globals.shm.packages_get_binary_versions_by_name(self._db(), binary) binaryversions = map(lambda b: dict(b), binaryversions) if len(binaryversions) == 0: abort(404, 'No such binary package') r = { '_comment': "foo", 'binary': binary, 'result': binaryversions } return r finally: self._db_close() @jsonify def mr_binary_version_binfiles(self, binary, binary_version): self._ensure_ascii(binary) self._ensure_ascii(binary_version) try: binfiles = app_globals.shm.packages_get_binary_files(self._db(), binary, binary_version) if len(binfiles) == 0: abort(404, 'No such package or no binary files found') binfiles = map(lambda b: dict(b), binfiles) r = { '_comment': "foo", 'binary': binary, 'binary_version': binary_version, 'result': binfiles } if ('fileinfo' in request.params) and (request.params['fileinfo'] == '1'): r['fileinfo'] = self._get_fileinfo_for_mr(map(lambda x: x['hash'], binfiles)) return r finally: self._db_close() @jsonify def mr_fileinfo(self, hash): if not re.match('[0-9a-f]{40}$', hash): # match matches only at start of string abort(404, 'Invalid hash format.') try: return { '_comment': "foo", 'hash': hash, 'result': self._get_fileinfo_for_mr_one(hash) } return r finally: self._db_close()