def proxy_remote_package(package):
pkg = yield threaded(Package.get_or_create)(name=package, proxy=True)
releases, cached_releases = yield [PYPIClient.releases(pkg.name), threaded(pkg.versions)(True)]
IOLoop.current().add_callback(threaded(pkg.hide_versions), filter(lambda x: not x.hidden, releases))
cached_releases = set(map(lambda x: x.version, cached_releases))
new_releases = list(releases - cached_releases)
for release_part in chunks(new_releases, 10):
yield [release_fetch(pkg, release) for release in release_part]
raise Return(pkg)
def get(self, package, version, filename):
try:
package = yield PYPIClient.find_real_name(package)
except (LookupError, HTTPClientError) as e:
if isinstance(e, HTTPClientError):
log.warning("PYPI backend return an error: %s", e)
package = yield self.thread_pool.submit(Package.find, package)
try:
pkg_file = yield self.find_file(package, version, filename)
if not pkg_file.fetched:
yield self.fetch_remote_file(pkg_file)
except LookupError:
self.send_error(404)
else:
self.set_header("MD5", pkg_file.md5)
self.set_header("ETag", pkg_file.md5)
self.set_header("Content-Length", pkg_file.size)
self.set_header("Content-Type", 'application/octet-stream')
self.set_header("Date", pkg_file.ts.strftime("%a, %d %b %Y %H:%M:%S %Z"))
with pkg_file.open() as f:
reader = threaded(f.read)
data = yield reader(self.CHUNK_SIZE)
self.write(data)
yield Task(self.flush)
while data:
data = yield reader(self.CHUNK_SIZE)
self.write(data)
yield Task(self.flush)
self.finish()
def proxy_remote_package(package):
pkg = yield threaded(Package.get_or_create)(name=package, proxy=True)
releases, cached_releases = yield [
PYPIClient.releases(pkg.name),
threaded(pkg.versions)(True)
]
IOLoop.current().add_callback(threaded(pkg.hide_versions),
filter(lambda x: not x.hidden, releases))
cached_releases = set(map(lambda x: x.version, cached_releases))
new_releases = list(releases - cached_releases)
for release_part in chunks(new_releases, 10):
yield [release_fetch(pkg, release) for release in release_part]
raise Return(pkg)
def get(self, package, version=None):
exists, is_proxy, pkg = yield self.packages_list(package)
if not exists or (is_proxy or not pkg):
pkg = yield self.proxy_package(package)
if not pkg:
raise HTTPError(404)
files = yield threaded(pkg.files)(version=version)
if not files:
raise HTTPError(404)
self.render(os.path.join('simple', 'files.html'),
package=pkg.lower_name,
files=files)
def get(self, package, version=None):
exists, is_proxy, pkg = yield self.packages_list(package)
if not exists or (is_proxy or not pkg):
pkg = yield self.proxy_package(package)
if not pkg:
raise HTTPError(404)
files = yield threaded(pkg.files)(version=version)
if not files:
raise HTTPError(404)
self.render(
os.path.join('simple', 'files.html'),
package=pkg.lower_name,
files=files
)
def get(self, package, version, filename):
try:
package = yield PYPIClient.find_real_name(package)
except (LookupError, HTTPClientError) as e:
if isinstance(e, HTTPClientError):
log.warning("PYPI backend return an error: %s", e)
package = yield self.thread_pool.submit(Package.find, package)
try:
pkg_file = yield self.find_file(package, version, filename)
if not pkg_file.fetched:
yield self.fetch_remote_file(pkg_file)
except LookupError:
self.send_error(404)
else:
self.set_header("MD5", pkg_file.md5)
self.set_header("ETag", pkg_file.md5)
self.set_header("Content-Length", pkg_file.size)
self.set_header("Content-Type", 'application/octet-stream')
self.set_header("Date",
pkg_file.ts.strftime("%a, %d %b %Y %H:%M:%S %Z"))
with pkg_file.open() as f:
reader = threaded(f.read)
data = yield reader(self.CHUNK_SIZE)
self.write(data)
yield Task(self.flush)
for chunk in iter(lambda: reader(self.CHUNK_SIZE), None):
data = yield chunk
if not data:
break
self.write(data)
yield Task(self.flush)
self.finish()
