def proxy_package(cls, package):
try:
remote_package_exists = yield PYPIClient.exists(package)
if remote_package_exists:
pkg_real_name = yield PYPIClient.find_real_name(package)
pkg = yield proxy_remote_package(pkg_real_name)
raise Return(pkg)
raise LookupError("Remote package not found")
except (LookupError, HTTPClientError) as e:
if isinstance(e, HTTPClientError):
log.warning("PYPI backend return an error: %s", e)
raise Return(Package.find(package))
def proxy_package(cls, package):
try:
remote_package_exists = yield PYPIClient.exists(package)
if remote_package_exists:
pkg_real_name = yield PYPIClient.find_real_name(package)
pkg = yield proxy_remote_package(pkg_real_name)
raise Return(pkg)
raise LookupError("Remote package not found")
except (LookupError, HTTPClientError) as e:
if isinstance(e, HTTPClientError):
log.warning("PYPI backend return an error: %s", e)
raise Return(Package.find(package))
def get(self, package, version, filename):
try:
package = yield PYPIClient.find_real_name(package)
except (LookupError, HTTPClientError) as e:
if isinstance(e, HTTPClientError):
log.warning("PYPI backend return an error: %s", e)
package = yield self.thread_pool.submit(Package.find, package)
try:
pkg_file = yield self.find_file(package, version, filename)
if not pkg_file.fetched:
yield self.fetch_remote_file(pkg_file)
except LookupError:
self.send_error(404)
else:
self.set_header("MD5", pkg_file.md5)
self.set_header("ETag", pkg_file.md5)
self.set_header("Content-Length", pkg_file.size)
self.set_header("Content-Type", 'application/octet-stream')
self.set_header("Date", pkg_file.ts.strftime("%a, %d %b %Y %H:%M:%S %Z"))
with pkg_file.open() as f:
reader = threaded(f.read)
data = yield reader(self.CHUNK_SIZE)
self.write(data)
yield Task(self.flush)
while data:
data = yield reader(self.CHUNK_SIZE)
self.write(data)
yield Task(self.flush)
self.finish()
def proxy_remote_package(package):
pkg = yield threaded(Package.get_or_create)(name=package, proxy=True)
releases, cached_releases = yield [PYPIClient.releases(pkg.name), threaded(pkg.versions)(True)]
IOLoop.current().add_callback(threaded(pkg.hide_versions), filter(lambda x: not x.hidden, releases))
cached_releases = set(map(lambda x: x.version, cached_releases))
new_releases = list(releases - cached_releases)
for release_part in chunks(new_releases, 10):
yield [release_fetch(pkg, release) for release in release_part]
raise Return(pkg)
def proxy_remote_package(package):
pkg = yield threaded(Package.get_or_create)(name=package, proxy=True)
releases, cached_releases = yield [
PYPIClient.releases(pkg.name),
threaded(pkg.versions)(True)
]
IOLoop.current().add_callback(threaded(pkg.hide_versions),
filter(lambda x: not x.hidden, releases))
cached_releases = set(map(lambda x: x.version, cached_releases))
new_releases = list(releases - cached_releases)
for release_part in chunks(new_releases, 10):
yield [release_fetch(pkg, release) for release in release_part]
raise Return(pkg)
def rpc_search(self, query, operator='or'):
results = []
assert operator in ("and", "or"), "Operator must be 'and' or 'or'"
names = tuple(query.get('name', []))
descriptions = tuple(query.get('summary', []))
async_queries = yield [
self._search(names, descriptions, operator),
PYPIClient.search(names, descriptions, operator)
]
for res in async_queries:
for result in res:
results.append(result)
raise Return(results)
def get(self, package, version, filename):
try:
package = yield PYPIClient.find_real_name(package)
except (LookupError, HTTPClientError) as e:
if isinstance(e, HTTPClientError):
log.warning("PYPI backend return an error: %s", e)
package = yield self.thread_pool.submit(Package.find, package)
try:
pkg_file = yield self.find_file(package, version, filename)
if not pkg_file.fetched:
yield self.fetch_remote_file(pkg_file)
except LookupError:
self.send_error(404)
else:
self.set_header("MD5", pkg_file.md5)
self.set_header("ETag", pkg_file.md5)
self.set_header("Content-Length", pkg_file.size)
self.set_header("Content-Type", 'application/octet-stream')
self.set_header("Date",
pkg_file.ts.strftime("%a, %d %b %Y %H:%M:%S %Z"))
with pkg_file.open() as f:
reader = threaded(f.read)
data = yield reader(self.CHUNK_SIZE)
self.write(data)
yield Task(self.flush)
for chunk in iter(lambda: reader(self.CHUNK_SIZE), None):
data = yield chunk
if not data:
break
self.write(data)
yield Task(self.flush)
self.finish()
def run():
options.parse_command_line()
if options.config:
options.parse_config_file(options.config)
options.storage = os.path.abspath(options.storage)
if os.getuid() == 0 and options.user:
pw = pwd.getpwnam(options.user)
uid, gid = pw.pw_uid, pw.pw_gid
log.info("Changind user to %s [%s:%s]", options.user, uid, gid)
os.setgid(uid)
os.setuid(uid)
try:
if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)):
log.info('Creating new package storage directory: "%s"', options.storage)
os.makedirs(options.storage)
def on_interrupt(*args):
log.warning("Receiving interrupt signal. Application will be stopped.")
exit(errno.EINTR)
log.debug("Preparing signal handling")
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
signal.signal(sig, on_interrupt)
def handle_pdb(sig, frame):
import pdb
pdb.Pdb().set_trace(frame)
if options.debug:
signal.signal(signal.SIGUSR2, handle_pdb)
log.debug("Creating application instance")
app = create_app(
options.debug,
options.secret,
options.gzip,
)
log.debug("Creating IOLoop instance.")
io_loop = IOLoop.current()
io_loop.run_sync(lambda: init_db(options.database))
if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)):
os.makedirs(options.cache_dir)
Cache.CACHE_DIR = options.cache_dir
log.info("Init thread pool with %d threads", options.pool_size)
handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size)
AsyncHTTPClient.configure(None, max_clients=options.max_http_clients)
PYPIClient.configure(
options.pypi_server,
handlers.base.BaseHandler.THREAD_POOL
)
pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop)
io_loop.add_callback(PYPIClient.packages)
io_loop.add_callback(pypi_updater.start)
log.info("Starting server http://%s:%d/", options.address, options.port)
http_server = HTTPServer(app, xheaders=options.proxy_mode)
http_server.listen(options.port, address=options.address)
log.debug('Setting "%s" as storage', options.storage)
PackageFile.set_storage(options.storage)
log.debug("Starting main loop")
io_loop.start()
except Exception as e:
log.fatal("Exception on main loop:")
log.exception(e)
exit(1)
else:
exit(0)
def release_fetch(package, rel):
version_info, release_files = yield PYPIClient.release_data(package.name, rel)
raise Return((yield release_db_save(package, rel, version_info, release_files)))
def run():
options.parse_command_line()
if options.config:
options.parse_config_file(options.config)
options.storage = os.path.abspath(options.storage)
if os.getuid() == 0 and options.user:
pw = pwd.getpwnam(options.user)
uid, gid = pw.pw_uid, pw.pw_gid
log.info("Changind user to %s [%s:%s]", options.user, uid, gid)
os.setgid(uid)
os.setuid(uid)
try:
if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)):
log.info('Creating new package storage directory: "%s"', options.storage)
os.makedirs(options.storage)
def on_interrupt(*args):
log.warning("Receiving interrupt signal. Application will be stopped.")
exit(errno.EINTR)
log.debug("Preparing signal handling")
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
signal.signal(sig, on_interrupt)
def handle_pdb(sig, frame):
import pdb
pdb.Pdb().set_trace(frame)
if options.debug:
signal.signal(signal.SIGUSR2, handle_pdb)
log.debug("Creating application instance")
app = create_app(
options.debug,
options.secret,
options.gzip,
)
log.debug("Creating IOLoop instance.")
io_loop = IOLoop.current()
io_loop.run_sync(lambda: init_db(options.database))
if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)):
os.makedirs(options.cache_dir)
Cache.CACHE_DIR = options.cache_dir
log.info("Init thread pool with %d threads", options.pool_size)
handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size)
AsyncHTTPClient.configure(None, max_clients=options.max_http_clients)
proxy_url = URL(os.getenv('{0}_proxy'.format(options.pypi_server.scheme)))
if proxy_url:
log.debug("Configuring for proxy: %s", proxy_url)
AsyncHTTPClient.configure(
'tornado.curl_httpclient.CurlAsyncHTTPClient',
defaults={
'proxy_host': proxy_url.host,
'proxy_port': proxy_url.port,
'proxy_username': proxy_url.user,
'proxy_password': proxy_url.password,
}
)
PYPIClient.configure(
options.pypi_server,
handlers.base.BaseHandler.THREAD_POOL
)
if options.pypi_proxy:
pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop)
io_loop.add_callback(PYPIClient.packages)
io_loop.add_callback(pypi_updater.start)
log.info("Starting server http://%s:%d/", options.address, options.port)
http_server = HTTPServer(app, xheaders=options.proxy_mode)
http_server.listen(options.port, address=options.address)
log.debug('Setting "%s" as storage', options.storage)
PackageFile.set_storage(options.storage)
log.debug("Starting main loop")
io_loop.start()
except Exception as e:
log.fatal("Exception on main loop:")
log.exception(e)
exit(1)
else:
exit(0)
def release_fetch(package, rel):
version_info, release_files = yield PYPIClient.release_data(
package.name, rel)
raise Return((yield release_db_save(package, rel, version_info,
release_files)))
