def proxy_package(cls, package): try: remote_package_exists = yield PYPIClient.exists(package) if remote_package_exists: pkg_real_name = yield PYPIClient.find_real_name(package) pkg = yield proxy_remote_package(pkg_real_name) raise Return(pkg) raise LookupError("Remote package not found") except (LookupError, HTTPClientError) as e: if isinstance(e, HTTPClientError): log.warning("PYPI backend return an error: %s", e) raise Return(Package.find(package))
def proxy_package(cls, package): try: remote_package_exists = yield PYPIClient.exists(package) if remote_package_exists: pkg_real_name = yield PYPIClient.find_real_name(package) pkg = yield proxy_remote_package(pkg_real_name) raise Return(pkg) raise LookupError("Remote package not found") except (LookupError, HTTPClientError) as e: if isinstance(e, HTTPClientError): log.warning("PYPI backend return an error: %s", e) raise Return(Package.find(package))
def get(self, package, version, filename): try: package = yield PYPIClient.find_real_name(package) except (LookupError, HTTPClientError) as e: if isinstance(e, HTTPClientError): log.warning("PYPI backend return an error: %s", e) package = yield self.thread_pool.submit(Package.find, package) try: pkg_file = yield self.find_file(package, version, filename) if not pkg_file.fetched: yield self.fetch_remote_file(pkg_file) except LookupError: self.send_error(404) else: self.set_header("MD5", pkg_file.md5) self.set_header("ETag", pkg_file.md5) self.set_header("Content-Length", pkg_file.size) self.set_header("Content-Type", 'application/octet-stream') self.set_header("Date", pkg_file.ts.strftime("%a, %d %b %Y %H:%M:%S %Z")) with pkg_file.open() as f: reader = threaded(f.read) data = yield reader(self.CHUNK_SIZE) self.write(data) yield Task(self.flush) while data: data = yield reader(self.CHUNK_SIZE) self.write(data) yield Task(self.flush) self.finish()
def proxy_remote_package(package): pkg = yield threaded(Package.get_or_create)(name=package, proxy=True) releases, cached_releases = yield [PYPIClient.releases(pkg.name), threaded(pkg.versions)(True)] IOLoop.current().add_callback(threaded(pkg.hide_versions), filter(lambda x: not x.hidden, releases)) cached_releases = set(map(lambda x: x.version, cached_releases)) new_releases = list(releases - cached_releases) for release_part in chunks(new_releases, 10): yield [release_fetch(pkg, release) for release in release_part] raise Return(pkg)
def proxy_remote_package(package): pkg = yield threaded(Package.get_or_create)(name=package, proxy=True) releases, cached_releases = yield [ PYPIClient.releases(pkg.name), threaded(pkg.versions)(True) ] IOLoop.current().add_callback(threaded(pkg.hide_versions), filter(lambda x: not x.hidden, releases)) cached_releases = set(map(lambda x: x.version, cached_releases)) new_releases = list(releases - cached_releases) for release_part in chunks(new_releases, 10): yield [release_fetch(pkg, release) for release in release_part] raise Return(pkg)
def rpc_search(self, query, operator='or'): results = [] assert operator in ("and", "or"), "Operator must be 'and' or 'or'" names = tuple(query.get('name', [])) descriptions = tuple(query.get('summary', [])) async_queries = yield [ self._search(names, descriptions, operator), PYPIClient.search(names, descriptions, operator) ] for res in async_queries: for result in res: results.append(result) raise Return(results)
def get(self, package, version, filename): try: package = yield PYPIClient.find_real_name(package) except (LookupError, HTTPClientError) as e: if isinstance(e, HTTPClientError): log.warning("PYPI backend return an error: %s", e) package = yield self.thread_pool.submit(Package.find, package) try: pkg_file = yield self.find_file(package, version, filename) if not pkg_file.fetched: yield self.fetch_remote_file(pkg_file) except LookupError: self.send_error(404) else: self.set_header("MD5", pkg_file.md5) self.set_header("ETag", pkg_file.md5) self.set_header("Content-Length", pkg_file.size) self.set_header("Content-Type", 'application/octet-stream') self.set_header("Date", pkg_file.ts.strftime("%a, %d %b %Y %H:%M:%S %Z")) with pkg_file.open() as f: reader = threaded(f.read) data = yield reader(self.CHUNK_SIZE) self.write(data) yield Task(self.flush) for chunk in iter(lambda: reader(self.CHUNK_SIZE), None): data = yield chunk if not data: break self.write(data) yield Task(self.flush) self.finish()
def run(): options.parse_command_line() if options.config: options.parse_config_file(options.config) options.storage = os.path.abspath(options.storage) if os.getuid() == 0 and options.user: pw = pwd.getpwnam(options.user) uid, gid = pw.pw_uid, pw.pw_gid log.info("Changind user to %s [%s:%s]", options.user, uid, gid) os.setgid(uid) os.setuid(uid) try: if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)): log.info('Creating new package storage directory: "%s"', options.storage) os.makedirs(options.storage) def on_interrupt(*args): log.warning("Receiving interrupt signal. Application will be stopped.") exit(errno.EINTR) log.debug("Preparing signal handling") for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): signal.signal(sig, on_interrupt) def handle_pdb(sig, frame): import pdb pdb.Pdb().set_trace(frame) if options.debug: signal.signal(signal.SIGUSR2, handle_pdb) log.debug("Creating application instance") app = create_app( options.debug, options.secret, options.gzip, ) log.debug("Creating IOLoop instance.") io_loop = IOLoop.current() io_loop.run_sync(lambda: init_db(options.database)) if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)): os.makedirs(options.cache_dir) Cache.CACHE_DIR = options.cache_dir log.info("Init thread pool with %d threads", options.pool_size) handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size) AsyncHTTPClient.configure(None, max_clients=options.max_http_clients) PYPIClient.configure( options.pypi_server, handlers.base.BaseHandler.THREAD_POOL ) pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop) io_loop.add_callback(PYPIClient.packages) io_loop.add_callback(pypi_updater.start) log.info("Starting server http://%s:%d/", options.address, options.port) http_server = HTTPServer(app, xheaders=options.proxy_mode) http_server.listen(options.port, address=options.address) log.debug('Setting "%s" as storage', options.storage) PackageFile.set_storage(options.storage) log.debug("Starting main loop") io_loop.start() except Exception as e: log.fatal("Exception on main loop:") log.exception(e) exit(1) else: exit(0)
def release_fetch(package, rel): version_info, release_files = yield PYPIClient.release_data(package.name, rel) raise Return((yield release_db_save(package, rel, version_info, release_files)))
def run(): options.parse_command_line() if options.config: options.parse_config_file(options.config) options.storage = os.path.abspath(options.storage) if os.getuid() == 0 and options.user: pw = pwd.getpwnam(options.user) uid, gid = pw.pw_uid, pw.pw_gid log.info("Changind user to %s [%s:%s]", options.user, uid, gid) os.setgid(uid) os.setuid(uid) try: if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)): log.info('Creating new package storage directory: "%s"', options.storage) os.makedirs(options.storage) def on_interrupt(*args): log.warning("Receiving interrupt signal. Application will be stopped.") exit(errno.EINTR) log.debug("Preparing signal handling") for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): signal.signal(sig, on_interrupt) def handle_pdb(sig, frame): import pdb pdb.Pdb().set_trace(frame) if options.debug: signal.signal(signal.SIGUSR2, handle_pdb) log.debug("Creating application instance") app = create_app( options.debug, options.secret, options.gzip, ) log.debug("Creating IOLoop instance.") io_loop = IOLoop.current() io_loop.run_sync(lambda: init_db(options.database)) if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)): os.makedirs(options.cache_dir) Cache.CACHE_DIR = options.cache_dir log.info("Init thread pool with %d threads", options.pool_size) handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size) AsyncHTTPClient.configure(None, max_clients=options.max_http_clients) proxy_url = URL(os.getenv('{0}_proxy'.format(options.pypi_server.scheme))) if proxy_url: log.debug("Configuring for proxy: %s", proxy_url) AsyncHTTPClient.configure( 'tornado.curl_httpclient.CurlAsyncHTTPClient', defaults={ 'proxy_host': proxy_url.host, 'proxy_port': proxy_url.port, 'proxy_username': proxy_url.user, 'proxy_password': proxy_url.password, } ) PYPIClient.configure( options.pypi_server, handlers.base.BaseHandler.THREAD_POOL ) if options.pypi_proxy: pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop) io_loop.add_callback(PYPIClient.packages) io_loop.add_callback(pypi_updater.start) log.info("Starting server http://%s:%d/", options.address, options.port) http_server = HTTPServer(app, xheaders=options.proxy_mode) http_server.listen(options.port, address=options.address) log.debug('Setting "%s" as storage', options.storage) PackageFile.set_storage(options.storage) log.debug("Starting main loop") io_loop.start() except Exception as e: log.fatal("Exception on main loop:") log.exception(e) exit(1) else: exit(0)
def release_fetch(package, rel): version_info, release_files = yield PYPIClient.release_data( package.name, rel) raise Return((yield release_db_save(package, rel, version_info, release_files)))