def from_target(cls, config, target, conn_timeout=None): from twitter.common.python.fetcher import PyPIFetcher, Fetcher from twitter.common.python.resolver import Resolver from twitter.common.python.http import Crawler from twitter.common.quantity import Amount, Time conn_timeout_amount = Amount( conn_timeout, Time.SECONDS) if conn_timeout is not None else None crawler = Crawler(cache=config.get('python-setup', 'download_cache'), conn_timeout=conn_timeout_amount) fetchers = [] fetchers.extend( Fetcher([url]) for url in config.getlist('python-repos', 'repos', [])) fetchers.extend( PyPIFetcher(url) for url in config.getlist('python-repos', 'indices', [])) platforms = config.getlist('python-setup', 'platforms', ['current']) if isinstance(target, PythonBinary) and target.platforms: platforms = target.platforms return cls(platforms=get_platforms(platforms), resolver=Resolver(cache=config.get('python-setup', 'install_cache'), crawler=crawler, fetchers=fetchers, install_cache=config.get( 'python-setup', 'install_cache'), conn_timeout=conn_timeout_amount))
def crawler(cls, config, conn_timeout=None): return Crawler(cache=config.get('python-setup', 'download_cache'), conn_timeout=conn_timeout)
def crawler_from_config(config, conn_timeout=None): download_cache = PythonSetup(config).scratch_dir('download_cache', default_name='downloads') return Crawler(cache=download_cache, conn_timeout=conn_timeout)