def download(self, allow_upstream=False): """ Download the tarball to the upstream directory. If allow_upstream is False and the package cannot be found on the sage mirrors, fall back to downloading it from the upstream URL if the package has one. """ destination = self.upstream_fqn if os.path.isfile(destination): if self.checksum_verifies(): log.info('Using cached file {destination}'.format( destination=destination)) return else: # Garbage in the upstream directory? Ignore it. # Don't delete it because maybe somebody just forgot to # update the checksum (Trac #23972). log.warning( 'Invalid checksum; ignoring cached file {destination}'. format(destination=destination)) successful_download = False log.info('Attempting to download package {0} from mirrors'.format( self.filename)) for mirror in MirrorList(): url = mirror + '/'.join( ['spkg', 'upstream', self.package.name, self.filename]) log.info(url) try: Download(url, destination).run() successful_download = True break except IOError: log.debug('File not on mirror') if not successful_download: url = self.package.tarball_upstream_url if allow_upstream and url: log.info('Attempting to download from {}'.format(url)) try: Download(url, destination).run() successful_download = True except IOError: raise FileNotMirroredError( 'tarball does not exist on mirror network and neither at the upstream URL' ) else: raise FileNotMirroredError( 'tarball does not exist on mirror network') if not self.checksum_verifies(): raise ChecksumError('checksum does not match')
def download(self): """ Download the tarball to the upstream directory. """ destination = os.path.join(SAGE_DISTFILES, self.filename) if os.path.isfile(destination): if self.checksum_verifies(): log.info('Using cached file {destination}'.format(destination=destination)) return else: # Garbage in the upstream directory? Delete and re-download log.info('Invalid checksum for cached file {destination}, deleting' .format(destination=destination)) os.remove(destination) successful_download = False log.info('Attempting to download package {0} from mirrors'.format(self.filename)) for mirror in MirrorList(): url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename]) log.info(url) try: Download(url, self.upstream_fqn).run() successful_download = True break except IOError: log.debug('File not on mirror') if not successful_download: raise FileNotMirroredError('tarball does not exist on mirror network') if not self.checksum_verifies(): raise ChecksumError('checksum does not match')
def run(self): progress = True url = None destination = None for arg in sys.argv[1:]: if arg.startswith('--print-fastest-mirror'): print(MirrorList().fastest) sys.exit(0) if arg.startswith('--quiet'): progress = False continue if url is None: url = arg continue if destination is None: destination = arg continue raise ValueError('too many arguments') if url is None: print(dedent(self.__doc__.format(SAGE_DISTFILES=SAGE_DISTFILES))) sys.exit(1) if url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'): Download(url, destination, progress=progress, ignore_errors=True).run() else: # url is a tarball name tarball = Tarball(url) tarball.download() if destination is not None: tarball.save_as(destination)
def download_upstream(self, download_url=None): tarball = self.package.tarball if download_url is None: download_url = self.package.tarball_upstream_url if download_url is None: raise ValueError( "package has no default upstream_url pattern, download_url needed" ) print('Downloading tarball to {0}'.format(tarball.upstream_fqn)) Download(download_url, tarball.upstream_fqn).run()
def run(self): progress = True url = None print_fastest_mirror = None destination = None for arg in sys.argv[1:]: if arg.startswith('--print-fastest-mirror'): url = "" print_fastest_mirror = True continue if arg.startswith('--quiet'): progress = False continue if url is None: url = arg continue if destination is None: destination = arg continue raise ValueError('too many arguments') if url is None: print(dedent(self.__doc__.format(SAGE_DISTFILES=SAGE_DISTFILES))) sys.exit(2) try: if url.startswith('http://') or url.startswith( 'https://') or url.startswith('ftp://'): Download(url, destination, progress=progress, ignore_errors=True).run() elif print_fastest_mirror: url = "fastest mirror" # For error message print(MirrorList().fastest) else: # url is a tarball name tarball = Tarball(url) tarball.download() if destination is not None: tarball.save_as(destination) except BaseException: try: stars = '*' * 72 + '\n' sys.stderr.write(stars) try: import traceback traceback.print_exc(file=sys.stderr) sys.stderr.write(stars) except: pass sys.stderr.write("Error downloading %s\n" % (url, )) sys.stderr.write(stars) finally: sys.exit(1)
def test_download_mirror_list(self): tmp = tempfile.NamedTemporaryFile() tmp.close() progress = StringIO() Download(MirrorList.URL, tmp.name, progress=progress).run() self.assertEqual( progress.getvalue(), '[......................................................................]\n' ) with open(tmp.name, 'r') as f: content = f.read() self.assertTrue(content.startswith('# Sage Mirror List'))
def test_error(self): URL = 'http://files.sagemath.org/sage_bootstrap/this_url_does_not_exist' progress = StringIO() download = Download(URL, progress=progress) log = CapturedLog() def action(): with log: download.run() self.assertRaises(IOError, action) self.assertIsNotFoundError(log.messages()) self.assertEqual( progress.getvalue(), '[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]\n' )
def test_ignore_errors(self): URL = 'http://files.sagemath.org/sage_bootstrap/this_url_does_not_exist' with CapturedLog() as log: Download(URL, progress=False, ignore_errors=True).run() self.assertIsNotFoundError(log.messages())
def download_upstream(self, download_url): tarball = self.package.tarball print('Downloading tarball to {0}'.format(tarball.upstream_fqn)) Download(download_url, tarball.upstream_fqn).run()