def download(self, allow_upstream=False):
"""
Download the tarball to the upstream directory.
If allow_upstream is False and the package cannot be found
on the sage mirrors, fall back to downloading it from
the upstream URL if the package has one.
"""
destination = self.upstream_fqn
if os.path.isfile(destination):
if self.checksum_verifies():
log.info('Using cached file {destination}'.format(
destination=destination))
return
else:
# Garbage in the upstream directory? Ignore it.
# Don't delete it because maybe somebody just forgot to
# update the checksum (Trac #23972).
log.warning(
'Invalid checksum; ignoring cached file {destination}'.
format(destination=destination))
successful_download = False
log.info('Attempting to download package {0} from mirrors'.format(
self.filename))
for mirror in MirrorList():
url = mirror + '/'.join(
['spkg', 'upstream', self.package.name, self.filename])
log.info(url)
try:
Download(url, destination).run()
successful_download = True
break
except IOError:
log.debug('File not on mirror')
if not successful_download:
url = self.package.tarball_upstream_url
if allow_upstream and url:
log.info('Attempting to download from {}'.format(url))
try:
Download(url, destination).run()
successful_download = True
except IOError:
raise FileNotMirroredError(
'tarball does not exist on mirror network and neither at the upstream URL'
)
else:
raise FileNotMirroredError(
'tarball does not exist on mirror network')
if not self.checksum_verifies():
raise ChecksumError('checksum does not match')
def download(self):
"""
Download the tarball to the upstream directory.
"""
destination = os.path.join(SAGE_DISTFILES, self.filename)
if os.path.isfile(destination):
if self.checksum_verifies():
log.info('Using cached file {destination}'.format(destination=destination))
return
else:
# Garbage in the upstream directory? Delete and re-download
log.info('Invalid checksum for cached file {destination}, deleting'
.format(destination=destination))
os.remove(destination)
successful_download = False
log.info('Attempting to download package {0} from mirrors'.format(self.filename))
for mirror in MirrorList():
url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename])
log.info(url)
try:
Download(url, self.upstream_fqn).run()
successful_download = True
break
except IOError:
log.debug('File not on mirror')
if not successful_download:
raise FileNotMirroredError('tarball does not exist on mirror network')
if not self.checksum_verifies():
raise ChecksumError('checksum does not match')
def run(self):
progress = True
url = None
destination = None
for arg in sys.argv[1:]:
if arg.startswith('--print-fastest-mirror'):
print(MirrorList().fastest)
sys.exit(0)
if arg.startswith('--quiet'):
progress = False
continue
if url is None:
url = arg
continue
if destination is None:
destination = arg
continue
raise ValueError('too many arguments')
if url is None:
print(dedent(self.__doc__.format(SAGE_DISTFILES=SAGE_DISTFILES)))
sys.exit(1)
if url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'):
Download(url, destination, progress=progress, ignore_errors=True).run()
else:
# url is a tarball name
tarball = Tarball(url)
tarball.download()
if destination is not None:
tarball.save_as(destination)
def download_upstream(self, download_url=None):
tarball = self.package.tarball
if download_url is None:
download_url = self.package.tarball_upstream_url
if download_url is None:
raise ValueError(
"package has no default upstream_url pattern, download_url needed"
)
print('Downloading tarball to {0}'.format(tarball.upstream_fqn))
Download(download_url, tarball.upstream_fqn).run()
def run(self):
progress = True
url = None
print_fastest_mirror = None
destination = None
for arg in sys.argv[1:]:
if arg.startswith('--print-fastest-mirror'):
url = ""
print_fastest_mirror = True
continue
if arg.startswith('--quiet'):
progress = False
continue
if url is None:
url = arg
continue
if destination is None:
destination = arg
continue
raise ValueError('too many arguments')
if url is None:
print(dedent(self.__doc__.format(SAGE_DISTFILES=SAGE_DISTFILES)))
sys.exit(2)
try:
if url.startswith('http://') or url.startswith(
'https://') or url.startswith('ftp://'):
Download(url,
destination,
progress=progress,
ignore_errors=True).run()
elif print_fastest_mirror:
url = "fastest mirror" # For error message
print(MirrorList().fastest)
else:
# url is a tarball name
tarball = Tarball(url)
tarball.download()
if destination is not None:
tarball.save_as(destination)
except BaseException:
try:
stars = '*' * 72 + '\n'
sys.stderr.write(stars)
try:
import traceback
traceback.print_exc(file=sys.stderr)
sys.stderr.write(stars)
except:
pass
sys.stderr.write("Error downloading %s\n" % (url, ))
sys.stderr.write(stars)
finally:
sys.exit(1)
def test_download_mirror_list(self):
tmp = tempfile.NamedTemporaryFile()
tmp.close()
progress = StringIO()
Download(MirrorList.URL, tmp.name, progress=progress).run()
self.assertEqual(
progress.getvalue(),
'[......................................................................]\n'
)
with open(tmp.name, 'r') as f:
content = f.read()
self.assertTrue(content.startswith('# Sage Mirror List'))
def test_error(self):
URL = 'http://files.sagemath.org/sage_bootstrap/this_url_does_not_exist'
progress = StringIO()
download = Download(URL, progress=progress)
log = CapturedLog()
def action():
with log:
download.run()
self.assertRaises(IOError, action)
self.assertIsNotFoundError(log.messages())
self.assertEqual(
progress.getvalue(),
'[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]\n'
)
def test_ignore_errors(self):
URL = 'http://files.sagemath.org/sage_bootstrap/this_url_does_not_exist'
with CapturedLog() as log:
Download(URL, progress=False, ignore_errors=True).run()
self.assertIsNotFoundError(log.messages())
def download_upstream(self, download_url):
tarball = self.package.tarball
print('Downloading tarball to {0}'.format(tarball.upstream_fqn))
Download(download_url, tarball.upstream_fqn).run()