def _downloadData(self):
"""
Download SWEETCAT and write it to file
"""
urls = ['https://www.astro.up.pt/resources/sweet-cat/download.php', \
"https://raw.githubusercontent.com/iastro-pt/SWEET-Cat/master/WEBSITE_online_EU.rdb"]
dfn = self._fs.composeFilename(self.dataFileName)
success = False
ex = []
for url in urls:
try:
# The next two lines by-pass certificate verification
context = ssl._create_unverified_context()
self._fs.downloadToFile(url, dfn, clobber=True, verbose=False, openMethod=gzip.open,
context=context)
success = True
except AttributeError:
# Python version does not support ssl._create_unverified_context()
self._fs.downloadToFile(
url, dfn, clobber=True, verbose=False, openMethod=gzip.open)
success = True
except Exception as e:
# "Handle" unexpected error
ex.append((url,e))
if success:
break
if not success:
raise(PE.PyANetworkError(
"Could not download SWEET-Cat data. Received the following errors:\n\n" + \
''.join(["URL: "+x[0]+"\n"+str(x[1])+"\n\n" for x in ex]) ))
self.source_url = url
def _checkContext(self, url="http://www.google.com", context=None, raiseOther=True):
"""
Check whether `context` parameter is available in urllib.
Parameters
----------
url : string, optional
The reference URL tried to be reached to check
network availability.
context : context, optional
Context used in attempt. Default is None.
raiseOther : boolean, optional
If True (default), an exception is raised if an error other
than the anticipated TypeError is raised.
"""
contextWorks = True
try:
_ = urllib.request.urlopen(url, context=context)
except TypeError as e:
# Context appears not to work
contextWorks = False
except Exception as e:
# Another error occurred
if raiseOther:
raise(PE.PyANetworkError("Unknown network error occurred.", \
tbfe=e))
return contextWorks
def _checkOnline(self, url="http://www.google.com", raiseNOC=True):
"""
Check whether network can be reached.
Parameters
----------
url : string, optional
The reference URL tried to be reached to check
network availability.
raiseNOC : boolean, optional
If True (default), raise an exception if network
cannot be reached.
Returns
-------
Online flag : boolean
True if network could be reached.
"""
online = True
try:
_ = urllib.request.urlopen(url)
except Exception as e:
# No connection seems to be available
online = False
if raiseNOC:
raise(PE.PyANetworkError("You seem to be offline. Could not reach URL: '" + str(url) + "'.", \
solution="Get online", \
tbfe=e))
return online
def _downloadData(self):
"""
Download SWEETCAT and write it to file
"""
url = 'https://www.astro.up.pt/resources/sweet-cat/download.php'
dfn = self._fs.composeFilename(self.dataFileName)
try:
# The next two lines by-pass certificate verification
context = ssl._create_unverified_context()
self._fs.downloadToFile(url, dfn, clobber=True, verbose=False, openMethod=gzip.open, \
context=context)
except AttributeError:
# Python version does not support ssl._create_unverified_context()
self._fs.downloadToFile(url,
dfn,
clobber=True,
verbose=False,
openMethod=gzip.open)
except Exception as e:
# "Handle" unexpected error
raise (PE.PyANetworkError(
"Could not download SWEET-Cat data. The following error was raised: "
+ str(e)))
def downloadToFile(self,
url,
fn,
clobber=False,
verbose=True,
openMethod=open,
context=None):
"""
Download content from URL.
Parameters
----------
url : string
The location of the content.
fn : string
The relative or absolute name of the file
to which the download shall be saved.
clobber : boolean, optional
If True, an existing file will be overwritten.
verbose : boolean, optional
If True, information on the download will be
printed to the screen.
openMethod : callable
The method used to open the file to write to (default is
open, other choices may be gzip.open or io.ipen)
context : ssl context
SSL context parameter handed to urlopen.
"""
if self.fileExists(fn) and (clobber == False):
return
if not pyaRC.supposedOnline():
PE.warn(PE.PyANetworkError("Internet connection disallowed by pyaRC.", \
solution="Use 'goOnline' method in pyaRC."))
return
def download(url, context, nocontext=False):
if not nocontext:
# Use context
response = urllib.request.urlopen(url, context=context)
else:
# Disregard context
response = urllib.request.urlopen(url)
data = response.read() # a `bytes` object
self.requestFile(fn, 'wb', openMethod).write(data)
ana = self._analyzeFilename(fn, True)
self.touchFile(ana["fullname"])
try:
if verbose:
print("PyA download info:")
print(" - Downloading from URL: " + str(url))
download(url, context)
except (KeyboardInterrupt, SystemExit):
self.removeFile(ana["fullname"])
raise
except TypeError as e:
# Possibly, context is not supported
cs = self._checkContext()
if not cs:
# Network is all right, but context parameter must
# not be specified.
if verbose:
print("PyA download info:")
print(" - Downloading from URL: " + str(url) +
", (no context)")
download(url, context, nocontext=True)
except URLError as e:
# Trying to download without context (side-lining ssl verification!)
if verbose:
print("PyA download info:")
print(" - Downloading from URL: " + str(url) +
", (unverified context)")
context = ssl._create_unverified_context()
download(url, context)
except Exception as e:
self.removeFile(ana["fullname"])
sols = ["Check whether URL exists and is spelled correctly."]
# Check whether network can be reached.
netreach = self._checkOnline(raiseNOC=False)
if not netreach:
sols.append(
"Network could not be reached. Check your network status.")
raise (PE.PyADownloadError(
"Could not download data from URL: " + str(url) + ".\n",
solution=sols,
tbfe=e,
addInfo="Could network be reached (online)? " + {
True: "yes",
False: "No"
}[netreach]))
if verbose:
print(" - Downloaded " +
str(os.path.getsize(ana["fullname"]) / 1000.0) + " kb")
print(" to file: " + ana["fullname"])