def mk_filename(pattern, response, sock, url, overwrite=False):
name = get_filename(sock, url)
if not name:
if not isinstance(response.fileid, unicode):
name = unicode(response.fileid, "ascii", "ignore")
else:
name = response.fileid
fs_encoding = sys.getfilesystemencoding()
if fs_encoding is None:
fs_encoding = "ascii"
name = slugify(name).encode(fs_encoding, "ignore")
if not name:
name = "file"
fname = pattern.format(file=name, **dict(response))
if not overwrite and os.path.exists(fname):
fname = replacement_filename(fname)
dir_ = os.path.abspath(os.path.dirname(fname))
if not os.path.exists(dir_):
os.makedirs(dir_)
return fname
def mk_filename(pattern, response, sock, url, overwrite=False):
name = get_filename(sock, url)
if not name:
if isinstance(response.fileid, bytes):
name = response.fileid.decode("ascii", "ignore")
else:
name = response.fileid
fs_encoding = sys.getfilesystemencoding()
if fs_encoding is None:
fs_encoding = "ascii"
name = slugify(name)
if not name:
name = "file"
fname = pattern.format(file=name, **dict(response))
if not overwrite and os.path.exists(fname):
fname = replacement_filename(fname)
dir_ = os.path.abspath(os.path.dirname(fname))
if not os.path.exists(dir_):
os.makedirs(dir_)
return fname
def download_fileobj(opn, directory, url='', default="file", overwrite=False):
"""
Download a file from a url into a directory.
Tries the "Content-Disposition", if unavailable, extracts name from the URL.
If this fails, the ``default`` keyword will be used.
Parameters
----------
opn : `~urllib.request.urlopen`
The `~urllib.request.urlopen` to download.
directory : `str`
The directory path to download the file in to.
url : `str`
The URL to parse for the filename.
default : `str`, optional
The name to use if the first two methods fail. Defaults to "file".
overwrite: `bool`, optional
If `True` will overwrite a file of the same name. Defaults to `False`.
Returns
-------
`str`:
The file path for the downloaded file.
"""
filename = get_system_filename(opn, url, default)
path = os.path.join(directory, filename.decode('utf-8'))
if overwrite and os.path.exists(path):
path = replacement_filename(path)
with open(path, 'wb') as fd:
shutil.copyfileobj(opn, fd)
return path
def get(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
Returns
-------
Results Object
"""
urls = []
for qrblock in qres:
urls.append(qrblock.url)
filenames = []
for url in urls:
filenames.append(url.split('/')[-1])
# Create function to compute the filepath to download to if not set
default_dir = sunpy.config.get("downloads", "download_dir")
paths = []
for i, filename in enumerate(filenames):
if path is None:
fname = os.path.join(default_dir, '{file}')
elif isinstance(path, six.string_types) and '{file}' not in path:
fname = os.path.join(path, '{file}')
temp_dict = qres[i].map_.copy()
temp_dict['file'] = filename
fname = fname.format(**temp_dict)
fname = os.path.expanduser(fname)
if os.path.exists(fname):
fname = replacement_filename(fname)
fname = partial(simple_path, fname)
paths.append(fname)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(
zip(urls, map(lambda x: res.require([x]), urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
return res
def download_fileobj(opn, directory, url='', default=u"file", overwrite=False):
""" Download file from url into directory. Try to get filename from
Content-Disposition header, otherwise get from path of url if given.
Fall back to default if both fail. Only overwrite existing files when
overwrite is True. """
filename = get_system_filename(opn, url, default)
path = os.path.join(directory, filename.decode('utf-8'))
if not overwrite and os.path.exists(path):
path = replacement_filename(path)
with open(path, 'wb') as fd:
shutil.copyfileobj(opn, fd)
return path
def download_fileobj(opn, directory, url='', default=u"file", overwrite=False):
""" Download file from url into directory. Try to get filename from
Content-Disposition header, otherwise get from path of url if given.
Fall back to default if both fail. Only overwrite existing files when
overwrite is True. """
filename = get_system_filename(opn, url, default)
path = os.path.join(directory, filename)
if not overwrite and os.path.exists(path):
path = replacement_filename(path)
with open(path, 'wb') as fd:
shutil.copyfileobj(opn, fd)
return path
def get(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
Returns
-------
Results Object
"""
urls = []
for qrblock in qres:
urls.append(qrblock.url)
filenames = []
for url in urls:
filenames.append(url.split('/')[-1])
paths = []
for i, filename in enumerate(filenames):
if path is None:
fname = os.path.join(get_and_create_download_dir(), '{file}')
elif isinstance(path, six.string_types) and '{file}' not in path:
fname = os.path.join(path, '{file}')
temp_dict = qres[i].map_.copy()
temp_dict['file'] = filename
fname = fname.format(**temp_dict)
fname = os.path.expanduser(fname)
if os.path.exists(fname):
fname = replacement_filename(fname)
fname = partial(simple_path, fname)
paths.append(fname)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(zip(urls, map(lambda x: res.require([x]),
urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
return res
def _get_full_filenames(self, qres, filenames, path):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
filenames : list
List of base filenames (ex - "xyz.txt")
path : string
Path to download files to
Returns
-------
List of full pathnames for each file (download_directory + filename)
"""
# Create function to compute the filepath to download to if not set
default_dir = sunpy.config.get("downloads", "download_dir")
paths = []
for i, filename in enumerate(filenames):
if path is None:
fname = os.path.join(default_dir, '{file}')
elif isinstance(path, str) and '{file}' not in path:
fname = os.path.join(path, '{file}')
temp_dict = qres[i]._map.copy()
temp_dict['file'] = filename
fname = fname.format(**temp_dict)
fname = os.path.expanduser(fname)
if os.path.exists(fname):
fname = replacement_filename(fname)
fname = partial(simple_path, fname)
paths.append(fname)
return paths
def _get_full_filenames(self, qres, filenames, path):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
filenames : list
List of base filenames (ex - "xyz.txt")
path : string
Path to download files to
Returns
-------
List of full pathnames for each file (download_directory + filename)
"""
# Create function to compute the filepath to download to if not set
default_dir = sunpy.config.get("downloads", "download_dir")
paths = []
for i, filename in enumerate(filenames):
if path is None:
fname = os.path.join(default_dir, '{file}')
elif isinstance(path, six.string_types) and '{file}' not in path:
fname = os.path.join(path, '{file}')
temp_dict = qres[i]._map.copy()
temp_dict['file'] = filename
fname = fname.format(**temp_dict)
fname = os.path.expanduser(fname)
if os.path.exists(fname):
fname = replacement_filename(fname)
fname = partial(simple_path, fname)
paths.append(fname)
return paths
fs_encoding = sys.getfilesystemencoding()
if fs_encoding is None:
fs_encoding = "ascii"
name = slugify(name)
if six.PY2:
name = name.encode(fs_encoding, "ignore")
if not name:
name = "file"
fname = pattern.format(file=name, **dict(response))
if not overwrite and os.path.exists(fname):
fname = replacement_filename(fname)
dir_ = os.path.abspath(os.path.dirname(fname))
if not os.path.exists(dir_):
os.makedirs(dir_)
return fname
# pylint: disable=R0914
def query_legacy(self, tstart=None, tend=None, **kwargs):
"""
Query data from the VSO mocking the IDL API as close as possible.
Either tstart and tend or date_start and date_end or date have
to be supplied.
Parameters
----------