def test_data_file(fn):
fpath = os.path.join(os.path.split(__file__)[0], 'data', fn)
if not os.path.exists(fpath):
url = 'http://kinherd.org/pyrocko_test_data/' + fn
util.download_file(url, fpath)
return fpath
def test_download(self):
fn = self.fpath('responses.xml')
url = 'http://data.pyrocko.org/examples/responses.xml'
stat = []
def status(d):
stat.append(d)
util.download_file(url, fn, status_callback=status)
url = 'http://data.pyrocko.org/testing/my_test_dir'
dn = self.fpath('my_test_dir')
util.download_dir(url, dn, status_callback=status)
d = stat[-1]
dwant = {
'ntotal_files': 4,
'nread_files': 4,
'ntotal_bytes_all_files': 22,
'nread_bytes_all_files': 22,
'ntotal_bytes_current_file': 8,
'nread_bytes_current_file': 8}
for k in dwant:
assert k in d
assert d[k] == dwant[k]
def test_data_file(fn):
fpath = test_data_file_no_download(fn)
if not os.path.exists(fpath):
url = 'http://kinherd.org/pyrocko_test_data/' + fn
util.download_file(url, fpath)
return fpath
def download_file(self, url, fpath, username=None, password=None):
util.download_file(
url,
fpath,
username,
password,
status_callback=get_download_callback(
'Downloading %s topography...' % self.__class__.__name__))
def _getRepositoryDatabase():
from pyrocko import config
name = path.basename(db_url)
data_path = path.join(config.config().crustdb_dir, name)
if not path.exists(data_path):
from pyrocko import util
util.download_file(db_url, data_path, None, None)
return data_path
def get_test_data(path):
fn = test_data_path(path)
if not op.exists(fn):
util.ensuredirs(fn)
if path.endswith('/'):
util.download_dir(op.join(url, path), fn)
else:
util.download_file(op.join(url, path), fn)
return fn
def _getRepositoryDatabase():
from pyrocko import config
name = path.basename(db_url)
data_path = path.join(config.config().crustdb_dir, name)
if not path.exists(data_path):
from pyrocko import util
util.download_file(db_url, data_path, None, None)
return data_path
def download_file(self,
url,
fpath,
username=None,
password=None,
status_callback=None):
util.download_file(url,
fpath,
username,
password,
status_callback=status_callback)
def test_data_file(fn):
fpath = test_data_file_no_download(fn)
if not os.path.exists(fpath):
if not have_internet():
raise unittest.SkipTest(
'need internet access to download data file')
url = 'http://data.pyrocko.org/testing/' + fn
logger.info('downloading %s' % url)
util.download_file(url, fpath)
return fpath
def download(self):
logger.info('Downloading Holocene volcanoes...')
util.download_file(self.URL_HOLOCENE,
self.fname_holocene,
status_callback=get_download_callback(
'Downloading Holocene volcanoe database...'))
logger.info('Downloading Pleistocene volcanoes...')
util.download_file(self.URL_PLEISTOCENE,
self.fname_pleistocene,
status_callback=get_download_callback(
'Downloading Pleistocene volcanoe database...'))
def get_test_data(self, path):
fn = op.join(test_data, path)
if not os.exists(test_data):
os.mkdir(test_data)
if not os.exists(fn):
if path.endswith('/'):
util.download_dir(op.join(url, path), fn)
else:
util.download_file(op.join(url, path), fn)
return fn
def _download_data(fn, dldir=False):
fpath = data_file_path(fn)
if not op.exists(fpath):
if not have_internet():
raise NoInternetConnection('need internet access to download data')
url = 'http://data.pyrocko.org/examples/' + fn
print('downloading %s' % url)
if dldir:
util.download_dir(url, fpath)
else:
util.download_file(url, fpath)
return fpath
def _get_database(cls, filename):
file = path.join(config.gshhg_dir, filename)
if not path.exists(file):
from pyrocko import util
import zipfile
archive_path = path.join(config.gshhg_dir,
path.basename(cls.gshhg_url))
util.download_file(cls.gshhg_url, archive_path)
if not zipfile.is_zipfile(archive_path):
raise util.DownloadError('GSHHG file is corrupted!')
logger.info('Unzipping GSHHG database...')
zipf = zipfile.ZipFile(archive_path)
zipf.extractall(config.gshhg_dir)
else:
logger.debug('Using cached %s' % filename)
return file
def _get_database(cls, filename):
file = path.join(config.gshhg_dir, filename)
if not path.exists(file):
from pyrocko import util
import zipfile
archive_path = path.join(config.gshhg_dir,
path.basename(cls.gshhg_url))
util.download_file(cls.gshhg_url, archive_path)
if not zipfile.is_zipfile(archive_path):
raise util.DownloadError('GSHHG file is corrupted!')
logger.info('Unzipping GSHHG database...')
zipf = zipfile.ZipFile(archive_path)
zipf.extractall(config.gshhg_dir)
else:
logger.debug('Using cached %s' % filename)
return file
def get_example_data(filename, url=None, recursive=False):
'''
Download example data file needed in tutorials.
The file is downloaded to given ``filename``. If there already exists a
file with that name, nothing is done.
:param filename: name of the required file
:param url: if not ``None`` get file from given URL otherwise fetch it from
http://data.pyrocko.org/examples/<filename>.
:returns: ``filename``
'''
if not os.path.exists(filename):
url = 'http://data.pyrocko.org/examples/' + filename
if recursive:
util.download_dir(url, filename)
else:
util.download_file(url, filename)
return filename
def get_example_data(filename, url=None, recursive=False):
'''
Download example data file needed in tutorials.
The file is downloaded to given ``filename``. If there already exists a
file with that name, nothing is done.
:param filename: name of the required file
:param url: if not ``None`` get file from given URL otherwise fetch it from
http://data.pyrocko.org/examples/<filename>.
:returns: ``filename``
'''
if not os.path.exists(filename):
url = 'http://data.pyrocko.org/examples/' + filename
if recursive:
util.download_dir(url, filename)
else:
util.download_file(url, filename)
return filename
def download_file(self, url, fpath, username=None, password=None):
util.download_file(url, fpath, username, password)
def download_file(self, url, fpath):
util.download_file(url, fpath)
