def test_from_config():
generic_url = 'http://example.com/generic_ne/{name}.zip'
land_downloader = cio.Downloader(generic_url, '', '')
generic_ne_downloader = cio.Downloader(generic_url, '', '')
ocean_spec = ('shapefile', 'natural_earth', '110m', 'physical', 'ocean')
land_spec = ('shapefile', 'natural_earth', '110m', 'physical', 'land')
generic_spec = ('shapefile', 'natural_earth')
target_config = {
land_spec: land_downloader,
generic_spec: generic_ne_downloader,
}
with config_replace(target_config):
# ocean spec is not explicitly in the config, but a subset of it is,
# so check that an appropriate downloader is returned
r = cio.Downloader.from_config(ocean_spec)
# check the resulting download item produces a sensible url.
assert_equal(r.url({'name': 'ocean'}),
'http://example.com/generic_ne/ocean.zip')
downloaders = cio.config['downloaders']
r = cio.Downloader.from_config(land_spec)
assert r is land_downloader
def test_downloading_simple_ascii():
# downloads a file from the Google APIs. (very high uptime and file will
# always be there - if this goes down, most of the internet would break!)
# to test the downloading mechanisms.
file_url = 'http://ajax.googleapis.com/ajax/libs/jquery/1.8.2/{name}.js'
format_dict = {'name': 'jquery'}
with download_to_temp() as tmp_dir:
target_template = os.path.join(tmp_dir, '{name}.txt')
tmp_fname = target_template.format(**format_dict)
dnld_item = cio.Downloader(file_url, target_template)
assert_equal(dnld_item.target_path(format_dict), tmp_fname)
with warnings.catch_warnings(record=True) as w:
assert_equal(dnld_item.path(format_dict), tmp_fname)
assert len(w) == 1, ('Expected a single download warning to be '
'raised. Got {}.'.format(len(w)))
assert issubclass(w[0].category, cio.DownloadWarning)
with open(tmp_fname, 'r') as fh:
_ = fh.readline()
assert_equal(" * jQuery JavaScript Library v1.8.2\n",
fh.readline())
# check that calling path again doesn't try re-downloading
with CallCounter(dnld_item, 'acquire_resource') as counter:
assert_equal(dnld_item.path(format_dict), tmp_fname)
assert counter.count == 0, 'Item was re-downloaded.'
def test_downloading_simple_ascii(download_to_temp):
# downloads a file from the Google APIs. (very high uptime and file will
# always be there - if this goes down, most of the internet would break!)
# to test the downloading mechanisms.
file_url = 'https://ajax.googleapis.com/ajax/libs/jquery/1.8.2/{name}.js'
format_dict = {'name': 'jquery'}
target_template = str(download_to_temp / '{name}.txt')
tmp_fname = target_template.format(**format_dict)
dnld_item = cio.Downloader(file_url, target_template)
assert dnld_item.target_path(format_dict) == tmp_fname
with pytest.warns(cio.DownloadWarning):
assert dnld_item.path(format_dict) == tmp_fname
with open(tmp_fname) as fh:
fh.readline()
assert fh.readline() == " * jQuery JavaScript Library v1.8.2\n"
# check that calling path again doesn't try re-downloading
with mock.patch.object(dnld_item,
'acquire_resource',
wraps=dnld_item.acquire_resource) as counter:
assert dnld_item.path(format_dict) == tmp_fname
counter.assert_not_called()
def test_Downloader_data():
di = cio.Downloader('https://testing.com/{category}/{name}.zip',
os.path.join('{data_dir}', '{category}',
'shape.shp'),
'/project/foobar/{category}/sample.shp')
replacement_dict = {'category': 'example',
'name': 'test',
'data_dir': os.path.join('/wibble', 'foo', 'bar')}
assert di.url(replacement_dict) == 'https://testing.com/example/test.zip'
assert (di.target_path(replacement_dict) ==
os.path.join('/wibble', 'foo', 'bar', 'example', 'shape.shp'))
assert (di.pre_downloaded_path(replacement_dict) ==
'/project/foobar/example/sample.shp')
