def test_from_config(): generic_url = 'http://example.com/generic_ne/{name}.zip' land_downloader = cio.Downloader(generic_url, '', '') generic_ne_downloader = cio.Downloader(generic_url, '', '') ocean_spec = ('shapefile', 'natural_earth', '110m', 'physical', 'ocean') land_spec = ('shapefile', 'natural_earth', '110m', 'physical', 'land') generic_spec = ('shapefile', 'natural_earth') target_config = { land_spec: land_downloader, generic_spec: generic_ne_downloader, } with config_replace(target_config): # ocean spec is not explicitly in the config, but a subset of it is, # so check that an appropriate downloader is returned r = cio.Downloader.from_config(ocean_spec) # check the resulting download item produces a sensible url. assert_equal(r.url({'name': 'ocean'}), 'http://example.com/generic_ne/ocean.zip') downloaders = cio.config['downloaders'] r = cio.Downloader.from_config(land_spec) assert r is land_downloader
def test_downloading_simple_ascii(): # downloads a file from the Google APIs. (very high uptime and file will # always be there - if this goes down, most of the internet would break!) # to test the downloading mechanisms. file_url = 'http://ajax.googleapis.com/ajax/libs/jquery/1.8.2/{name}.js' format_dict = {'name': 'jquery'} with download_to_temp() as tmp_dir: target_template = os.path.join(tmp_dir, '{name}.txt') tmp_fname = target_template.format(**format_dict) dnld_item = cio.Downloader(file_url, target_template) assert_equal(dnld_item.target_path(format_dict), tmp_fname) with warnings.catch_warnings(record=True) as w: assert_equal(dnld_item.path(format_dict), tmp_fname) assert len(w) == 1, ('Expected a single download warning to be ' 'raised. Got {}.'.format(len(w))) assert issubclass(w[0].category, cio.DownloadWarning) with open(tmp_fname, 'r') as fh: _ = fh.readline() assert_equal(" * jQuery JavaScript Library v1.8.2\n", fh.readline()) # check that calling path again doesn't try re-downloading with CallCounter(dnld_item, 'acquire_resource') as counter: assert_equal(dnld_item.path(format_dict), tmp_fname) assert counter.count == 0, 'Item was re-downloaded.'
def test_downloading_simple_ascii(download_to_temp): # downloads a file from the Google APIs. (very high uptime and file will # always be there - if this goes down, most of the internet would break!) # to test the downloading mechanisms. file_url = 'https://ajax.googleapis.com/ajax/libs/jquery/1.8.2/{name}.js' format_dict = {'name': 'jquery'} target_template = str(download_to_temp / '{name}.txt') tmp_fname = target_template.format(**format_dict) dnld_item = cio.Downloader(file_url, target_template) assert dnld_item.target_path(format_dict) == tmp_fname with pytest.warns(cio.DownloadWarning): assert dnld_item.path(format_dict) == tmp_fname with open(tmp_fname) as fh: fh.readline() assert fh.readline() == " * jQuery JavaScript Library v1.8.2\n" # check that calling path again doesn't try re-downloading with mock.patch.object(dnld_item, 'acquire_resource', wraps=dnld_item.acquire_resource) as counter: assert dnld_item.path(format_dict) == tmp_fname counter.assert_not_called()
def test_Downloader_data(): di = cio.Downloader('https://testing.com/{category}/{name}.zip', os.path.join('{data_dir}', '{category}', 'shape.shp'), '/project/foobar/{category}/sample.shp') replacement_dict = {'category': 'example', 'name': 'test', 'data_dir': os.path.join('/wibble', 'foo', 'bar')} assert di.url(replacement_dict) == 'https://testing.com/example/test.zip' assert (di.target_path(replacement_dict) == os.path.join('/wibble', 'foo', 'bar', 'example', 'shape.shp')) assert (di.pre_downloaded_path(replacement_dict) == '/project/foobar/example/sample.shp')