def _test_get_url_straight_filename(suf):
eq_(get_url_straight_filename('http://a.b/' + suf), '')
eq_(get_url_straight_filename('http://a.b/p1' + suf), 'p1')
eq_(get_url_straight_filename('http://a.b/p1/' + suf), '')
eq_(get_url_straight_filename('http://a.b/p1/' + suf, allowdir=True), 'p1')
eq_(get_url_straight_filename('http://a.b/p1/p2' + suf), 'p2')
eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf), '')
eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True), 'p2')
eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True, strip=('p2', 'xxx')), 'p1')
eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, strip=('p2', 'xxx')), '')
def check_download_external_url(url,
failed_str,
success_str,
d,
url_final=None):
fpath = opj(d, get_url_straight_filename(url))
providers = get_test_providers(url) # url for check of credentials
provider = providers.get_provider(url)
downloader = provider.get_downloader(url)
# we will load/fetch binary blobs
success_bytes, failed_bytes = None, None
if success_str is not None:
success_bytes = success_str.encode()
if failed_str is not None:
failed_bytes = failed_str.encode()
# Download way
with swallow_outputs() as cmo:
downloaded_path = downloader.download(url, path=d)
assert_equal(fpath, downloaded_path)
content = read_file(fpath, decode=False)
if success_bytes is not None:
assert_in(success_bytes, content)
if failed_str is not None:
assert_false(failed_bytes in content)
# And if we specify size
for s in [1, 2]:
with swallow_outputs() as cmo:
downloaded_path_ = downloader.download(url,
path=d,
size=s,
overwrite=True)
# should not be affected
assert_equal(downloaded_path, downloaded_path_)
content_ = read_file(fpath, decode=False)
assert_equal(len(content_), s)
assert_equal(content_, content[:s])
# Fetch way
content = downloader.fetch(url, decode=False)
if success_bytes is not None:
assert_in(success_bytes, content)
if failed_bytes is not None:
assert_false(failed_bytes in content)
# And if we specify size
for s in [1, 2]:
with swallow_outputs() as cmo:
content_ = downloader.fetch(url, size=s, decode=False)
assert_equal(len(content_), s)
assert_equal(content_, content[:s])
# Verify status
status = downloader.get_status(url)
assert (isinstance(status, FileStatus))
if not url.startswith('ftp://'):
# TODO introduce support for mtime into requests_ftp?
assert (status.mtime)
assert (status.size)
# Verify possible redirections
if url_final is None:
url_final = url
assert_equal(downloader.get_target_url(url), url_final)
def get_url_filename(data):
yield updated(data, {'filename': get_url_straight_filename(data['url'])})
def get_url_filename(data):
yield updated(data, {'filename': get_url_straight_filename(data['url'])})