def _test_get_url_straight_filename(suf): eq_(get_url_straight_filename('http://a.b/' + suf), '') eq_(get_url_straight_filename('http://a.b/p1' + suf), 'p1') eq_(get_url_straight_filename('http://a.b/p1/' + suf), '') eq_(get_url_straight_filename('http://a.b/p1/' + suf, allowdir=True), 'p1') eq_(get_url_straight_filename('http://a.b/p1/p2' + suf), 'p2') eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf), '') eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True), 'p2') eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True, strip=('p2', 'xxx')), 'p1') eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, strip=('p2', 'xxx')), '')
def check_download_external_url(url, failed_str, success_str, d, url_final=None): fpath = opj(d, get_url_straight_filename(url)) providers = get_test_providers(url) # url for check of credentials provider = providers.get_provider(url) downloader = provider.get_downloader(url) # we will load/fetch binary blobs success_bytes, failed_bytes = None, None if success_str is not None: success_bytes = success_str.encode() if failed_str is not None: failed_bytes = failed_str.encode() # Download way with swallow_outputs() as cmo: downloaded_path = downloader.download(url, path=d) assert_equal(fpath, downloaded_path) content = read_file(fpath, decode=False) if success_bytes is not None: assert_in(success_bytes, content) if failed_str is not None: assert_false(failed_bytes in content) # And if we specify size for s in [1, 2]: with swallow_outputs() as cmo: downloaded_path_ = downloader.download(url, path=d, size=s, overwrite=True) # should not be affected assert_equal(downloaded_path, downloaded_path_) content_ = read_file(fpath, decode=False) assert_equal(len(content_), s) assert_equal(content_, content[:s]) # Fetch way content = downloader.fetch(url, decode=False) if success_bytes is not None: assert_in(success_bytes, content) if failed_bytes is not None: assert_false(failed_bytes in content) # And if we specify size for s in [1, 2]: with swallow_outputs() as cmo: content_ = downloader.fetch(url, size=s, decode=False) assert_equal(len(content_), s) assert_equal(content_, content[:s]) # Verify status status = downloader.get_status(url) assert (isinstance(status, FileStatus)) if not url.startswith('ftp://'): # TODO introduce support for mtime into requests_ftp? assert (status.mtime) assert (status.size) # Verify possible redirections if url_final is None: url_final = url assert_equal(downloader.get_target_url(url), url_final)
def get_url_filename(data): yield updated(data, {'filename': get_url_straight_filename(data['url'])})
def get_url_filename(data): yield updated(data, {'filename': get_url_straight_filename(data['url'])})