def mk_filename(pattern, response, sock, url, overwrite=False):
name = get_filename(sock, url)
if not name:
if not isinstance(response.fileid, unicode):
name = unicode(response.fileid, "ascii", "ignore")
else:
name = response.fileid
fs_encoding = sys.getfilesystemencoding()
if fs_encoding is None:
fs_encoding = "ascii"
name = name.encode(fs_encoding, "ignore")
if not name:
name = "file"
fname = pattern.format(file=name, **dict(response))
if not overwrite and os.path.exists(fname):
fname = replacement_filename(fname)
dir_ = os.path.dirname(fname)
if not os.path.exists(dir_):
os.makedirs(dir_)
return fname
def download(url):
path = self._cache_dir / get_filename(urlopen(url), url)
# replacement_filename returns a string and we want a Path object
path = Path(replacement_filename(str(path)))
self._downloader.download(url, path)
shahash = hash_file(path)
return path, shahash, url
def mk_filename(pattern, response, sock, url, overwrite=False):
name = get_filename(sock, url)
if not name:
if not isinstance(response.fileid, unicode):
name = unicode(response.fileid, "ascii", "ignore")
else:
name = response.fileid
fs_encoding = sys.getfilesystemencoding()
if fs_encoding is None:
fs_encoding = "ascii"
name = name.encode(fs_encoding, "ignore")
if not name:
name = "file"
fname = pattern.format(file=name, **dict(response))
if not overwrite and os.path.exists(fname):
fname = replacement_filename(fname)
dir_ = os.path.dirname(fname)
if not os.path.exists(dir_):
os.makedirs(dir_)
return fname
def test_replacement_filename_path_not_exists(mocker):
"""
If a candidate path does not exist, then just return it as it is OK to use
"""
path_not_exists = '/tmp'
mocker.patch('os.path.exists', return_value=False)
assert util.replacement_filename(path_not_exists) == path_not_exists
def test_replacement_filename_path_not_exists(mocker):
"""
If a candidate path does not exist, then just return it as it is OK to use
"""
path_not_exists = '/tmp'
mocker.patch('os.path.exists', return_value=False)
assert util.replacement_filename(path_not_exists) == path_not_exists
def download_fileobj(opn, directory, url='', default=u"file", overwrite=False):
""" Download file from url into directory. Try to get filename from
Content-Disposition header, otherwise get from path of url if given.
Fall back to default if both fail. Only overwrite existing files when
overwrite is True. """
filename = get_system_filename(opn, url, default)
path = os.path.join(directory, filename)
if not overwrite and os.path.exists(path):
path = replacement_filename(path)
with open(path, 'wb') as fd:
shutil.copyfileobj(opn, fd)
return path
def download_fileobj(opn, directory, url='', default=u"file", overwrite=False):
""" Download file from url into directory. Try to get filename from
Content-Disposition header, otherwise get from path of url if given.
Fall back to default if both fail. Only overwrite existing files when
overwrite is True. """
filename = get_system_filename(opn, url, default)
path = os.path.join(directory, filename)
if not overwrite and os.path.exists(path):
path = replacement_filename(path)
with open(path, 'wb') as fd:
shutil.copyfileobj(opn, fd)
return path
def test_replacement_filename():
"""
This should return a replacement path for the current file.
"""
assert util.replacement_filename(__file__).endswith('test_util.0.py')
def test_replacement_filename():
"""
This should return a replacement path for the current file.
"""
assert util.replacement_filename(__file__).endswith('test_util.0.py')
def download(url):
path = self._cache_dir / get_filename(urlopen(url), url)
path = replacement_filename(path)
self._downloader.download(url, path)
shahash = hash_file(path)
return path, shahash, url
