def mk_filename(pattern, response, sock, url, overwrite=False): name = get_filename(sock, url) if not name: if not isinstance(response.fileid, unicode): name = unicode(response.fileid, "ascii", "ignore") else: name = response.fileid fs_encoding = sys.getfilesystemencoding() if fs_encoding is None: fs_encoding = "ascii" name = slugify(name).encode(fs_encoding, "ignore") if not name: name = "file" fname = pattern.format(file=name, **dict(response)) if not overwrite and os.path.exists(fname): fname = replacement_filename(fname) dir_ = os.path.abspath(os.path.dirname(fname)) if not os.path.exists(dir_): os.makedirs(dir_) return fname
def mk_filename(pattern, response, sock, url, overwrite=False): name = get_filename(sock, url) if not name: if isinstance(response.fileid, bytes): name = response.fileid.decode("ascii", "ignore") else: name = response.fileid fs_encoding = sys.getfilesystemencoding() if fs_encoding is None: fs_encoding = "ascii" name = slugify(name) if not name: name = "file" fname = pattern.format(file=name, **dict(response)) if not overwrite and os.path.exists(fname): fname = replacement_filename(fname) dir_ = os.path.abspath(os.path.dirname(fname)) if not os.path.exists(dir_): os.makedirs(dir_) return fname
def download(url): path = self._cache_dir / get_filename(urlopen(url), url) # replacement_filename returns a string and we want a Path object path = Path(replacement_filename(str(path))) self._downloader.download(url, path) shahash = hash_file(path) return path, shahash, url
def mk_filename(pattern, response, sock, url, overwrite=False): name = get_filename(sock, url) if not name: if not isinstance(response.fileid, text_type): name = six.u(response.fileid, "ascii", "ignore") else: name = response.fileid fs_encoding = sys.getfilesystemencoding() if fs_encoding is None: fs_encoding = "ascii"
def download(url): path = self._cache_dir / (namespace + get_filename(urlopen(url), url)) self._downloader.download(url, path) shahash = hash_file(path) return path, shahash, url
def download(url): path = self._cache_dir / get_filename(urlopen(url), url) path = replacement_filename(path) self._downloader.download(url, path) shahash = hash_file(path) return path, shahash, url