def _fetch_from_cache(self): """Attempt to fetch a file from cache or return None. Returns: *iter of open, decompressed file pointer from cache or None. """ if CACHE_DIR is None: raise Exception, "Set environ var CACHE_DIR to cache directory." filepath = os.path.join(CACHE_DIR, self.cache_name) if os.path.exists(filepath): return gzip.open(filepath, "rb") else: return None
def __init__(self, fp, size=None, cache=None, report=True, finalize=True, ftp=False): """Initialize self. Args: fp: open file pointer like object like HTTP connection size: int of expected bytes in download or None if unknown cache: str of cache file name or None if cache is disabled report: bool to report download status finalize: bool if close() is called before self.completed, finish downloading before closing buffer and any cache """ self.buffer = fp self.cache = cache self.size = size if self.size: # Convert to float to avoid integer division complications. self.size = float(self.size) self.report = report self.finalize = finalize self.bytes_read = 0 self.bytes_reported = 0 self.fp_out = None self.completed = False self.tmp_filepath = None self.dest_filepath = None self.closed = False if cache: # Finalized (completely downloaded) cache filename global CACHE_DIR if CACHE_DIR is None: # try to get cache directory from settings in runtime if "CACHE_DIR" in os.environ: CACHE_DIR = os.environ["CACHE_DIR"] if CACHE_DIR is None: raise Exception, "Set environ variable CACHE_DIR to a system path that this program can use as a cache directory. For example: `export CACHE_DIR=/home/z/Desktop` or in Python, `os.environ['CACHE_DIR'] = '/my/path'`" self.dest_filepath = os.path.join(CACHE_DIR, cache) # Add ".tmp" to end of cache filename to indicate cache is incomplete. self.tmp_filepath = self.dest_filepath + ".tmp" # Cache files are compressed (even if the underlying data is compressed) self.fp_out = gzip.open(self.tmp_filepath, "wb")