def extract_zipped_index_file(src_file): """ Extract a index file from a zip-archive if zipped or pass the index file unchanged if not zipped. """ dst_file, ext = splitext(src_file) if ext == ".zip": logger.debug("Extracting zip-file %s ...", src_file) # NOTE: Avoid possible filesystem-to-filesystem copy. # ans store the file directly in the destination folder. tmp_file = dst_file + ".tmp" timer = Timer() try: with closing(zipfile.ZipFile(src_file)) as archive: fin = archive.open(basename(dst_file)) with open(tmp_file, "wb") as fout: shutil.copyfileobj(fin, fout, CHUNK_SIZE) size = fout.tell() os.rename(tmp_file, dst_file) except Exception as exc: logger.error("Extraction of %s failed: %s", src_file, exc) raise else: os.remove(src_file) finally: if isfile(tmp_file): os.remove(tmp_file) logger.info( "'%s' -> '%s' %dB %.3fs", src_file, dst_file, size, timer.stop() ) return dst_file else: return src_file
def retrieve(self, url, file_name, target_dir): path = join(target_dir, file_name) tmp_path = path + ".tmp" logger.debug("Retrieving %s and storing it under %s", url, path) timer = Timer() try: with closing(urllib2.urlopen(url, timeout=self.timeout)) as fin: with open(tmp_path, "wb") as fout: shutil.copyfileobj(fin, fout, 64*1024) size = fout.tell() os.rename(tmp_path, path) except IOError as exc: logger.error("Error saving %s: %s", path, exc) raise RetrieveError(str(exc)) except Exception as exc: logger.error("Error retrieving %s: %s", url, exc) raise finally: if isfile(tmp_path): os.remove(tmp_path) logger.info("'%s' -> '%s' %dB %.3fs", url, path, size, timer.stop()) return file_name