def extract_zipped_index_file(src_file):
""" Extract a index file from a zip-archive if zipped or pass the index
file unchanged if not zipped.
"""
dst_file, ext = splitext(src_file)
if ext == ".zip":
logger.debug("Extracting zip-file %s ...", src_file)
# NOTE: Avoid possible filesystem-to-filesystem copy.
# ans store the file directly in the destination folder.
tmp_file = dst_file + ".tmp"
timer = Timer()
try:
with closing(zipfile.ZipFile(src_file)) as archive:
fin = archive.open(basename(dst_file))
with open(tmp_file, "wb") as fout:
shutil.copyfileobj(fin, fout, CHUNK_SIZE)
size = fout.tell()
os.rename(tmp_file, dst_file)
except Exception as exc:
logger.error("Extraction of %s failed: %s", src_file, exc)
raise
else:
os.remove(src_file)
finally:
if isfile(tmp_file):
os.remove(tmp_file)
logger.info(
"'%s' -> '%s' %dB %.3fs", src_file, dst_file, size, timer.stop()
)
return dst_file
else:
return src_file
def retrieve(self, url, file_name, target_dir):
path = join(target_dir, file_name)
tmp_path = path + ".tmp"
logger.debug("Retrieving %s and storing it under %s", url, path)
timer = Timer()
try:
with closing(urllib2.urlopen(url, timeout=self.timeout)) as fin:
with open(tmp_path, "wb") as fout:
shutil.copyfileobj(fin, fout, 64*1024)
size = fout.tell()
os.rename(tmp_path, path)
except IOError as exc:
logger.error("Error saving %s: %s", path, exc)
raise RetrieveError(str(exc))
except Exception as exc:
logger.error("Error retrieving %s: %s", url, exc)
raise
finally:
if isfile(tmp_path):
os.remove(tmp_path)
logger.info("'%s' -> '%s' %dB %.3fs", url, path, size, timer.stop())
return file_name
