def download_blob(blob, filename):
"""A helper function to download a storage Blob to a blob_file (the filename)
and validate it using the Crc32cCalculator.
Arguments:
blob (storage.Blob) : the Google storage blob object
blob_file (str) : the file path to download to
Returns: boolean to indicate doing retry (True) or not (False)
"""
# create parent directories if necessary
os.makedirs(os.path.dirname(filename), exist_ok=True)
# ideally we could calculate hash while streaming to file with provided function
# https://github.com/googleapis/python-storage/issues/29
with open(filename, "wb") as blob_file:
parser = Crc32cCalculator(blob_file)
blob.download_to_file(parser)
os.sync()
# **Important** hash can be incorrect or missing if not refreshed
blob.reload()
# Compute local hash and verify correct
if parser.hexdigest() != blob.crc32c:
os.remove(filename)
raise CheckSumMismatchException("The checksum of %s does not match." %
filename)
return filename
def download(self):
"""Download with maximum retry duration of 600 seconds (10 minutes)"""
if not self.exists():
return None
# Create the directory for the intended file
os.makedirs(os.path.dirname(self.local_file()), exist_ok=True)
# ideally we could calculate hash while streaming to file with provided function
# https://github.com/googleapis/python-storage/issues/29
with open(self.local_file(), "wb") as blob_file:
parser = Crc32cCalculator(blob_file)
self.blob.download_to_file(parser)
os.sync()
# **Important** hash can be incorrect or missing if not refreshed
self.blob.reload()
# Compute local hash and verify correct
if parser.hexdigest() != self.blob.crc32c:
os.remove(self.local_file())
raise CheckSumMismatchException(
"The checksum of %s does not match." % self.local_file()
)
return self.local_file()