def save_file(filename, response_data, last_update):
"""
This writes to the file upon downloading. It reads the data in the
response. Makes a new directory if needed and then writes the
data to the file. It also modifies the last modified time to that
of the S3 object.
"""
body = response_data['Body']
etag = response_data['ETag'][1:-1]
d = os.path.dirname(filename)
try:
if not os.path.exists(d):
os.makedirs(d)
except OSError as e:
if not e.errno == errno.EEXIST:
raise CreateDirectoryError("Could not create directory %s: %s" %
(d, e))
md5 = hashlib.md5()
file_chunks = iter(partial(body.read, 1024 * 1024), b'')
with open(filename, 'wb') as out_file:
if not _is_multipart_etag(etag):
for chunk in file_chunks:
md5.update(chunk)
out_file.write(chunk)
else:
for chunk in file_chunks:
out_file.write(chunk)
if not _is_multipart_etag(etag):
if etag != md5.hexdigest():
os.remove(filename)
raise MD5Error(filename)
last_update_tuple = last_update.timetuple()
mod_timestamp = time.mktime(last_update_tuple)
os.utime(filename, (int(mod_timestamp), int(mod_timestamp)))
def save_file(filename, response_data, last_update, is_stream=False):
"""
This writes to the file upon downloading. It reads the data in the
response. Makes a new directory if needed and then writes the
data to the file. It also modifies the last modified time to that
of the S3 object.
"""
body = response_data['Body']
etag = response_data['ETag'][1:-1]
if not is_stream:
d = os.path.dirname(filename)
try:
if not os.path.exists(d):
os.makedirs(d)
except OSError as e:
if not e.errno == errno.EEXIST:
raise CreateDirectoryError(
"Could not create directory %s: %s" % (d, e))
if MD5_AVAILABLE and _can_validate_md5_with_etag(etag, response_data):
md5 = hashlib.md5()
else:
md5 = None
file_chunks = iter(partial(body.read, 1024 * 1024), b'')
if is_stream:
# Need to save the data to be able to check the etag for a stream
# because once the data is written to the stream there is no
# undoing it.
payload = write_to_file(None, etag, file_chunks, md5, True)
else:
with open(filename, 'wb') as out_file:
write_to_file(out_file, etag, file_chunks, md5)
if md5 is not None and etag != md5.hexdigest():
if not is_stream:
os.remove(filename)
raise MD5Error(filename)
if not is_stream:
last_update_tuple = last_update.timetuple()
mod_timestamp = time.mktime(last_update_tuple)
set_file_utime(filename, int(mod_timestamp))
else:
# Now write the output to stdout since the md5 is correct.
bytes_print(payload)
sys.stdout.flush()