def test_passes_through_other_os_errors(self):
now = datetime.datetime.now(tzlocal())
epoch_now = time.mktime(now.timetuple())
with mock.patch("os.utime") as utime_mock:
utime_mock.side_effect = OSError(2, "")
with self.assertRaises(OSError):
set_file_utime("not_real_file", epoch_now)
def test_successfully_sets_utime(self):
now = datetime.datetime.now(tzlocal())
epoch_now = time.mktime(now.timetuple())
with temporary_file("w") as f:
set_file_utime(f.name, epoch_now)
_, update_time = get_file_stat(f.name)
self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
def test_throws_more_relevant_error_when_errno_1(self):
now = datetime.datetime.now(tzlocal())
epoch_now = time.mktime(now.timetuple())
with mock.patch("os.utime") as utime_mock:
utime_mock.side_effect = OSError(1, "")
with self.assertRaises(SetFileUtimeError):
set_file_utime("not_real_file", epoch_now)
def test_passes_through_other_os_errors(self):
now = datetime.datetime.now(tzlocal())
epoch_now = time.mktime(now.timetuple())
with mock.patch('os.utime') as utime_mock:
utime_mock.side_effect = OSError(2, '')
with self.assertRaises(OSError):
set_file_utime('not_real_file', epoch_now)
def test_throws_more_relevant_error_when_errno_1(self):
now = datetime.datetime.now(tzlocal())
epoch_now = time.mktime(now.timetuple())
with mock.patch('os.utime') as utime_mock:
utime_mock.side_effect = OSError(1, '')
with self.assertRaises(SetFileUtimeError):
set_file_utime('not_real_file', epoch_now)
def test_successfully_sets_utime(self):
now = datetime.datetime.now(tzlocal())
epoch_now = time.mktime(now.timetuple())
with temporary_file('w') as f:
set_file_utime(f.name, epoch_now)
_, update_time = get_file_stat(f.name)
self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
def _handle_io_close_request(self, task):
LOGGER.debug("IOCloseRequest received for %s, closing file.",
task.filename)
fileobj = self.fd_descriptor_cache.get(task.filename)
if fileobj is not None:
fileobj.close()
del self.fd_descriptor_cache[task.filename]
if task.desired_mtime is not None:
set_file_utime(task.filename, task.desired_mtime)
def _handle_io_close_request(self, task):
LOGGER.debug("IOCloseRequest received for %s, closing file.",
task.filename)
fileobj = self.fd_descriptor_cache.get(task.filename)
if fileobj is not None:
fileobj.close()
del self.fd_descriptor_cache[task.filename]
if task.desired_mtime is not None:
set_file_utime(task.filename, task.desired_mtime)
def _on_success(self, future, **kwargs):
filename = future.meta.call_args.fileobj
try:
last_update_tuple = self._last_modified_time.timetuple()
mod_timestamp = time.mktime(last_update_tuple)
utils.set_file_utime(filename, int(mod_timestamp))
except Exception as e:
warning_message = (
'Successfully Downloaded %s but was unable to update the '
'last modified time. %s' % (filename, e))
self._result_queue.put(
utils.create_warning(filename, warning_message))
def save_file(filename, response_data, last_update, is_stream=False):
"""
This writes to the file upon downloading. It reads the data in the
response. Makes a new directory if needed and then writes the
data to the file. It also modifies the last modified time to that
of the S3 object.
"""
body = response_data['Body']
etag = response_data['ETag'][1:-1]
if not is_stream:
d = os.path.dirname(filename)
try:
if not os.path.exists(d):
os.makedirs(d)
except OSError as e:
if not e.errno == errno.EEXIST:
raise CreateDirectoryError(
"Could not create directory %s: %s" % (d, e))
if MD5_AVAILABLE and _can_validate_md5_with_etag(etag, response_data):
md5 = hashlib.md5()
else:
md5 = None
file_chunks = iter(partial(body.read, 1024 * 1024), b'')
if is_stream:
# Need to save the data to be able to check the etag for a stream
# because once the data is written to the stream there is no
# undoing it.
payload = write_to_file(None, etag, file_chunks, md5, True)
else:
with open(filename, 'wb') as out_file:
write_to_file(out_file, etag, file_chunks, md5)
if md5 is not None and etag != md5.hexdigest():
if not is_stream:
os.remove(filename)
raise MD5Error(filename)
if not is_stream:
last_update_tuple = last_update.timetuple()
mod_timestamp = time.mktime(last_update_tuple)
set_file_utime(filename, int(mod_timestamp))
else:
# Now write the output to stdout since the md5 is correct.
bytes_print(payload)
sys.stdout.flush()
def save_file(filename, response_data, last_update, is_stream=False):
"""
This writes to the file upon downloading. It reads the data in the
response. Makes a new directory if needed and then writes the
data to the file. It also modifies the last modified time to that
of the S3 object.
"""
body = response_data['Body']
etag = response_data['ETag'][1:-1]
if not is_stream:
d = os.path.dirname(filename)
try:
if not os.path.exists(d):
os.makedirs(d)
except OSError as e:
if not e.errno == errno.EEXIST:
raise CreateDirectoryError(
"Could not create directory %s: %s" % (d, e))
if MD5_AVAILABLE and _can_validate_md5_with_etag(etag, response_data):
md5 = hashlib.md5()
else:
md5 = None
file_chunks = iter(partial(body.read, 1024 * 1024), b'')
if is_stream:
# Need to save the data to be able to check the etag for a stream
# because once the data is written to the stream there is no
# undoing it.
payload = write_to_file(None, etag, file_chunks, md5, True)
else:
with open(filename, 'wb') as out_file:
write_to_file(out_file, etag, file_chunks, md5)
if md5 is not None and etag != md5.hexdigest():
if not is_stream:
os.remove(filename)
raise MD5Error(filename)
if not is_stream:
last_update_tuple = last_update.timetuple()
mod_timestamp = time.mktime(last_update_tuple)
set_file_utime(filename, int(mod_timestamp))
else:
# Now write the output to stdout since the md5 is correct.
bytes_print(payload)
sys.stdout.flush()
