def test_passes_through_other_os_errors(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with mock.patch("os.utime") as utime_mock: utime_mock.side_effect = OSError(2, "") with self.assertRaises(OSError): set_file_utime("not_real_file", epoch_now)
def test_successfully_sets_utime(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with temporary_file("w") as f: set_file_utime(f.name, epoch_now) _, update_time = get_file_stat(f.name) self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
def test_throws_more_relevant_error_when_errno_1(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with mock.patch("os.utime") as utime_mock: utime_mock.side_effect = OSError(1, "") with self.assertRaises(SetFileUtimeError): set_file_utime("not_real_file", epoch_now)
def test_passes_through_other_os_errors(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with mock.patch('os.utime') as utime_mock: utime_mock.side_effect = OSError(2, '') with self.assertRaises(OSError): set_file_utime('not_real_file', epoch_now)
def test_throws_more_relevant_error_when_errno_1(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with mock.patch('os.utime') as utime_mock: utime_mock.side_effect = OSError(1, '') with self.assertRaises(SetFileUtimeError): set_file_utime('not_real_file', epoch_now)
def test_successfully_sets_utime(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with temporary_file('w') as f: set_file_utime(f.name, epoch_now) _, update_time = get_file_stat(f.name) self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
def _handle_io_close_request(self, task): LOGGER.debug("IOCloseRequest received for %s, closing file.", task.filename) fileobj = self.fd_descriptor_cache.get(task.filename) if fileobj is not None: fileobj.close() del self.fd_descriptor_cache[task.filename] if task.desired_mtime is not None: set_file_utime(task.filename, task.desired_mtime)
def _on_success(self, future, **kwargs): filename = future.meta.call_args.fileobj try: last_update_tuple = self._last_modified_time.timetuple() mod_timestamp = time.mktime(last_update_tuple) utils.set_file_utime(filename, int(mod_timestamp)) except Exception as e: warning_message = ( 'Successfully Downloaded %s but was unable to update the ' 'last modified time. %s' % (filename, e)) self._result_queue.put( utils.create_warning(filename, warning_message))
def save_file(filename, response_data, last_update, is_stream=False): """ This writes to the file upon downloading. It reads the data in the response. Makes a new directory if needed and then writes the data to the file. It also modifies the last modified time to that of the S3 object. """ body = response_data['Body'] etag = response_data['ETag'][1:-1] if not is_stream: d = os.path.dirname(filename) try: if not os.path.exists(d): os.makedirs(d) except OSError as e: if not e.errno == errno.EEXIST: raise CreateDirectoryError( "Could not create directory %s: %s" % (d, e)) if MD5_AVAILABLE and _can_validate_md5_with_etag(etag, response_data): md5 = hashlib.md5() else: md5 = None file_chunks = iter(partial(body.read, 1024 * 1024), b'') if is_stream: # Need to save the data to be able to check the etag for a stream # because once the data is written to the stream there is no # undoing it. payload = write_to_file(None, etag, file_chunks, md5, True) else: with open(filename, 'wb') as out_file: write_to_file(out_file, etag, file_chunks, md5) if md5 is not None and etag != md5.hexdigest(): if not is_stream: os.remove(filename) raise MD5Error(filename) if not is_stream: last_update_tuple = last_update.timetuple() mod_timestamp = time.mktime(last_update_tuple) set_file_utime(filename, int(mod_timestamp)) else: # Now write the output to stdout since the md5 is correct. bytes_print(payload) sys.stdout.flush()