def exists(self, modified_since=None, unmodified_since=None): """ return True if we can HEAD the key, and it fits one of the optional date_modified restrctions. Not that you cannot specify both modified_since and unmodified_since """ found = False if self._bucket is None: raise ValueError("No bucket") if self._name is None: raise ValueError("No name") if modified_since is not None and unmodified_since is not None: raise ValueError( "Can't specify both modified_since and unmodified_since") method = "HEAD" uri = compute_uri("data", self._name) headers = {} if modified_since is not None: timestamp = datetime.utcfromtimestamp(modified_since) headers["If-Modified-Since"] = http_timestamp_str(timestamp) if unmodified_since is not None: timestamp = datetime.utcfromtimestamp(unmodified_since) headers["If-Unmodified-Since"] = http_timestamp_str(timestamp) http_connection = self._bucket.create_http_connection() self._log.info("requesting HEAD {0} {1}".format(uri, headers)) try: response = http_connection.request(method, uri, body=None, headers=headers) except LumberyardHTTPError: instance = sys.exc_info()[1] # not modified, not found, precondition not met if instance.status in [304, 404, 412]: pass else: self._log.error(str(instance)) http_connection.close() raise else: found = True if found: response.read() http_connection.close() return found
def exists(self, modified_since=None, unmodified_since=None): """ return True if we can HEAD the key, and it fits one of the optional date_modified restrctions. Not that you cannot specify both modified_since and unmodified_since """ found = False if self._bucket is None: raise ValueError("No bucket") if self._name is None: raise ValueError("No name") if modified_since is not None and unmodified_since is not None: raise ValueError("Can't specify both modified_since and unmodified_since") method = "HEAD" uri = compute_uri("data", self._name) headers = {} if modified_since is not None: timestamp = datetime.utcfromtimestamp(modified_since) headers["If-Modified-Since"] = http_timestamp_str(timestamp) if unmodified_since is not None: timestamp = datetime.utcfromtimestamp(unmodified_since) headers["If-Unmodified-Since"] = http_timestamp_str(timestamp) http_connection = self._bucket.create_http_connection() self._log.info("requesting HEAD {0} {1}".format(uri, headers)) try: response = http_connection.request(method, uri, body=None, headers=headers) except LumberyardHTTPError: instance = sys.exc_info()[1] # not modified, not found, precondition not met if instance.status in [304, 404, 412]: pass else: self._log.error(str(instance)) http_connection.close() raise else: found = True if found: response.read() http_connection.close() return found
def get_contents_to_file(self, file_object, cb=None, cb_count=10, version_id=None, slice_offset=None, slice_size=None, modified_since=None, unmodified_since=None, resumable=False, res_download_handler=None): """ file_object Python file-like object, must support write() must support seek() and tell() for resumable=True cb callback function for reporting progress cb_count number of callbacks to be made during the archvie process version_id identifier of a specific version to retrieve None means retrieve the most recent version slice_offset byte offset for start of retrieve None means start at byte 0 slice_size number of bytes to retrieve None means retrieve to end of file modified_since only retrieve the file if it has been modified since the specified timestamp. Otherwise: raise KeyUnmodified Note: you cannot specify both modified_since and unmodified_since unmodified_since only retrieve the file if it has not been modified since the specified timestamp. Otherwise: raise KeyModified Note: you cannot specify both modified_since and unmodified_since resumable True means append to an existing file if there is one res_download_handler included for boto compatibility. We have a ResumeableDownloadHandler object, but actually if you put anything besides None in this argument, it has the same effect as setting resumable to True. retrieve the contents from nimbus.io to a file """ if self._bucket is None: raise ValueError("No bucket") if self._name is None: raise ValueError("No name") if modified_since is not None and unmodified_since is not None: raise ValueError( "Can't specify both modified_since and unmodified_since") kwargs = { "version_identifier": version_id, } if resumable == True or res_download_handler is not None: file_object.seek(0, os.SEEK_END) current_file_size = file_object.tell() if slice_size is not None: assert current_file_size < slice_size slice_size -= current_file_size if slice_offset is not None: slice_offset += current_file_size else: slice_offset = current_file_size headers = {} _convert_slice_to_range_header(headers, slice_offset, slice_size) expected_status = (PARTIAL_CONTENT if "Range" in headers else OK) if modified_since is not None: timestamp = datetime.utcfromtimestamp(modified_since) headers["If-Modified-Since"] = http_timestamp_str(timestamp) if unmodified_since is not None: timestamp = datetime.utcfromtimestamp(unmodified_since) headers["If-Unmodified-Since"] = http_timestamp_str(timestamp) method = "GET" uri = compute_uri("data", self._name, **kwargs) http_connection = self._bucket.create_http_connection() self._log.info("requesting GET {0} {1}".format(uri, headers)) try: response = http_connection.request(method, uri, body=None, headers=headers, expected_status=expected_status) except LumberyardHTTPError: instance = sys.exc_info()[1] http_connection.close() if instance.status == NOT_MODIFIED and modified_since is not None: raise KeyUnmodified() if instance.status == PRECONDITION_FAILED and \ unmodified_since is not None: raise KeyModified() raise if cb is None: reporter = NullCallbackWrapper() else: reporter = RetrieveCallbackWrapper(self.size, cb, cb_count) self._log.info("reading response") reporter.start() while True: data = response.read(_read_buffer_size) bytes_read = len(data) self._log.debug("read {0} bytes".format(bytes_read)) if bytes_read == 0: break file_object.write(data) reporter.bytes_written(bytes_read) reporter.finish() http_connection.close()
def get_contents_as_string(self, cb=None, cb_count=10, version_id=None, slice_offset=None, slice_size=None, modified_since=None, unmodified_since=None): """ cb callback function for reporting progress cb_count number of callbacks to be made during the archvie process version_id the identifier of a specific version to retrieve None means retrieve the most recent version slice_offset byte offset for start of retrieve None means start at byte 0 slice_size number of bytes to retrieve None means retrieve to end of file modified_since only retrieve the file if it has been modified since the specified timestamp. Otherwise: raise KeyUnmodified Note: you cannot specify both modified_since and unmodified_since unmodified_since only retrieve the file if it has not been modified since the specified timestamp. Otherwise: raise KeyModified Note: you cannot specify both modified_since and unmodified_since retrieve the contents from nimbus.io as a string """ if self._bucket is None: raise ValueError("No bucket") if self._name is None: raise ValueError("No name") if modified_since is not None and unmodified_since is not None: raise ValueError( "Can't specify both modified_since and unmodified_since") kwargs = { "version_identifier": version_id, } headers = {} _convert_slice_to_range_header(headers, slice_offset, slice_size) expected_status = (PARTIAL_CONTENT if "Range" in headers else OK) if modified_since is not None: timestamp = datetime.utcfromtimestamp(modified_since) headers["If-Modified-Since"] = http_timestamp_str(timestamp) if unmodified_since is not None: timestamp = datetime.utcfromtimestamp(unmodified_since) headers["If-Unmodified-Since"] = http_timestamp_str(timestamp) method = "GET" uri = compute_uri("data", self._name, **kwargs) http_connection = self._bucket.create_http_connection() self._log.info("requesting GET {0} {1}".format(uri, headers)) try: response = http_connection.request(method, uri, body=None, headers=headers, expected_status=expected_status) except LumberyardHTTPError: instance = sys.exc_info()[1] http_connection.close() if instance.status == NOT_MODIFIED and modified_since is not None: raise KeyUnmodified() if instance.status == PRECONDITION_FAILED and \ unmodified_since is not None: raise KeyModified() raise body_list = list() while True: data = response.read(_read_buffer_size) if len(data) == 0: break body_list.append(data) http_connection.close() return b"".join(body_list)
def get_contents_to_file( self, file_object, cb=None, cb_count=10, version_id=None, slice_offset=None, slice_size=None, modified_since=None, unmodified_since=None, resumable=False, res_download_handler=None, ): """ file_object Python file-like object, must support write() must support seek() and tell() for resumable=True cb callback function for reporting progress cb_count number of callbacks to be made during the archvie process version_id identifier of a specific version to retrieve None means retrieve the most recent version slice_offset byte offset for start of retrieve None means start at byte 0 slice_size number of bytes to retrieve None means retrieve to end of file modified_since only retrieve the file if it has been modified since the specified timestamp. Otherwise: raise KeyUnmodified Note: you cannot specify both modified_since and unmodified_since unmodified_since only retrieve the file if it has not been modified since the specified timestamp. Otherwise: raise KeyModified Note: you cannot specify both modified_since and unmodified_since resumable True means append to an existing file if there is one res_download_handler included for boto compatibility. We have a ResumeableDownloadHandler object, but actually if you put anything besides None in this argument, it has the same effect as setting resumable to True. retrieve the contents from nimbus.io to a file """ if self._bucket is None: raise ValueError("No bucket") if self._name is None: raise ValueError("No name") if modified_since is not None and unmodified_since is not None: raise ValueError("Can't specify both modified_since and unmodified_since") kwargs = {"version_identifier": version_id} if resumable == True or res_download_handler is not None: file_object.seek(0, os.SEEK_END) current_file_size = file_object.tell() if slice_size is not None: assert current_file_size < slice_size slice_size -= current_file_size if slice_offset is not None: slice_offset += current_file_size else: slice_offset = current_file_size headers = {} _convert_slice_to_range_header(headers, slice_offset, slice_size) expected_status = PARTIAL_CONTENT if "Range" in headers else OK if modified_since is not None: timestamp = datetime.utcfromtimestamp(modified_since) headers["If-Modified-Since"] = http_timestamp_str(timestamp) if unmodified_since is not None: timestamp = datetime.utcfromtimestamp(unmodified_since) headers["If-Unmodified-Since"] = http_timestamp_str(timestamp) method = "GET" uri = compute_uri("data", self._name, **kwargs) http_connection = self._bucket.create_http_connection() self._log.info("requesting GET {0} {1}".format(uri, headers)) try: response = http_connection.request(method, uri, body=None, headers=headers, expected_status=expected_status) except LumberyardHTTPError: instance = sys.exc_info()[1] http_connection.close() if instance.status == NOT_MODIFIED and modified_since is not None: raise KeyUnmodified() if instance.status == PRECONDITION_FAILED and unmodified_since is not None: raise KeyModified() raise if cb is None: reporter = NullCallbackWrapper() else: reporter = RetrieveCallbackWrapper(self.size, cb, cb_count) self._log.info("reading response") reporter.start() while True: data = response.read(_read_buffer_size) bytes_read = len(data) self._log.debug("read {0} bytes".format(bytes_read)) if bytes_read == 0: break file_object.write(data) reporter.bytes_written(bytes_read) reporter.finish() http_connection.close()
def get_contents_as_string( self, cb=None, cb_count=10, version_id=None, slice_offset=None, slice_size=None, modified_since=None, unmodified_since=None, ): """ cb callback function for reporting progress cb_count number of callbacks to be made during the archvie process version_id the identifier of a specific version to retrieve None means retrieve the most recent version slice_offset byte offset for start of retrieve None means start at byte 0 slice_size number of bytes to retrieve None means retrieve to end of file modified_since only retrieve the file if it has been modified since the specified timestamp. Otherwise: raise KeyUnmodified Note: you cannot specify both modified_since and unmodified_since unmodified_since only retrieve the file if it has not been modified since the specified timestamp. Otherwise: raise KeyModified Note: you cannot specify both modified_since and unmodified_since retrieve the contents from nimbus.io as a string """ if self._bucket is None: raise ValueError("No bucket") if self._name is None: raise ValueError("No name") if modified_since is not None and unmodified_since is not None: raise ValueError("Can't specify both modified_since and unmodified_since") kwargs = {"version_identifier": version_id} headers = {} _convert_slice_to_range_header(headers, slice_offset, slice_size) expected_status = PARTIAL_CONTENT if "Range" in headers else OK if modified_since is not None: timestamp = datetime.utcfromtimestamp(modified_since) headers["If-Modified-Since"] = http_timestamp_str(timestamp) if unmodified_since is not None: timestamp = datetime.utcfromtimestamp(unmodified_since) headers["If-Unmodified-Since"] = http_timestamp_str(timestamp) method = "GET" uri = compute_uri("data", self._name, **kwargs) http_connection = self._bucket.create_http_connection() self._log.info("requesting GET {0} {1}".format(uri, headers)) try: response = http_connection.request(method, uri, body=None, headers=headers, expected_status=expected_status) except LumberyardHTTPError: instance = sys.exc_info()[1] http_connection.close() if instance.status == NOT_MODIFIED and modified_since is not None: raise KeyUnmodified() if instance.status == PRECONDITION_FAILED and unmodified_since is not None: raise KeyModified() raise body_list = list() while True: data = response.read(_read_buffer_size) if len(data) == 0: break body_list.append(data) http_connection.close() return b"".join(body_list)