Example #1
0
    def exists(self, modified_since=None, unmodified_since=None):
        """
        return True if we can HEAD the key, and it fits one of the
        optional date_modified restrctions.

        Not that you cannot specify both modified_since and unmodified_since
        """
        found = False

        if self._bucket is None:
            raise ValueError("No bucket")
        if self._name is None:
            raise ValueError("No name")
        if modified_since is not None and unmodified_since is not None:
            raise ValueError(
                "Can't specify both modified_since and unmodified_since")

        method = "HEAD"
        uri = compute_uri("data", self._name)
        headers = {}
        if modified_since is not None:
            timestamp = datetime.utcfromtimestamp(modified_since)
            headers["If-Modified-Since"] = http_timestamp_str(timestamp)
        if unmodified_since is not None:
            timestamp = datetime.utcfromtimestamp(unmodified_since)
            headers["If-Unmodified-Since"] = http_timestamp_str(timestamp)

        http_connection = self._bucket.create_http_connection()

        self._log.info("requesting HEAD {0} {1}".format(uri, headers))
        try:
            response = http_connection.request(method,
                                               uri,
                                               body=None,
                                               headers=headers)
        except LumberyardHTTPError:
            instance = sys.exc_info()[1]
            # not modified, not found, precondition not met
            if instance.status in [304, 404, 412]:
                pass
            else:
                self._log.error(str(instance))
                http_connection.close()
                raise
        else:
            found = True

        if found:
            response.read()

        http_connection.close()

        return found
Example #2
0
    def exists(self, modified_since=None, unmodified_since=None):
        """
        return True if we can HEAD the key, and it fits one of the
        optional date_modified restrctions.

        Not that you cannot specify both modified_since and unmodified_since
        """
        found = False

        if self._bucket is None:
            raise ValueError("No bucket")
        if self._name is None:
            raise ValueError("No name")
        if modified_since is not None and unmodified_since is not None:
            raise ValueError("Can't specify both modified_since and unmodified_since")

        method = "HEAD"
        uri = compute_uri("data", self._name)
        headers = {}
        if modified_since is not None:
            timestamp = datetime.utcfromtimestamp(modified_since)
            headers["If-Modified-Since"] = http_timestamp_str(timestamp)
        if unmodified_since is not None:
            timestamp = datetime.utcfromtimestamp(unmodified_since)
            headers["If-Unmodified-Since"] = http_timestamp_str(timestamp)

        http_connection = self._bucket.create_http_connection()

        self._log.info("requesting HEAD {0} {1}".format(uri, headers))
        try:
            response = http_connection.request(method, uri, body=None, headers=headers)
        except LumberyardHTTPError:
            instance = sys.exc_info()[1]
            # not modified, not found, precondition not met
            if instance.status in [304, 404, 412]:
                pass
            else:
                self._log.error(str(instance))
                http_connection.close()
                raise
        else:
            found = True

        if found:
            response.read()

        http_connection.close()

        return found
Example #3
0
    def get_contents_to_file(self,
                             file_object,
                             cb=None,
                             cb_count=10,
                             version_id=None,
                             slice_offset=None,
                             slice_size=None,
                             modified_since=None,
                             unmodified_since=None,
                             resumable=False,
                             res_download_handler=None):
        """
        file_object
            Python file-like object, must support write()
            must support seek() and tell() for resumable=True

        cb
            callback function for reporting progress

        cb_count
            number of callbacks to be made during the archvie process

        version_id
            identifier of a specific version to retrieve

            None means retrieve the most recent version

        slice_offset
            byte offset for start of retrieve

            None means start at byte 0

        slice_size
            number of bytes to retrieve

            None means retrieve to end of file

        modified_since
            only retrieve the file if it has been modified since the specified 
            timestamp.

            Otherwise: raise KeyUnmodified

            Note: you cannot specify both modified_since and unmodified_since

        unmodified_since
            only retrieve the file if it has not been modified since the 
            specified timestamp.

            Otherwise: raise KeyModified

            Note: you cannot specify both modified_since and unmodified_since

        resumable 
            True means append to an existing file if there is one

        res_download_handler
            included for boto compatibility. We have a 
            ResumeableDownloadHandler object, but actually if you put
            anything besides None in this argument, it has the same effect
            as setting resumable to True.

        retrieve the contents from nimbus.io to a file
        """
        if self._bucket is None:
            raise ValueError("No bucket")
        if self._name is None:
            raise ValueError("No name")
        if modified_since is not None and unmodified_since is not None:
            raise ValueError(
                "Can't specify both modified_since and unmodified_since")

        kwargs = {
            "version_identifier": version_id,
        }

        if resumable == True or res_download_handler is not None:
            file_object.seek(0, os.SEEK_END)
            current_file_size = file_object.tell()
            if slice_size is not None:
                assert current_file_size < slice_size
                slice_size -= current_file_size
            if slice_offset is not None:
                slice_offset += current_file_size
            else:
                slice_offset = current_file_size

        headers = {}
        _convert_slice_to_range_header(headers, slice_offset, slice_size)
        expected_status = (PARTIAL_CONTENT if "Range" in headers else OK)

        if modified_since is not None:
            timestamp = datetime.utcfromtimestamp(modified_since)
            headers["If-Modified-Since"] = http_timestamp_str(timestamp)
        if unmodified_since is not None:
            timestamp = datetime.utcfromtimestamp(unmodified_since)
            headers["If-Unmodified-Since"] = http_timestamp_str(timestamp)

        method = "GET"
        uri = compute_uri("data", self._name, **kwargs)

        http_connection = self._bucket.create_http_connection()

        self._log.info("requesting GET {0} {1}".format(uri, headers))
        try:
            response = http_connection.request(method,
                                               uri,
                                               body=None,
                                               headers=headers,
                                               expected_status=expected_status)
        except LumberyardHTTPError:
            instance = sys.exc_info()[1]
            http_connection.close()
            if instance.status == NOT_MODIFIED and modified_since is not None:
                raise KeyUnmodified()
            if instance.status == PRECONDITION_FAILED and \
                unmodified_since is not None:
                raise KeyModified()
            raise

        if cb is None:
            reporter = NullCallbackWrapper()
        else:
            reporter = RetrieveCallbackWrapper(self.size, cb, cb_count)

        self._log.info("reading response")
        reporter.start()
        while True:
            data = response.read(_read_buffer_size)
            bytes_read = len(data)
            self._log.debug("read {0} bytes".format(bytes_read))
            if bytes_read == 0:
                break
            file_object.write(data)
            reporter.bytes_written(bytes_read)
        reporter.finish()
        http_connection.close()
Example #4
0
    def get_contents_as_string(self,
                               cb=None,
                               cb_count=10,
                               version_id=None,
                               slice_offset=None,
                               slice_size=None,
                               modified_since=None,
                               unmodified_since=None):
        """
        cb
            callback function for reporting progress

        cb_count
            number of callbacks to be made during the archvie process

        version_id
            the identifier of a specific version to retrieve

            None means retrieve the most recent version

        slice_offset
            byte offset for start of retrieve

            None means start at byte 0

        slice_size
            number of bytes to retrieve

            None means retrieve to end of file

        modified_since
            only retrieve the file if it has been modified since the specified 
            timestamp.

            Otherwise: raise KeyUnmodified

            Note: you cannot specify both modified_since and unmodified_since

        unmodified_since
            only retrieve the file if it has not been modified since the 
            specified timestamp.

            Otherwise: raise KeyModified

            Note: you cannot specify both modified_since and unmodified_since

        retrieve the contents from nimbus.io as a string
        """
        if self._bucket is None:
            raise ValueError("No bucket")
        if self._name is None:
            raise ValueError("No name")
        if modified_since is not None and unmodified_since is not None:
            raise ValueError(
                "Can't specify both modified_since and unmodified_since")

        kwargs = {
            "version_identifier": version_id,
        }
        headers = {}
        _convert_slice_to_range_header(headers, slice_offset, slice_size)
        expected_status = (PARTIAL_CONTENT if "Range" in headers else OK)

        if modified_since is not None:
            timestamp = datetime.utcfromtimestamp(modified_since)
            headers["If-Modified-Since"] = http_timestamp_str(timestamp)
        if unmodified_since is not None:
            timestamp = datetime.utcfromtimestamp(unmodified_since)
            headers["If-Unmodified-Since"] = http_timestamp_str(timestamp)

        method = "GET"
        uri = compute_uri("data", self._name, **kwargs)

        http_connection = self._bucket.create_http_connection()

        self._log.info("requesting GET {0} {1}".format(uri, headers))

        try:
            response = http_connection.request(method,
                                               uri,
                                               body=None,
                                               headers=headers,
                                               expected_status=expected_status)
        except LumberyardHTTPError:
            instance = sys.exc_info()[1]
            http_connection.close()
            if instance.status == NOT_MODIFIED and modified_since is not None:
                raise KeyUnmodified()
            if instance.status == PRECONDITION_FAILED and \
                unmodified_since is not None:
                raise KeyModified()
            raise

        body_list = list()
        while True:
            data = response.read(_read_buffer_size)
            if len(data) == 0:
                break
            body_list.append(data)

        http_connection.close()

        return b"".join(body_list)
Example #5
0
    def get_contents_to_file(
        self,
        file_object,
        cb=None,
        cb_count=10,
        version_id=None,
        slice_offset=None,
        slice_size=None,
        modified_since=None,
        unmodified_since=None,
        resumable=False,
        res_download_handler=None,
    ):
        """
        file_object
            Python file-like object, must support write()
            must support seek() and tell() for resumable=True

        cb
            callback function for reporting progress

        cb_count
            number of callbacks to be made during the archvie process

        version_id
            identifier of a specific version to retrieve

            None means retrieve the most recent version

        slice_offset
            byte offset for start of retrieve

            None means start at byte 0

        slice_size
            number of bytes to retrieve

            None means retrieve to end of file

        modified_since
            only retrieve the file if it has been modified since the specified 
            timestamp.

            Otherwise: raise KeyUnmodified

            Note: you cannot specify both modified_since and unmodified_since

        unmodified_since
            only retrieve the file if it has not been modified since the 
            specified timestamp.

            Otherwise: raise KeyModified

            Note: you cannot specify both modified_since and unmodified_since

        resumable 
            True means append to an existing file if there is one

        res_download_handler
            included for boto compatibility. We have a 
            ResumeableDownloadHandler object, but actually if you put
            anything besides None in this argument, it has the same effect
            as setting resumable to True.

        retrieve the contents from nimbus.io to a file
        """
        if self._bucket is None:
            raise ValueError("No bucket")
        if self._name is None:
            raise ValueError("No name")
        if modified_since is not None and unmodified_since is not None:
            raise ValueError("Can't specify both modified_since and unmodified_since")

        kwargs = {"version_identifier": version_id}

        if resumable == True or res_download_handler is not None:
            file_object.seek(0, os.SEEK_END)
            current_file_size = file_object.tell()
            if slice_size is not None:
                assert current_file_size < slice_size
                slice_size -= current_file_size
            if slice_offset is not None:
                slice_offset += current_file_size
            else:
                slice_offset = current_file_size

        headers = {}
        _convert_slice_to_range_header(headers, slice_offset, slice_size)
        expected_status = PARTIAL_CONTENT if "Range" in headers else OK

        if modified_since is not None:
            timestamp = datetime.utcfromtimestamp(modified_since)
            headers["If-Modified-Since"] = http_timestamp_str(timestamp)
        if unmodified_since is not None:
            timestamp = datetime.utcfromtimestamp(unmodified_since)
            headers["If-Unmodified-Since"] = http_timestamp_str(timestamp)

        method = "GET"
        uri = compute_uri("data", self._name, **kwargs)

        http_connection = self._bucket.create_http_connection()

        self._log.info("requesting GET {0} {1}".format(uri, headers))
        try:
            response = http_connection.request(method, uri, body=None, headers=headers, expected_status=expected_status)
        except LumberyardHTTPError:
            instance = sys.exc_info()[1]
            http_connection.close()
            if instance.status == NOT_MODIFIED and modified_since is not None:
                raise KeyUnmodified()
            if instance.status == PRECONDITION_FAILED and unmodified_since is not None:
                raise KeyModified()
            raise

        if cb is None:
            reporter = NullCallbackWrapper()
        else:
            reporter = RetrieveCallbackWrapper(self.size, cb, cb_count)

        self._log.info("reading response")
        reporter.start()
        while True:
            data = response.read(_read_buffer_size)
            bytes_read = len(data)
            self._log.debug("read {0} bytes".format(bytes_read))
            if bytes_read == 0:
                break
            file_object.write(data)
            reporter.bytes_written(bytes_read)
        reporter.finish()
        http_connection.close()
Example #6
0
    def get_contents_as_string(
        self,
        cb=None,
        cb_count=10,
        version_id=None,
        slice_offset=None,
        slice_size=None,
        modified_since=None,
        unmodified_since=None,
    ):
        """
        cb
            callback function for reporting progress

        cb_count
            number of callbacks to be made during the archvie process

        version_id
            the identifier of a specific version to retrieve

            None means retrieve the most recent version

        slice_offset
            byte offset for start of retrieve

            None means start at byte 0

        slice_size
            number of bytes to retrieve

            None means retrieve to end of file

        modified_since
            only retrieve the file if it has been modified since the specified 
            timestamp.

            Otherwise: raise KeyUnmodified

            Note: you cannot specify both modified_since and unmodified_since

        unmodified_since
            only retrieve the file if it has not been modified since the 
            specified timestamp.

            Otherwise: raise KeyModified

            Note: you cannot specify both modified_since and unmodified_since

        retrieve the contents from nimbus.io as a string
        """
        if self._bucket is None:
            raise ValueError("No bucket")
        if self._name is None:
            raise ValueError("No name")
        if modified_since is not None and unmodified_since is not None:
            raise ValueError("Can't specify both modified_since and unmodified_since")

        kwargs = {"version_identifier": version_id}
        headers = {}
        _convert_slice_to_range_header(headers, slice_offset, slice_size)
        expected_status = PARTIAL_CONTENT if "Range" in headers else OK

        if modified_since is not None:
            timestamp = datetime.utcfromtimestamp(modified_since)
            headers["If-Modified-Since"] = http_timestamp_str(timestamp)
        if unmodified_since is not None:
            timestamp = datetime.utcfromtimestamp(unmodified_since)
            headers["If-Unmodified-Since"] = http_timestamp_str(timestamp)

        method = "GET"
        uri = compute_uri("data", self._name, **kwargs)

        http_connection = self._bucket.create_http_connection()

        self._log.info("requesting GET {0} {1}".format(uri, headers))

        try:
            response = http_connection.request(method, uri, body=None, headers=headers, expected_status=expected_status)
        except LumberyardHTTPError:
            instance = sys.exc_info()[1]
            http_connection.close()
            if instance.status == NOT_MODIFIED and modified_since is not None:
                raise KeyUnmodified()
            if instance.status == PRECONDITION_FAILED and unmodified_since is not None:
                raise KeyModified()
            raise

        body_list = list()
        while True:
            data = response.read(_read_buffer_size)
            if len(data) == 0:
                break
            body_list.append(data)

        http_connection.close()

        return b"".join(body_list)