def setup_openstacksdk():
auth_plugin = utils.get_auth_plugin('neutron')
session = utils.get_keystone_session('neutron', auth_plugin)
# NOTE(mdulko): To get rid of warnings about connection pool being full
# we need to "tweak" the keystoneauth's adapters increasing
# the maximum pool size.
for scheme in list(session.session.adapters):
session.session.mount(scheme,
k_session.TCPKeepAliveAdapter(pool_maxsize=1000))
# TODO(mdulko): To use Neutron's ability to do compare-and-swap updates we
# need to manually add support for inserting If-Match header
# into requests. At the moment we only need it for ports.
# Remove when lower-constraints openstacksdk supports this.
os_port.Port.if_match = os_resource.Header('If-Match')
# TODO(maysams): We need to manually insert allowed_cidrs option
# as it's only supported from 0.41.0 version. Remove it once
# lower-constraints supports it.
os_listener.Listener.allowed_cidrs = os_resource.Body('allowed_cidrs',
type=list)
conn = connection.Connection(session=session,
region_name=getattr(config.CONF.neutron,
'region_name', None))
conn.network.create_ports = partial(_create_ports, conn.network)
conn.network.add_trunk_subports = partial(_add_trunk_subports,
conn.network)
conn.network.delete_trunk_subports = partial(_delete_trunk_subports,
conn.network)
_clients[_OPENSTACKSDK] = conn
class Account(_base.BaseResource):
_custom_metadata_prefix = "X-Account-Meta-"
base_path = "/"
allow_fetch = True
allow_commit = True
allow_head = True
#: The total number of bytes that are stored in Object Storage for
#: the account.
account_bytes_used = resource.Header("x-account-bytes-used", type=int)
#: The number of containers.
account_container_count = resource.Header("x-account-container-count",
type=int)
#: The number of objects in the account.
account_object_count = resource.Header("x-account-object-count", type=int)
#: The secret key value for temporary URLs. If not set,
#: this header is not returned by this operation.
meta_temp_url_key = resource.Header("x-account-meta-temp-url-key")
#: A second secret key value for temporary URLs. If not set,
#: this header is not returned by this operation.
meta_temp_url_key_2 = resource.Header("x-account-meta-temp-url-key-2")
#: The timestamp of the transaction.
timestamp = resource.Header("x-timestamp")
has_body = False
requires_id = False
def setup_openstacksdk():
auth_plugin = utils.get_auth_plugin('neutron')
session = utils.get_keystone_session('neutron', auth_plugin)
# TODO(mdulko): To use Neutron's ability to do compare-and-swap updates we
# need to manually add support for inserting If-Match header
# into requests. At the moment we only need it for ports.
# Remove when lower-constraints openstacksdk supports this.
os_port.Port.if_match = os_resource.Header('If-Match')
conn = connection.Connection(session=session,
region_name=getattr(config.CONF.neutron,
'region_name', None))
conn.network.create_ports = partial(_create_ports, conn.network)
conn.network.add_trunk_subports = partial(_add_trunk_subports,
conn.network)
conn.network.delete_trunk_subports = partial(_delete_trunk_subports,
conn.network)
_clients[_OPENSTACKSDK] = conn
class Fake(resource.Resource):
resource_key = "resource"
resources_key = "resources"
base_path = "/fake"
allow_create = True
allow_fetch = True
allow_commit = True
allow_delete = True
allow_list = True
allow_head = True
#: The transaction date and time.
timestamp = resource.Header("x-timestamp")
#: The name of this resource.
name = resource.Body("name", alternate_id=True)
#: The value of the resource. Also available in headers.
value = resource.Body("value", alias="x-resource-value")
#: Is this resource cool? If so, set it to True.
#: This is a multi-line comment about cool stuff.
cool = resource.Body("cool", type=bool)
class Account(_base.BaseResource):
_custom_metadata_prefix = "X-Account-Meta-"
base_path = "/"
allow_fetch = True
allow_commit = True
allow_head = True
#: The total number of bytes that are stored in Object Storage for
#: the account.
account_bytes_used = resource.Header("x-account-bytes-used", type=int)
#: The number of containers.
account_container_count = resource.Header("x-account-container-count",
type=int)
#: The number of objects in the account.
account_object_count = resource.Header("x-account-object-count", type=int)
#: The secret key value for temporary URLs. If not set,
#: this header is not returned by this operation.
meta_temp_url_key = resource.Header("x-account-meta-temp-url-key")
#: A second secret key value for temporary URLs. If not set,
#: this header is not returned by this operation.
meta_temp_url_key_2 = resource.Header("x-account-meta-temp-url-key-2")
#: The timestamp of the transaction.
timestamp = resource.Header("x-timestamp")
has_body = False
requires_id = False
def set_temp_url_key(self, proxy, key, secondary=False):
"""Set the temporary url key for the account.
:param proxy: The proxy to use for making this request.
:type proxy: :class:`~openstack.proxy.Proxy`
:param key:
Text of the key to use.
:param bool secondary:
Whether this should set the secondary key. (defaults to False)
"""
header = 'Temp-URL-Key'
if secondary:
header += '-2'
return self.set_metadata(proxy, {header: key})
class Container(_base.BaseResource):
_custom_metadata_prefix = "X-Container-Meta-"
_system_metadata = {
"content_type": "content-type",
"is_content_type_detected": "x-detect-content-type",
"versions_location": "x-versions-location",
"read_ACL": "x-container-read",
"write_ACL": "x-container-write",
"sync_to": "x-container-sync-to",
"sync_key": "x-container-sync-key"
}
base_path = "/"
pagination_key = 'X-Account-Container-Count'
allow_create = True
allow_fetch = True
allow_commit = True
allow_delete = True
allow_list = True
allow_head = True
_query_mapping = resource.QueryParameters('prefix', )
# Container body data (when id=None)
#: The name of the container.
name = resource.Body("name", alternate_id=True, alias='id')
#: The number of objects in the container.
count = resource.Body("count", type=int, alias='object_count')
#: The total number of bytes that are stored in Object Storage
#: for the container.
bytes = resource.Body("bytes", type=int, alias='bytes_used')
# Container metadata (when id=name)
#: The number of objects.
object_count = resource.Header("x-container-object-count",
type=int,
alias='count')
#: The count of bytes used in total.
bytes_used = resource.Header("x-container-bytes-used",
type=int,
alias='bytes')
#: The timestamp of the transaction.
timestamp = resource.Header("x-timestamp")
# Request headers (when id=None)
#: If set to True, Object Storage queries all replicas to return the
#: most recent one. If you omit this header, Object Storage responds
#: faster after it finds one valid replica. Because setting this
#: header to True is more expensive for the back end, use it only
#: when it is absolutely needed. *Type: bool*
is_newest = resource.Header("x-newest", type=bool)
# Request headers (when id=name)
#: The ACL that grants read access. If not set, this header is not
#: returned by this operation.
read_ACL = resource.Header("x-container-read")
#: The ACL that grants write access. If not set, this header is not
#: returned by this operation.
write_ACL = resource.Header("x-container-write")
#: The destination for container synchronization. If not set,
#: this header is not returned by this operation.
sync_to = resource.Header("x-container-sync-to")
#: The secret key for container synchronization. If not set,
#: this header is not returned by this operation.
sync_key = resource.Header("x-container-sync-key")
#: Enables versioning on this container. The value is the name
#: of another container. You must UTF-8-encode and then URL-encode
#: the name before you include it in the header. To disable
#: versioning, set the header to an empty string.
versions_location = resource.Header("x-versions-location")
#: The MIME type of the list of names.
content_type = resource.Header("content-type")
#: If set to true, Object Storage guesses the content type based
#: on the file extension and ignores the value sent in the
#: Content-Type header, if present. *Type: bool*
is_content_type_detected = resource.Header("x-detect-content-type",
type=bool)
# TODO(mordred) Shouldn't if-none-match be handled more systemically?
#: In combination with Expect: 100-Continue, specify an
#: "If-None-Match: \*" header to query whether the server already
#: has a copy of the object before any data is sent.
if_none_match = resource.Header("if-none-match")
@classmethod
def new(cls, **kwargs):
# Container uses name as id. Proxy._get_resource calls
# Resource.new(id=name) but then we need to do container.name
# It's the same thing for Container - make it be the same.
name = kwargs.pop('id', None)
if name:
kwargs.setdefault('name', name)
return Container(_synchronized=True, **kwargs)
def create(self, session, prepend_key=True, base_path=None):
"""Create a remote resource based on this instance.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param prepend_key: A boolean indicating whether the resource_key
should be prepended in a resource creation
request. Default to True.
:return: This :class:`Resource` instance.
:raises: :exc:`~openstack.exceptions.MethodNotSupported` if
:data:`Resource.allow_create` is not set to ``True``.
"""
request = self._prepare_request(requires_id=True,
prepend_key=prepend_key,
base_path=base_path)
response = session.put(request.url,
json=request.body,
headers=request.headers)
self._translate_response(response, has_body=False)
return self
class Object(_base.BaseResource):
_custom_metadata_prefix = "X-Object-Meta-"
_system_metadata = {
"content_disposition": "content-disposition",
"content_encoding": "content-encoding",
"content_type": "content-type",
"delete_after": "x-delete-after",
"delete_at": "x-delete-at",
"is_content_type_detected": "x-detect-content-type",
}
base_path = "/%(container)s"
pagination_key = 'X-Container-Object-Count'
allow_create = True
allow_fetch = True
allow_commit = True
allow_delete = True
allow_list = True
allow_head = True
_query_mapping = resource.QueryParameters('prefix', 'format')
# Data to be passed during a POST call to create an object on the server.
# TODO(mordred) Make a base class BaseDataResource that can be used here
# and with glance images that has standard overrides for dealing with
# binary data.
data = None
# URL parameters
#: The unique name for the container.
container = resource.URI("container")
#: The unique name for the object.
name = resource.Body("name", alternate_id=True)
# Object details
# Make these private because they should only matter in the case where
# we have a Body with no headers (like if someone programmatically is
# creating an Object)
_hash = resource.Body("hash")
_bytes = resource.Body("bytes", type=int)
_last_modified = resource.Body("last_modified")
_content_type = resource.Body("content_type")
# Headers for HEAD and GET requests
#: If set to True, Object Storage queries all replicas to return
#: the most recent one. If you omit this header, Object Storage
#: responds faster after it finds one valid replica. Because
#: setting this header to True is more expensive for the back end,
#: use it only when it is absolutely needed. *Type: bool*
is_newest = resource.Header("x-newest", type=bool)
#: TODO(briancurtin) there's a lot of content here...
range = resource.Header("range", type=dict)
#: See http://www.ietf.org/rfc/rfc2616.txt.
if_match = resource.Header("if-match", type=list)
#: In combination with Expect: 100-Continue, specify an
#: "If-None-Match: \*" header to query whether the server already
#: has a copy of the object before any data is sent.
if_none_match = resource.Header("if-none-match", type=list)
#: See http://www.ietf.org/rfc/rfc2616.txt.
if_modified_since = resource.Header("if-modified-since", type=str)
#: See http://www.ietf.org/rfc/rfc2616.txt.
if_unmodified_since = resource.Header("if-unmodified-since", type=str)
# Query parameters
#: Used with temporary URLs to sign the request. For more
#: information about temporary URLs, see OpenStack Object Storage
#: API v1 Reference.
signature = resource.Header("signature")
#: Used with temporary URLs to specify the expiry time of the
#: signature. For more information about temporary URLs, see
#: OpenStack Object Storage API v1 Reference.
expires_at = resource.Header("expires")
#: If you include the multipart-manifest=get query parameter and
#: the object is a large object, the object contents are not
#: returned. Instead, the manifest is returned in the
#: X-Object-Manifest response header for dynamic large objects
#: or in the response body for static large objects.
multipart_manifest = resource.Header("multipart-manifest")
# Response headers from HEAD and GET
#: HEAD operations do not return content. However, in this
#: operation the value in the Content-Length header is not the
#: size of the response body. Instead it contains the size of
#: the object, in bytes.
content_length = resource.Header("content-length",
type=int,
alias='_bytes')
#: The MIME type of the object.
content_type = resource.Header("content-type", alias="_content_type")
#: The type of ranges that the object accepts.
accept_ranges = resource.Header("accept-ranges")
#: For objects smaller than 5 GB, this value is the MD5 checksum
#: of the object content. The value is not quoted.
#: For manifest objects, this value is the MD5 checksum of the
#: concatenated string of MD5 checksums and ETags for each of
#: the segments in the manifest, and not the MD5 checksum of
#: the content that was downloaded. Also the value is enclosed
#: in double-quote characters.
#: You are strongly recommended to compute the MD5 checksum of
#: the response body as it is received and compare this value
#: with the one in the ETag header. If they differ, the content
#: was corrupted, so retry the operation.
etag = resource.Header("etag", alias='_hash')
#: Set to True if this object is a static large object manifest object.
#: *Type: bool*
is_static_large_object = resource.Header("x-static-large-object",
type=bool)
#: If set, the value of the Content-Encoding metadata.
#: If not set, this header is not returned by this operation.
content_encoding = resource.Header("content-encoding")
#: If set, specifies the override behavior for the browser.
#: For example, this header might specify that the browser use
#: a download program to save this file rather than show the file,
#: which is the default.
#: If not set, this header is not returned by this operation.
content_disposition = resource.Header("content-disposition")
#: Specifies the number of seconds after which the object is
#: removed. Internally, the Object Storage system stores this
#: value in the X-Delete-At metadata item.
delete_after = resource.Header("x-delete-after", type=int)
#: If set, the time when the object will be deleted by the system
#: in the format of a UNIX Epoch timestamp.
#: If not set, this header is not returned by this operation.
delete_at = resource.Header("x-delete-at")
#: If set, to this is a dynamic large object manifest object.
#: The value is the container and object name prefix of the
#: segment objects in the form container/prefix.
object_manifest = resource.Header("x-object-manifest")
#: The timestamp of the transaction.
timestamp = resource.Header("x-timestamp")
#: The date and time that the object was created or the last
#: time that the metadata was changed.
last_modified_at = resource.Header("last-modified", alias='_last_modified')
# Headers for PUT and POST requests
#: Set to chunked to enable chunked transfer encoding. If used,
#: do not set the Content-Length header to a non-zero value.
transfer_encoding = resource.Header("transfer-encoding")
#: If set to true, Object Storage guesses the content type based
#: on the file extension and ignores the value sent in the
#: Content-Type header, if present. *Type: bool*
is_content_type_detected = resource.Header("x-detect-content-type",
type=bool)
#: If set, this is the name of an object used to create the new
#: object by copying the X-Copy-From object. The value is in form
#: {container}/{object}. You must UTF-8-encode and then URL-encode
#: the names of the container and object before you include them
#: in the header.
#: Using PUT with X-Copy-From has the same effect as using the
#: COPY operation to copy an object.
copy_from = resource.Header("x-copy-from")
has_body = False
def __init__(self, data=None, **attrs):
super(_base.BaseResource, self).__init__(**attrs)
self.data = data
# The Object Store treats the metadata for its resources inconsistently so
# Object.set_metadata must override the BaseResource.set_metadata to
# account for it.
def set_metadata(self, session, metadata):
# Filter out items with empty values so the create metadata behaviour
# is the same as account and container
filtered_metadata = \
{key: value for key, value in metadata.items() if value}
# Update from remote if we only have locally created information
if not self.last_modified_at:
self.head(session)
# Get a copy of the original metadata so it doesn't get erased on POST
# and update it with the new metadata values.
metadata = copy.deepcopy(self.metadata)
metadata.update(filtered_metadata)
# Include any original system metadata so it doesn't get erased on POST
for key in self._system_metadata:
value = getattr(self, key)
if value and key not in metadata:
metadata[key] = value
request = self._prepare_request()
headers = self._calculate_headers(metadata)
response = session.post(request.url, headers=headers)
self._translate_response(response, has_body=False)
self.metadata.update(metadata)
return self
# The Object Store treats the metadata for its resources inconsistently so
# Object.delete_metadata must override the BaseResource.delete_metadata to
# account for it.
def delete_metadata(self, session, keys):
if not keys:
return
# If we have an empty object, update it from the remote side so that
# we have a copy of the original metadata. Deleting metadata requires
# POSTing and overwriting all of the metadata. If we already have
# metadata locally, assume this is an existing object.
if not self.metadata:
self.head(session)
metadata = copy.deepcopy(self.metadata)
# Include any original system metadata so it doesn't get erased on POST
for key in self._system_metadata:
value = getattr(self, key)
if value:
metadata[key] = value
# Remove the requested metadata keys
# TODO(mordred) Why don't we just look at self._header_mapping()
# instead of having system_metadata?
deleted = False
attr_keys_to_delete = set()
for key in keys:
if key == 'delete_after':
del (metadata['delete_at'])
else:
if key in metadata:
del (metadata[key])
# Delete the attribute from the local copy of the object.
# Metadata that doesn't have Component attributes is
# handled by self.metadata being reset when we run
# self.head
if hasattr(self, key):
attr_keys_to_delete.add(key)
deleted = True
# Nothing to delete, skip the POST
if not deleted:
return self
request = self._prepare_request()
response = session.post(request.url,
headers=self._calculate_headers(metadata))
exceptions.raise_from_response(
response, error_message="Error deleting metadata keys")
# Only delete from local object if the remote delete was successful
for key in attr_keys_to_delete:
delattr(self, key)
# Just update ourselves from remote again.
return self.head(session)
def _download(self, session, error_message=None, stream=False):
request = self._prepare_request()
request.headers['Accept'] = 'bytes'
response = session.get(request.url,
headers=request.headers,
stream=stream)
exceptions.raise_from_response(response, error_message=error_message)
return response
def download(self, session, error_message=None):
response = self._download(session, error_message=error_message)
return response.content
def stream(self, session, error_message=None, chunk_size=1024):
response = self._download(session,
error_message=error_message,
stream=True)
return response.iter_content(chunk_size, decode_unicode=False)
def create(self, session, base_path=None):
request = self._prepare_request(base_path=base_path)
request.headers['Accept'] = ''
response = session.put(request.url,
data=self.data,
headers=request.headers)
self._translate_response(response, has_body=False)
return self
def _raw_delete(self, session):
if not self.allow_delete:
raise exceptions.MethodNotSupported(self, "delete")
request = self._prepare_request()
session = self._get_session(session)
microversion = self._get_microversion_for(session, 'delete')
if self.is_static_large_object is None:
# Fetch metadata to determine SLO flag
self.head(session)
headers = {'Accept': ""}
if self.is_static_large_object:
headers['multipart-manifest'] = 'delete'
return session.delete(request.url,
headers=headers,
microversion=microversion)
class Queue(resource.Resource):
# FIXME(anyone): The name string of `location` field of Zaqar API response
# is lower case. That is inconsistent with the guide from API-WG. This is
# a workaround for this issue.
location = resource.Header("location")
resources_key = "queues"
base_path = "/queues"
# capabilities
allow_create = True
allow_list = True
allow_fetch = True
allow_delete = True
# Properties
#: The default TTL of messages defined for a queue, which will effect for
#: any messages posted to the queue.
default_message_ttl = resource.Body("_default_message_ttl")
#: Description of the queue.
description = resource.Body("description")
#: The max post size of messages defined for a queue, which will effect
#: for any messages posted to the queue.
max_messages_post_size = resource.Body("_max_messages_post_size")
#: Name of the queue. The name is the unique identity of a queue. It
#: must not exceed 64 bytes in length, and it is limited to US-ASCII
#: letters, digits, underscores, and hyphens.
name = resource.Body("name", alternate_id=True)
#: The ID to identify the client accessing Zaqar API. Must be specified
#: in header for each API request.
client_id = resource.Header("Client-ID")
#: The ID to identify the project accessing Zaqar API. Must be specified
#: in case keystone auth is not enabled in Zaqar service.
project_id = resource.Header("X-PROJECT-ID")
def create(self, session, prepend_key=True, base_path=None):
request = self._prepare_request(requires_id=True,
prepend_key=prepend_key,
base_path=None)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.put(request.url,
json=request.body,
headers=request.headers)
self._translate_response(response, has_body=False)
return self
@classmethod
def list(cls, session, paginated=False, base_path=None, **params):
"""This method is a generator which yields queue objects.
This is almost the copy of list method of resource.Resource class.
The only difference is the request header now includes `Client-ID`
and `X-PROJECT-ID` fields which are required by Zaqar v2 API.
"""
more_data = True
query_params = cls._query_mapping._transpose(params)
if base_path is None:
base_path = cls.base_path
uri = base_path % params
headers = {
"Client-ID":
params.get('client_id', None) or str(uuid.uuid4()),
"X-PROJECT-ID":
params.get('project_id', None) or session.get_project_id()
}
while more_data:
resp = session.get(uri, headers=headers, params=query_params)
resp = resp.json()
resp = resp[cls.resources_key]
if not resp:
more_data = False
yielded = 0
new_marker = None
for data in resp:
value = cls.existing(**data)
new_marker = value.id
yielded += 1
yield value
if not paginated:
return
if "limit" in query_params and yielded < query_params["limit"]:
return
query_params["limit"] = yielded
query_params["marker"] = new_marker
def fetch(self,
session,
requires_id=True,
base_path=None,
error_message=None):
request = self._prepare_request(requires_id=requires_id,
base_path=base_path)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.get(request.url, headers=headers)
self._translate_response(response)
return self
def delete(self, session):
request = self._prepare_request()
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.delete(request.url, headers=headers)
self._translate_response(response, has_body=False)
return self
class Claim(resource.Resource):
# FIXME(anyone): The name string of `location` field of Zaqar API response
# is lower case. That is inconsistent with the guide from API-WG. This is
# a workaround for this issue.
location = resource.Header("location")
resources_key = 'claims'
base_path = '/queues/%(queue_name)s/claims'
service = message_service.MessageService()
# capabilities
allow_create = True
allow_fetch = True
allow_commit = True
allow_delete = True
commit_method = 'PATCH'
# Properties
#: The value in seconds indicating how long the claim has existed.
age = resource.Body("age")
#: In case worker stops responding for a long time, the server will
#: extend the lifetime of claimed messages to be at least as long as
#: the lifetime of the claim itself, plus the specified grace period.
#: Must between 60 and 43200 seconds(12 hours).
grace = resource.Body("grace")
#: The number of messages to claim. Default 10, up to 20.
limit = resource.Body("limit")
#: Messages have been successfully claimed.
messages = resource.Body("messages")
#: Number of seconds the server wait before releasing the claim. Must
#: between 60 and 43200 seconds(12 hours).
ttl = resource.Body("ttl")
#: The name of queue to claim message from.
queue_name = resource.URI("queue_name")
#: The ID to identify the client accessing Zaqar API. Must be specified
#: in header for each API request.
client_id = resource.Header("Client-ID")
#: The ID to identify the project. Must be provided when keystone
#: authentication is not enabled in Zaqar service.
project_id = resource.Header("X-PROJECT-ID")
def _translate_response(self, response, has_body=True):
super(Claim, self)._translate_response(response, has_body=has_body)
if has_body and self.location:
# Extract claim ID from location
self.id = self.location.split("claims/")[1]
def create(self, session, prepend_key=False):
request = self._prepare_request(requires_id=False,
prepend_key=prepend_key)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.post(request.url,
json=request.body,
headers=request.headers)
# For case no message was claimed successfully, 204 No Content
# message will be returned. In other cases, we translate response
# body which has `messages` field(list) included.
if response.status_code != 204:
self._translate_response(response)
return self
def fetch(self, session, requires_id=True, error_message=None):
request = self._prepare_request(requires_id=requires_id)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.get(request.url, headers=request.headers)
self._translate_response(response)
return self
def commit(self, session, prepend_key=False, has_body=False):
request = self._prepare_request(prepend_key=prepend_key)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
session.patch(request.url, json=request.body, headers=request.headers)
return self
def delete(self, session):
request = self._prepare_request()
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.delete(request.url, headers=request.headers)
self._translate_response(response, has_body=False)
return self
class Object(_base.BaseResource):
base_path = '/'
allow_create = True
allow_get = True
allow_commit = True
allow_delete = True
allow_list = True
allow_head = True
resources_key = ''
resource_key = 'Contents'
_query_mapping = resource.QueryParameters('prefix',
'delimiter',
'limit',
prefix='prefix',
delimiter='delimiter',
limit='max-keys')
data = None
name = resource.Body('Key', alternate_id=True)
last_modified = resource.Body('LastModified')
etag = resource.Body('ETag')
content_length = resource.Body('Size', type=int)
storage_class = resource.Body('StorageClass')
content_md5 = resource.Header('Content-MD5', type=str)
#: private, public-read, public-read-write, authenticated-read
#: bucket-owner-read, bucket-owner-full-control
acl = resource.Header('x-amz-acl')
object_storage_class = resource.Header('x-amz-storage-class')
container = resource.URI('container')
def __init__(self, data=None, **attrs):
super(_base.BaseResource, self).__init__(**attrs)
self.data = data
def _translate_response(self, response, has_body=True, error_message=None):
"""Given a KSA response, inflate this instance with its data
This method updates attributes that correspond to headers
and body on this instance and clears the dirty set.
"""
exceptions.raise_from_response(response, error_message=response.text)
_logger.debug(response.text)
if response:
if has_body:
# TODO(agoncharov): do nothing so far. Generally need
# to parse different responses
pass
@classmethod
def list(cls,
session,
paginated=False,
endpoint_override=None,
headers=None,
requests_auth=None,
**params):
if not cls.allow_list:
raise exceptions.MethodNotSupported(cls, "list")
cls._query_mapping._validate(params, base_path=cls.base_path)
query_params = cls._query_mapping._transpose(params)
uri = cls.base_path % params
# Build additional arguments to the GET call
get_args = cls._prepare_override_args(
endpoint_override=endpoint_override, additional_headers=headers)
while uri:
response = session.get(uri,
params=query_params.copy(),
requests_auth=requests_auth,
**get_args)
uri = None
next_params = {}
root = ET.fromstring(response.content)
if root.tag != ET.QName(cls.OBS_NS, 'ListBucketResult'):
_logger.warn('Namespace in the response does not match '
'expectation')
cls.OBS_NS = root.tag.split('}', 1)[0][1:]
for element in root:
if element.tag == ET.QName(cls.OBS_NS, cls.resource_key):
# Convert XML part into dict
dict_raw_resource = cls.etree_to_dict(element)
# extract resource data
dict_resource = dict_raw_resource[cls.resource_key]
value = cls.existing(**dict_resource)
yield value
elif element.tag == ET.QName(cls.OBS_NS, 'NextMarker'):
next_params['marker'] = element.text
if 'marker' in next_params:
uri = cls.base_path % params
query_params.update(next_params)
return
def create(self,
session,
prepend_key=True,
endpoint_override=None,
headers=None,
requests_auth=None):
if not self.allow_create:
raise exceptions.MethodNotSupported(self, 'create')
session = self._get_session(session)
if not self.content_md5 and self.data:
md5 = hashlib.md5()
md5.update(str.encode(self.data))
self.content_md5 = base64.b64encode(md5.digest()).decode()
request = self._prepare_request(requires_id=True,
prepend_key=prepend_key)
req_args = self._prepare_override_args(
endpoint_override=endpoint_override,
request_headers=request.headers,
additional_headers=headers,
requests_auth=requests_auth)
response = session.put(request.url, data=self.data, **req_args)
self._translate_response(response)
return self
def download(self,
session,
filename=None,
endpoint_override=None,
requests_auth=None):
session = self._get_session(session)
request = self._prepare_request(requires_id=True)
req_args = self._prepare_override_args(
endpoint_override=endpoint_override,
request_headers=request.headers,
requests_auth=requests_auth)
response = session.get(request.url, **req_args)
self._translate_response(response)
_logger.debug(response.content)
with open(filename, 'wb') as f:
f.write(response.content)
return
class Subscription(resource.Resource):
# FIXME(anyone): The name string of `location` field of Zaqar API response
# is lower case. That is inconsistent with the guide from API-WG. This is
# a workaround for this issue.
location = resource.Header("location")
resources_key = 'subscriptions'
base_path = '/queues/%(queue_name)s/subscriptions'
# capabilities
allow_create = True
allow_list = True
allow_fetch = True
allow_delete = True
# Properties
#: The value in seconds indicating how long the subscription has existed.
age = resource.Body("age")
#: Alternate id of the subscription. This key is used in response of
#: subscription create API to return id of subscription created.
subscription_id = resource.Body("subscription_id", alternate_id=True)
#: The extra metadata for the subscription. The value must be a dict.
#: If the subscriber is `mailto`. The options can contain `from` and
#: `subject` to indicate the email's author and title.
options = resource.Body("options", type=dict)
#: The queue name which the subscription is registered on.
source = resource.Body("source")
#: The destination of the message. Two kinds of subscribers are supported:
#: http/https and email. The http/https subscriber should start with
#: `http/https`. The email subscriber should start with `mailto`.
subscriber = resource.Body("subscriber")
#: Number of seconds the subscription remains alive? The ttl value must
#: be great than 60 seconds. The default value is 3600 seconds.
ttl = resource.Body("ttl")
#: The queue name which the subscription is registered on.
queue_name = resource.URI("queue_name")
#: The ID to identify the client accessing Zaqar API. Must be specified
#: in header for each API request.
client_id = resource.Header("Client-ID")
#: The ID to identify the project. Must be provided when keystone
#: authentication is not enabled in Zaqar service.
project_id = resource.Header("X-PROJECT-ID")
def create(self, session, prepend_key=True, base_path=None):
request = self._prepare_request(requires_id=False,
prepend_key=prepend_key,
base_path=base_path)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.post(request.url,
json=request.body,
headers=request.headers)
self._translate_response(response)
return self
@classmethod
def list(cls, session, paginated=True, base_path=None, **params):
"""This method is a generator which yields subscription objects.
This is almost the copy of list method of resource.Resource class.
The only difference is the request header now includes `Client-ID`
and `X-PROJECT-ID` fields which are required by Zaqar v2 API.
"""
more_data = True
if base_path is None:
base_path = cls.base_path
uri = base_path % params
headers = {
"Client-ID":
params.get('client_id', None) or str(uuid.uuid4()),
"X-PROJECT-ID":
params.get('project_id', None) or session.get_project_id()
}
query_params = cls._query_mapping._transpose(params, cls)
while more_data:
resp = session.get(uri, headers=headers, params=query_params)
resp = resp.json()
resp = resp[cls.resources_key]
if not resp:
more_data = False
yielded = 0
new_marker = None
for data in resp:
value = cls.existing(**data)
new_marker = value.id
yielded += 1
yield value
if not paginated:
return
if "limit" in query_params and yielded < query_params["limit"]:
return
query_params["limit"] = yielded
query_params["marker"] = new_marker
def fetch(self,
session,
requires_id=True,
base_path=None,
error_message=None):
request = self._prepare_request(requires_id=requires_id,
base_path=base_path)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.get(request.url, headers=request.headers)
self._translate_response(response)
return self
def delete(self, session):
request = self._prepare_request()
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.delete(request.url, headers=request.headers)
self._translate_response(response, has_body=False)
return self
class Message(resource.Resource):
# FIXME(anyone): The name string of `location` field of Zaqar API response
# is lower case. That is inconsistent with the guide from API-WG. This is
# a workaround for this issue.
location = resource.Header("location")
resources_key = 'messages'
base_path = '/queues/%(queue_name)s/messages'
# capabilities
allow_create = True
allow_list = True
allow_fetch = True
allow_delete = True
_query_mapping = resource.QueryParameters("echo", "include_claimed")
# Properties
#: The value in second to specify how long the message has been
#: posted to the queue.
age = resource.Body("age")
#: A dictionary specifies an arbitrary document that constitutes the
#: body of the message being sent.
body = resource.Body("body")
#: An uri string describe the location of the message resource.
href = resource.Body("href")
#: The value in seconds to specify how long the server waits before
#: marking the message as expired and removing it from the queue.
ttl = resource.Body("ttl")
#: The name of target queue message is post to or got from.
queue_name = resource.URI("queue_name")
#: The ID to identify the client accessing Zaqar API. Must be specified
#: in header for each API request.
client_id = resource.Header("Client-ID")
#: The ID to identify the project accessing Zaqar API. Must be specified
#: in case keystone auth is not enabled in Zaqar service.
project_id = resource.Header("X-PROJECT-ID")
def post(self, session, messages):
request = self._prepare_request(requires_id=False, prepend_key=True)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
request.body = {'messages': messages}
response = session.post(request.url,
json=request.body, headers=request.headers)
return response.json()['resources']
@classmethod
def list(cls, session, paginated=True, base_path=None, **params):
"""This method is a generator which yields message objects.
This is almost the copy of list method of resource.Resource class.
The only difference is the request header now includes `Client-ID`
and `X-PROJECT-ID` fields which are required by Zaqar v2 API.
"""
more_data = True
if base_path is None:
base_path = cls.base_path
uri = base_path % params
headers = {
"Client-ID": params.get('client_id', None) or str(uuid.uuid4()),
"X-PROJECT-ID": params.get('project_id', None
) or session.get_project_id()
}
query_params = cls._query_mapping._transpose(params, cls)
while more_data:
resp = session.get(uri,
headers=headers, params=query_params)
resp = resp.json()
resp = resp[cls.resources_key]
if not resp:
more_data = False
yielded = 0
new_marker = None
for data in resp:
value = cls.existing(**data)
new_marker = value.id
yielded += 1
yield value
if not paginated:
return
if "limit" in query_params and yielded < query_params["limit"]:
return
query_params["limit"] = yielded
query_params["marker"] = new_marker
def fetch(self, session, requires_id=True,
base_path=None, error_message=None):
request = self._prepare_request(requires_id=requires_id,
base_path=base_path)
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
response = session.get(request.url,
headers=headers)
self._translate_response(response)
return self
def delete(self, session):
request = self._prepare_request()
headers = {
"Client-ID": self.client_id or str(uuid.uuid4()),
"X-PROJECT-ID": self.project_id or session.get_project_id()
}
request.headers.update(headers)
# For Zaqar v2 API requires client to specify claim_id as query
# parameter when deleting a message that has been claimed, we
# rebuild the request URI if claim_id is not None.
if self.claim_id:
request.url += '?claim_id=%s' % self.claim_id
response = session.delete(request.url,
headers=headers)
self._translate_response(response, has_body=False)
return self
class Container(_base.BaseResource):
resources_key = 'Buckets'
resource_key = 'Bucket'
allow_get = True
allow_head = True
allow_list = True
allow_create = True
allow_delete = True
create_method = 'PUT'
base_path = '/'
# all requests (except create) will default to requires_id = None
requires_id = None
name = resource.Body('Name', alternate_id=True, alias='id')
creation_date = resource.Body('CreationDate')
storage_acl = resource.Header('x-amz-acl')
storage_class = resource.Header('x-default-storage-class')
def _translate_response(self, response, has_body=True, error_message=None):
"""Given a KSA response, inflate this instance with its data
This method updates attributes that correspond to headers
and body on this instance and clears the dirty set.
"""
exceptions.raise_from_response(response, error_message=response.text)
if response:
if has_body:
# TODO(agoncharov): do nothing so far. Generally need
# to parse different responses
pass
def _prepare_request(self, requires_id=None, prepend_key=False):
"""Prepare a request to be sent to the server
Create operations don't require an ID, but all others do,
so only try to append an ID when it's needed with
requires_id. Create and update operations sometimes require
their bodies to be contained within an dict -- if the
instance contains a resource_key and prepend_key=True,
the body will be wrapped in a dict with that key.
Return a _Request object that contains the constructed URI
as well a body and headers that are ready to send.
Only dirty body and header contents will be returned.
"""
if requires_id is None:
requires_id = self.requires_id
body = None
# body = self._body.dirty
# if prepend_key and self.resource_key is not None:
# body = {self.resource_key: body}
# if self.name:
# body['Bucket'] = self.name
base_path = '/'
headers = {}
uri = base_path % self._uri.attributes
if requires_id:
if self.id is None:
raise exceptions.InvalidRequest(
"Request requires an ID but none was found")
uri = utils.urljoin(uri, self.id)
return resource._Request(uri, body, headers)
@classmethod
def list(cls, session, paginated=False,
endpoint_override=None, headers=None, requests_auth=None,
**params):
if not cls.allow_list:
raise exceptions.MethodNotSupported(cls, "list")
cls._query_mapping._validate(params, base_path=cls.base_path)
query_params = cls._query_mapping._transpose(params, cls)
response = session.get(
session.get_endpoint(),
params=query_params.copy(),
requests_auth=requests_auth
)
root = ET.fromstring(response.content)
if root.tag != ET.QName(cls.OBS_NS, 'ListAllMyBucketsResult'):
_logger.warn('Namespace in the response does not match '
'expectation')
cls.OBS_NS = root.tag.split('}', 1)[0][1:]
for elements in root:
if elements.tag == ET.QName(cls.OBS_NS, cls.resources_key):
for el in elements:
if el.tag == ET.QName(cls.OBS_NS, cls.resource_key):
# Convert XML part into dict
dict_raw_resource = cls.etree_to_dict(el)
# extract resource data
dict_resource = dict_raw_resource[cls.resource_key]
value = cls.existing(**dict_resource)
yield value
return
def create(self, session, prepend_key=True,
endpoint_override=None, headers=None, requests_auth=None):
if not self.allow_create:
raise exceptions.MethodNotSupported(self, "create")
session = self._get_session(session)
request = self._prepare_request()
req_args = self._prepare_override_args(
endpoint_override=endpoint_override,
request_headers=request.headers,
additional_headers=headers,
requests_auth=requests_auth)
response = session.put(request.url,
data=request.body, **req_args)
self._translate_response(response)
return self
class Object(_base.BaseResource):
_custom_metadata_prefix = "x-amz-meta-"
base_path = '/'
allow_create = True
allow_get = True
allow_commit = True
allow_delete = True
allow_list = True
allow_head = True
resources_key = ''
resource_key = 'Contents'
_query_mapping = resource.QueryParameters(
'prefix', 'delimiter',
'limit',
prefix='prefix',
delimiter='delimiter',
limit='max-keys'
)
# Data to be passed during a POST call to create an object on the server.
data = None
# URL parameters
#: The unique name for the container.
container = resource.URI("container")
#: The unique name for the object.
name = resource.Body('Key', alternate_id=True)
#: The date and time that the object was created or the last
#: time that the metadata was changed.
last_modified = resource.Body('LastModified')
#: size of the response body. Instead it contains the size of
#: the object, in bytes.
content_length = resource.Body('Size', type=int)
# Headers for requests
#: private, public-read, public-read-write, authenticated-read
#: bucket-owner-read, bucket-owner-full-control
acl = resource.Header('x-amz-acl')
accept_ranges = resource.Header('Accept-Ranges')
#: The MD5 digest string of the message body is calculated according
#: to the RFC 1864 standard. That is, calculate the 128-bit binary array
#: (the message header data encrypted with MD5) first,
#: and then use Base 64 encoding to convert the binary data to
#: a character string.
content_md5 = resource.Header('Content-MD5', type=str)
#: Indicates the content type of a requested resource, for example,
#: text/plain.
content_type = resource.Header('Content-Type', type=str)
#: Indicates the hash value of an object.
#: The entity tag (ETag) only reflects changes to the contents
#: of an object, not its metadata.
etag = resource.Header('ETag', type=str)
#: Indicates the value created by OBS to uniquely identify a request.
#: OBS uses this value to troubleshoot faults.
request_id = resource.Header('x-amz-request-id', type=str)
#: Indicates a special token that helps OBS troubleshoot faults.
request_id_2 = resource.Header('x-amz-id-2', type=str)
#: Indicates that SSE-KMS is used.
#: Example: x-amz-server-side-encryption:aws:kms
sse = resource.Header('x-amz-server-side-encryption')
#: Indicates the master key ID. This header is used in SSE-KMS mode.
#: If the customer does not provide the master key,
#: the default master key will be used.
sse_key_id = resource.Header('x-amz-server-side-encryption-aws-kms-key-id')
#: Indicates a decryption algorithm. The header is used in SSE-C mode.
#: Constraints: This header must be used together with
#: x-amz-server-side-encryption-customer-key and
#: x-amz-server-side-encryption-customer-key-MD5.
sse_algorithm = resource.Header(
'x-amz-server-side-encryption-customer-algorithm'
)
#: Indicates a key used to decrypt objects.
#: The header is used in SSE-C mode.
#: Constraints: This header is a base64-encoded 256-bit or 512-bit key and
#: must be used together with
# x-amz-server-side-encryption-customer-algorithm and
# x-amz-server-side-encryption-customer-key-MD5
sse_key = resource.Header('x-amz-server-side-encryption-customer-key')
#: Indicates the MD5 value of a key used to decrypt objects.
#: The header is used in SSE-C mode.
#: The MD5 value is used to check whether any error
#: occurs during the transmission of the key.
#: Constraints: This header is a base64-encoded 128-bit MD5 value and
#: must be used together with
#: x-amz-server-side-encryption-customer-algorithm and
#: x-amz-server-side-encryption-customer-key.
sse_key_md5 = resource.Header(
'x-amz-server-side-encryption-customer-key-MD5'
)
#: When creating an object, you can add this header in the request
#: to set the storage class of the object. If you do not add this header,
#: the object will use the default storage class of the bucket.
#: Note: The storage class can be STANDARD (OBS Standard),
#: STANDARD_IA (OBS Warm), or GLACIER (OBS Cold).
#: Note that the three storage class values are case-sensitive.
storage_class = resource.Header('x-amz-storage-class')
#: Server name
server = resource.Header('Server', type=str)
#: If a bucket is configured as a website, redirects requests
#: for this object to another object in the same bucket or to
#: an external URL.
#: OBS stores the value of this header in the object metadata.
website_redirect = resource.Header('x-amz-website-redirect-location')
#: Obtains the specified range bytes of an object.
#: The value is a range starting from 0 to maximum object length minus one.
#: If the range is invalid, all object data is returned.
range = resource.Header("range", type=str)
#: Returns the object only if it has been modified since
#: the time specified by this header,
#: otherwise 304 Not Modified is returned.
if_modified_since = resource.Header("if-modified-since", type=str)
#: Returns the object only if it has not been modified since
#: the time specified by this header,
#: otherwise 412 Precondition Failed is returned.
#: http://www.ietf.org/rfc/rfc2616.txt.
if_unmodified_since = resource.Header("if-unmodified-since", type=str)
#: Returns the object only if its ETag is the same
#: as the one specified by this header,
#: otherwise 412 Precondition Failed is returned.
#: http://www.ietf.org/rfc/rfc2616.txt.
if_match = resource.Header("if-match", type=list)
#: Returns the object only if its ETag is different from the one
#: specified by this header,
#: otherwise 304 Not Modified is returned.
if_none_match = resource.Header("if-none-match", type=list)
#: Indicates an origin specified by a pre-request.
#: Generally, it is a domain name
origin = resource.Header("Origin", type=bool)
def __init__(self, data=None, **attrs):
super(_base.BaseResource, self).__init__(**attrs)
self.data = data
def _translate_response(self, response, has_body=True, error_message=None):
"""Given a KSA response, inflate this instance with its data
This method updates attributes that correspond to headers
and body on this instance and clears the dirty set.
"""
exceptions.raise_from_response(response, error_message=response.text)
_logger.debug(response.text)
if response:
if has_body:
# TODO(agoncharov): do nothing so far. Generally need
# to parse different responses
pass
headers = self._consume_header_attrs(response.headers)
self._header.attributes.update(headers)
self._header.clean()
self._update_location()
dict.update(self, self.to_dict())
@classmethod
def list(cls, session, paginated=False,
endpoint_override=None, headers=None, requests_auth=None,
**params):
if not cls.allow_list:
raise exceptions.MethodNotSupported(cls, "list")
cls._query_mapping._validate(params, base_path=cls.base_path)
query_params = cls._query_mapping._transpose(params, cls)
uri = cls.base_path % params
# Build additional arguments to the GET call
get_args = cls._prepare_override_args(
endpoint_override=endpoint_override,
additional_headers=headers)
while uri:
response = session.get(
uri,
params=query_params.copy(),
requests_auth=requests_auth,
**get_args
)
uri = None
next_params = {}
root = ET.fromstring(response.content)
if root.tag != ET.QName(cls.OBS_NS, 'ListBucketResult'):
_logger.warn('Namespace in the response does not match '
'expectation')
cls.OBS_NS = root.tag.split('}', 1)[0][1:]
for element in root:
if element.tag == ET.QName(cls.OBS_NS, cls.resource_key):
# Convert XML part into dict
dict_raw_resource = cls.etree_to_dict(element)
# extract resource data
dict_resource = dict_raw_resource[cls.resource_key]
value = cls.existing(**dict_resource)
yield value
elif element.tag == ET.QName(cls.OBS_NS, 'NextMarker'):
next_params['marker'] = element.text
if 'marker' in next_params:
uri = cls.base_path % params
query_params.update(next_params)
return
def create(self, session, prepend_key=True,
endpoint_override=None, headers=None, requests_auth=None):
if not self.allow_create:
raise exceptions.MethodNotSupported(self, 'create')
session = self._get_session(session)
if not self.content_md5 and self.data:
md5 = hashlib.md5()
md5.update(str.encode(self.data))
self.content_md5 = base64.b64encode(md5.digest()).decode()
request = self._prepare_request(
requires_id=True,
prepend_key=prepend_key)
req_args = self._prepare_override_args(
endpoint_override=endpoint_override,
request_headers=request.headers,
additional_headers=headers,
requests_auth=requests_auth)
response = session.put(
request.url,
data=self.data,
**req_args)
self._translate_response(response)
return self
def download(self, session, filename=None,
endpoint_override=None, requests_auth=None):
session = self._get_session(session)
request = self._prepare_request(requires_id=True)
req_args = self._prepare_override_args(
endpoint_override=endpoint_override,
request_headers=request.headers,
requests_auth=requests_auth)
response = session.get(
request.url,
**req_args)
self._translate_response(response)
_logger.debug(response.content)
with open(filename, 'wb') as f:
f.write(response.content)
return
