def setup_openstacksdk(): auth_plugin = utils.get_auth_plugin('neutron') session = utils.get_keystone_session('neutron', auth_plugin) # NOTE(mdulko): To get rid of warnings about connection pool being full # we need to "tweak" the keystoneauth's adapters increasing # the maximum pool size. for scheme in list(session.session.adapters): session.session.mount(scheme, k_session.TCPKeepAliveAdapter(pool_maxsize=1000)) # TODO(mdulko): To use Neutron's ability to do compare-and-swap updates we # need to manually add support for inserting If-Match header # into requests. At the moment we only need it for ports. # Remove when lower-constraints openstacksdk supports this. os_port.Port.if_match = os_resource.Header('If-Match') # TODO(maysams): We need to manually insert allowed_cidrs option # as it's only supported from 0.41.0 version. Remove it once # lower-constraints supports it. os_listener.Listener.allowed_cidrs = os_resource.Body('allowed_cidrs', type=list) conn = connection.Connection(session=session, region_name=getattr(config.CONF.neutron, 'region_name', None)) conn.network.create_ports = partial(_create_ports, conn.network) conn.network.add_trunk_subports = partial(_add_trunk_subports, conn.network) conn.network.delete_trunk_subports = partial(_delete_trunk_subports, conn.network) _clients[_OPENSTACKSDK] = conn
class Account(_base.BaseResource): _custom_metadata_prefix = "X-Account-Meta-" base_path = "/" allow_fetch = True allow_commit = True allow_head = True #: The total number of bytes that are stored in Object Storage for #: the account. account_bytes_used = resource.Header("x-account-bytes-used", type=int) #: The number of containers. account_container_count = resource.Header("x-account-container-count", type=int) #: The number of objects in the account. account_object_count = resource.Header("x-account-object-count", type=int) #: The secret key value for temporary URLs. If not set, #: this header is not returned by this operation. meta_temp_url_key = resource.Header("x-account-meta-temp-url-key") #: A second secret key value for temporary URLs. If not set, #: this header is not returned by this operation. meta_temp_url_key_2 = resource.Header("x-account-meta-temp-url-key-2") #: The timestamp of the transaction. timestamp = resource.Header("x-timestamp") has_body = False requires_id = False
def setup_openstacksdk(): auth_plugin = utils.get_auth_plugin('neutron') session = utils.get_keystone_session('neutron', auth_plugin) # TODO(mdulko): To use Neutron's ability to do compare-and-swap updates we # need to manually add support for inserting If-Match header # into requests. At the moment we only need it for ports. # Remove when lower-constraints openstacksdk supports this. os_port.Port.if_match = os_resource.Header('If-Match') conn = connection.Connection(session=session, region_name=getattr(config.CONF.neutron, 'region_name', None)) conn.network.create_ports = partial(_create_ports, conn.network) conn.network.add_trunk_subports = partial(_add_trunk_subports, conn.network) conn.network.delete_trunk_subports = partial(_delete_trunk_subports, conn.network) _clients[_OPENSTACKSDK] = conn
class Fake(resource.Resource): resource_key = "resource" resources_key = "resources" base_path = "/fake" allow_create = True allow_fetch = True allow_commit = True allow_delete = True allow_list = True allow_head = True #: The transaction date and time. timestamp = resource.Header("x-timestamp") #: The name of this resource. name = resource.Body("name", alternate_id=True) #: The value of the resource. Also available in headers. value = resource.Body("value", alias="x-resource-value") #: Is this resource cool? If so, set it to True. #: This is a multi-line comment about cool stuff. cool = resource.Body("cool", type=bool)
class Account(_base.BaseResource): _custom_metadata_prefix = "X-Account-Meta-" base_path = "/" allow_fetch = True allow_commit = True allow_head = True #: The total number of bytes that are stored in Object Storage for #: the account. account_bytes_used = resource.Header("x-account-bytes-used", type=int) #: The number of containers. account_container_count = resource.Header("x-account-container-count", type=int) #: The number of objects in the account. account_object_count = resource.Header("x-account-object-count", type=int) #: The secret key value for temporary URLs. If not set, #: this header is not returned by this operation. meta_temp_url_key = resource.Header("x-account-meta-temp-url-key") #: A second secret key value for temporary URLs. If not set, #: this header is not returned by this operation. meta_temp_url_key_2 = resource.Header("x-account-meta-temp-url-key-2") #: The timestamp of the transaction. timestamp = resource.Header("x-timestamp") has_body = False requires_id = False def set_temp_url_key(self, proxy, key, secondary=False): """Set the temporary url key for the account. :param proxy: The proxy to use for making this request. :type proxy: :class:`~openstack.proxy.Proxy` :param key: Text of the key to use. :param bool secondary: Whether this should set the secondary key. (defaults to False) """ header = 'Temp-URL-Key' if secondary: header += '-2' return self.set_metadata(proxy, {header: key})
class Container(_base.BaseResource): _custom_metadata_prefix = "X-Container-Meta-" _system_metadata = { "content_type": "content-type", "is_content_type_detected": "x-detect-content-type", "versions_location": "x-versions-location", "read_ACL": "x-container-read", "write_ACL": "x-container-write", "sync_to": "x-container-sync-to", "sync_key": "x-container-sync-key" } base_path = "/" pagination_key = 'X-Account-Container-Count' allow_create = True allow_fetch = True allow_commit = True allow_delete = True allow_list = True allow_head = True _query_mapping = resource.QueryParameters('prefix', ) # Container body data (when id=None) #: The name of the container. name = resource.Body("name", alternate_id=True, alias='id') #: The number of objects in the container. count = resource.Body("count", type=int, alias='object_count') #: The total number of bytes that are stored in Object Storage #: for the container. bytes = resource.Body("bytes", type=int, alias='bytes_used') # Container metadata (when id=name) #: The number of objects. object_count = resource.Header("x-container-object-count", type=int, alias='count') #: The count of bytes used in total. bytes_used = resource.Header("x-container-bytes-used", type=int, alias='bytes') #: The timestamp of the transaction. timestamp = resource.Header("x-timestamp") # Request headers (when id=None) #: If set to True, Object Storage queries all replicas to return the #: most recent one. If you omit this header, Object Storage responds #: faster after it finds one valid replica. Because setting this #: header to True is more expensive for the back end, use it only #: when it is absolutely needed. *Type: bool* is_newest = resource.Header("x-newest", type=bool) # Request headers (when id=name) #: The ACL that grants read access. If not set, this header is not #: returned by this operation. read_ACL = resource.Header("x-container-read") #: The ACL that grants write access. If not set, this header is not #: returned by this operation. write_ACL = resource.Header("x-container-write") #: The destination for container synchronization. If not set, #: this header is not returned by this operation. sync_to = resource.Header("x-container-sync-to") #: The secret key for container synchronization. If not set, #: this header is not returned by this operation. sync_key = resource.Header("x-container-sync-key") #: Enables versioning on this container. The value is the name #: of another container. You must UTF-8-encode and then URL-encode #: the name before you include it in the header. To disable #: versioning, set the header to an empty string. versions_location = resource.Header("x-versions-location") #: The MIME type of the list of names. content_type = resource.Header("content-type") #: If set to true, Object Storage guesses the content type based #: on the file extension and ignores the value sent in the #: Content-Type header, if present. *Type: bool* is_content_type_detected = resource.Header("x-detect-content-type", type=bool) # TODO(mordred) Shouldn't if-none-match be handled more systemically? #: In combination with Expect: 100-Continue, specify an #: "If-None-Match: \*" header to query whether the server already #: has a copy of the object before any data is sent. if_none_match = resource.Header("if-none-match") @classmethod def new(cls, **kwargs): # Container uses name as id. Proxy._get_resource calls # Resource.new(id=name) but then we need to do container.name # It's the same thing for Container - make it be the same. name = kwargs.pop('id', None) if name: kwargs.setdefault('name', name) return Container(_synchronized=True, **kwargs) def create(self, session, prepend_key=True, base_path=None): """Create a remote resource based on this instance. :param session: The session to use for making this request. :type session: :class:`~keystoneauth1.adapter.Adapter` :param prepend_key: A boolean indicating whether the resource_key should be prepended in a resource creation request. Default to True. :return: This :class:`Resource` instance. :raises: :exc:`~openstack.exceptions.MethodNotSupported` if :data:`Resource.allow_create` is not set to ``True``. """ request = self._prepare_request(requires_id=True, prepend_key=prepend_key, base_path=base_path) response = session.put(request.url, json=request.body, headers=request.headers) self._translate_response(response, has_body=False) return self
class Object(_base.BaseResource): _custom_metadata_prefix = "X-Object-Meta-" _system_metadata = { "content_disposition": "content-disposition", "content_encoding": "content-encoding", "content_type": "content-type", "delete_after": "x-delete-after", "delete_at": "x-delete-at", "is_content_type_detected": "x-detect-content-type", } base_path = "/%(container)s" pagination_key = 'X-Container-Object-Count' allow_create = True allow_fetch = True allow_commit = True allow_delete = True allow_list = True allow_head = True _query_mapping = resource.QueryParameters('prefix', 'format') # Data to be passed during a POST call to create an object on the server. # TODO(mordred) Make a base class BaseDataResource that can be used here # and with glance images that has standard overrides for dealing with # binary data. data = None # URL parameters #: The unique name for the container. container = resource.URI("container") #: The unique name for the object. name = resource.Body("name", alternate_id=True) # Object details # Make these private because they should only matter in the case where # we have a Body with no headers (like if someone programmatically is # creating an Object) _hash = resource.Body("hash") _bytes = resource.Body("bytes", type=int) _last_modified = resource.Body("last_modified") _content_type = resource.Body("content_type") # Headers for HEAD and GET requests #: If set to True, Object Storage queries all replicas to return #: the most recent one. If you omit this header, Object Storage #: responds faster after it finds one valid replica. Because #: setting this header to True is more expensive for the back end, #: use it only when it is absolutely needed. *Type: bool* is_newest = resource.Header("x-newest", type=bool) #: TODO(briancurtin) there's a lot of content here... range = resource.Header("range", type=dict) #: See http://www.ietf.org/rfc/rfc2616.txt. if_match = resource.Header("if-match", type=list) #: In combination with Expect: 100-Continue, specify an #: "If-None-Match: \*" header to query whether the server already #: has a copy of the object before any data is sent. if_none_match = resource.Header("if-none-match", type=list) #: See http://www.ietf.org/rfc/rfc2616.txt. if_modified_since = resource.Header("if-modified-since", type=str) #: See http://www.ietf.org/rfc/rfc2616.txt. if_unmodified_since = resource.Header("if-unmodified-since", type=str) # Query parameters #: Used with temporary URLs to sign the request. For more #: information about temporary URLs, see OpenStack Object Storage #: API v1 Reference. signature = resource.Header("signature") #: Used with temporary URLs to specify the expiry time of the #: signature. For more information about temporary URLs, see #: OpenStack Object Storage API v1 Reference. expires_at = resource.Header("expires") #: If you include the multipart-manifest=get query parameter and #: the object is a large object, the object contents are not #: returned. Instead, the manifest is returned in the #: X-Object-Manifest response header for dynamic large objects #: or in the response body for static large objects. multipart_manifest = resource.Header("multipart-manifest") # Response headers from HEAD and GET #: HEAD operations do not return content. However, in this #: operation the value in the Content-Length header is not the #: size of the response body. Instead it contains the size of #: the object, in bytes. content_length = resource.Header("content-length", type=int, alias='_bytes') #: The MIME type of the object. content_type = resource.Header("content-type", alias="_content_type") #: The type of ranges that the object accepts. accept_ranges = resource.Header("accept-ranges") #: For objects smaller than 5 GB, this value is the MD5 checksum #: of the object content. The value is not quoted. #: For manifest objects, this value is the MD5 checksum of the #: concatenated string of MD5 checksums and ETags for each of #: the segments in the manifest, and not the MD5 checksum of #: the content that was downloaded. Also the value is enclosed #: in double-quote characters. #: You are strongly recommended to compute the MD5 checksum of #: the response body as it is received and compare this value #: with the one in the ETag header. If they differ, the content #: was corrupted, so retry the operation. etag = resource.Header("etag", alias='_hash') #: Set to True if this object is a static large object manifest object. #: *Type: bool* is_static_large_object = resource.Header("x-static-large-object", type=bool) #: If set, the value of the Content-Encoding metadata. #: If not set, this header is not returned by this operation. content_encoding = resource.Header("content-encoding") #: If set, specifies the override behavior for the browser. #: For example, this header might specify that the browser use #: a download program to save this file rather than show the file, #: which is the default. #: If not set, this header is not returned by this operation. content_disposition = resource.Header("content-disposition") #: Specifies the number of seconds after which the object is #: removed. Internally, the Object Storage system stores this #: value in the X-Delete-At metadata item. delete_after = resource.Header("x-delete-after", type=int) #: If set, the time when the object will be deleted by the system #: in the format of a UNIX Epoch timestamp. #: If not set, this header is not returned by this operation. delete_at = resource.Header("x-delete-at") #: If set, to this is a dynamic large object manifest object. #: The value is the container and object name prefix of the #: segment objects in the form container/prefix. object_manifest = resource.Header("x-object-manifest") #: The timestamp of the transaction. timestamp = resource.Header("x-timestamp") #: The date and time that the object was created or the last #: time that the metadata was changed. last_modified_at = resource.Header("last-modified", alias='_last_modified') # Headers for PUT and POST requests #: Set to chunked to enable chunked transfer encoding. If used, #: do not set the Content-Length header to a non-zero value. transfer_encoding = resource.Header("transfer-encoding") #: If set to true, Object Storage guesses the content type based #: on the file extension and ignores the value sent in the #: Content-Type header, if present. *Type: bool* is_content_type_detected = resource.Header("x-detect-content-type", type=bool) #: If set, this is the name of an object used to create the new #: object by copying the X-Copy-From object. The value is in form #: {container}/{object}. You must UTF-8-encode and then URL-encode #: the names of the container and object before you include them #: in the header. #: Using PUT with X-Copy-From has the same effect as using the #: COPY operation to copy an object. copy_from = resource.Header("x-copy-from") has_body = False def __init__(self, data=None, **attrs): super(_base.BaseResource, self).__init__(**attrs) self.data = data # The Object Store treats the metadata for its resources inconsistently so # Object.set_metadata must override the BaseResource.set_metadata to # account for it. def set_metadata(self, session, metadata): # Filter out items with empty values so the create metadata behaviour # is the same as account and container filtered_metadata = \ {key: value for key, value in metadata.items() if value} # Update from remote if we only have locally created information if not self.last_modified_at: self.head(session) # Get a copy of the original metadata so it doesn't get erased on POST # and update it with the new metadata values. metadata = copy.deepcopy(self.metadata) metadata.update(filtered_metadata) # Include any original system metadata so it doesn't get erased on POST for key in self._system_metadata: value = getattr(self, key) if value and key not in metadata: metadata[key] = value request = self._prepare_request() headers = self._calculate_headers(metadata) response = session.post(request.url, headers=headers) self._translate_response(response, has_body=False) self.metadata.update(metadata) return self # The Object Store treats the metadata for its resources inconsistently so # Object.delete_metadata must override the BaseResource.delete_metadata to # account for it. def delete_metadata(self, session, keys): if not keys: return # If we have an empty object, update it from the remote side so that # we have a copy of the original metadata. Deleting metadata requires # POSTing and overwriting all of the metadata. If we already have # metadata locally, assume this is an existing object. if not self.metadata: self.head(session) metadata = copy.deepcopy(self.metadata) # Include any original system metadata so it doesn't get erased on POST for key in self._system_metadata: value = getattr(self, key) if value: metadata[key] = value # Remove the requested metadata keys # TODO(mordred) Why don't we just look at self._header_mapping() # instead of having system_metadata? deleted = False attr_keys_to_delete = set() for key in keys: if key == 'delete_after': del (metadata['delete_at']) else: if key in metadata: del (metadata[key]) # Delete the attribute from the local copy of the object. # Metadata that doesn't have Component attributes is # handled by self.metadata being reset when we run # self.head if hasattr(self, key): attr_keys_to_delete.add(key) deleted = True # Nothing to delete, skip the POST if not deleted: return self request = self._prepare_request() response = session.post(request.url, headers=self._calculate_headers(metadata)) exceptions.raise_from_response( response, error_message="Error deleting metadata keys") # Only delete from local object if the remote delete was successful for key in attr_keys_to_delete: delattr(self, key) # Just update ourselves from remote again. return self.head(session) def _download(self, session, error_message=None, stream=False): request = self._prepare_request() request.headers['Accept'] = 'bytes' response = session.get(request.url, headers=request.headers, stream=stream) exceptions.raise_from_response(response, error_message=error_message) return response def download(self, session, error_message=None): response = self._download(session, error_message=error_message) return response.content def stream(self, session, error_message=None, chunk_size=1024): response = self._download(session, error_message=error_message, stream=True) return response.iter_content(chunk_size, decode_unicode=False) def create(self, session, base_path=None): request = self._prepare_request(base_path=base_path) request.headers['Accept'] = '' response = session.put(request.url, data=self.data, headers=request.headers) self._translate_response(response, has_body=False) return self def _raw_delete(self, session): if not self.allow_delete: raise exceptions.MethodNotSupported(self, "delete") request = self._prepare_request() session = self._get_session(session) microversion = self._get_microversion_for(session, 'delete') if self.is_static_large_object is None: # Fetch metadata to determine SLO flag self.head(session) headers = {'Accept': ""} if self.is_static_large_object: headers['multipart-manifest'] = 'delete' return session.delete(request.url, headers=headers, microversion=microversion)
class Queue(resource.Resource): # FIXME(anyone): The name string of `location` field of Zaqar API response # is lower case. That is inconsistent with the guide from API-WG. This is # a workaround for this issue. location = resource.Header("location") resources_key = "queues" base_path = "/queues" # capabilities allow_create = True allow_list = True allow_fetch = True allow_delete = True # Properties #: The default TTL of messages defined for a queue, which will effect for #: any messages posted to the queue. default_message_ttl = resource.Body("_default_message_ttl") #: Description of the queue. description = resource.Body("description") #: The max post size of messages defined for a queue, which will effect #: for any messages posted to the queue. max_messages_post_size = resource.Body("_max_messages_post_size") #: Name of the queue. The name is the unique identity of a queue. It #: must not exceed 64 bytes in length, and it is limited to US-ASCII #: letters, digits, underscores, and hyphens. name = resource.Body("name", alternate_id=True) #: The ID to identify the client accessing Zaqar API. Must be specified #: in header for each API request. client_id = resource.Header("Client-ID") #: The ID to identify the project accessing Zaqar API. Must be specified #: in case keystone auth is not enabled in Zaqar service. project_id = resource.Header("X-PROJECT-ID") def create(self, session, prepend_key=True, base_path=None): request = self._prepare_request(requires_id=True, prepend_key=prepend_key, base_path=None) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.put(request.url, json=request.body, headers=request.headers) self._translate_response(response, has_body=False) return self @classmethod def list(cls, session, paginated=False, base_path=None, **params): """This method is a generator which yields queue objects. This is almost the copy of list method of resource.Resource class. The only difference is the request header now includes `Client-ID` and `X-PROJECT-ID` fields which are required by Zaqar v2 API. """ more_data = True query_params = cls._query_mapping._transpose(params) if base_path is None: base_path = cls.base_path uri = base_path % params headers = { "Client-ID": params.get('client_id', None) or str(uuid.uuid4()), "X-PROJECT-ID": params.get('project_id', None) or session.get_project_id() } while more_data: resp = session.get(uri, headers=headers, params=query_params) resp = resp.json() resp = resp[cls.resources_key] if not resp: more_data = False yielded = 0 new_marker = None for data in resp: value = cls.existing(**data) new_marker = value.id yielded += 1 yield value if not paginated: return if "limit" in query_params and yielded < query_params["limit"]: return query_params["limit"] = yielded query_params["marker"] = new_marker def fetch(self, session, requires_id=True, base_path=None, error_message=None): request = self._prepare_request(requires_id=requires_id, base_path=base_path) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.get(request.url, headers=headers) self._translate_response(response) return self def delete(self, session): request = self._prepare_request() headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.delete(request.url, headers=headers) self._translate_response(response, has_body=False) return self
class Claim(resource.Resource): # FIXME(anyone): The name string of `location` field of Zaqar API response # is lower case. That is inconsistent with the guide from API-WG. This is # a workaround for this issue. location = resource.Header("location") resources_key = 'claims' base_path = '/queues/%(queue_name)s/claims' service = message_service.MessageService() # capabilities allow_create = True allow_fetch = True allow_commit = True allow_delete = True commit_method = 'PATCH' # Properties #: The value in seconds indicating how long the claim has existed. age = resource.Body("age") #: In case worker stops responding for a long time, the server will #: extend the lifetime of claimed messages to be at least as long as #: the lifetime of the claim itself, plus the specified grace period. #: Must between 60 and 43200 seconds(12 hours). grace = resource.Body("grace") #: The number of messages to claim. Default 10, up to 20. limit = resource.Body("limit") #: Messages have been successfully claimed. messages = resource.Body("messages") #: Number of seconds the server wait before releasing the claim. Must #: between 60 and 43200 seconds(12 hours). ttl = resource.Body("ttl") #: The name of queue to claim message from. queue_name = resource.URI("queue_name") #: The ID to identify the client accessing Zaqar API. Must be specified #: in header for each API request. client_id = resource.Header("Client-ID") #: The ID to identify the project. Must be provided when keystone #: authentication is not enabled in Zaqar service. project_id = resource.Header("X-PROJECT-ID") def _translate_response(self, response, has_body=True): super(Claim, self)._translate_response(response, has_body=has_body) if has_body and self.location: # Extract claim ID from location self.id = self.location.split("claims/")[1] def create(self, session, prepend_key=False): request = self._prepare_request(requires_id=False, prepend_key=prepend_key) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.post(request.url, json=request.body, headers=request.headers) # For case no message was claimed successfully, 204 No Content # message will be returned. In other cases, we translate response # body which has `messages` field(list) included. if response.status_code != 204: self._translate_response(response) return self def fetch(self, session, requires_id=True, error_message=None): request = self._prepare_request(requires_id=requires_id) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.get(request.url, headers=request.headers) self._translate_response(response) return self def commit(self, session, prepend_key=False, has_body=False): request = self._prepare_request(prepend_key=prepend_key) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) session.patch(request.url, json=request.body, headers=request.headers) return self def delete(self, session): request = self._prepare_request() headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.delete(request.url, headers=request.headers) self._translate_response(response, has_body=False) return self
class Object(_base.BaseResource): base_path = '/' allow_create = True allow_get = True allow_commit = True allow_delete = True allow_list = True allow_head = True resources_key = '' resource_key = 'Contents' _query_mapping = resource.QueryParameters('prefix', 'delimiter', 'limit', prefix='prefix', delimiter='delimiter', limit='max-keys') data = None name = resource.Body('Key', alternate_id=True) last_modified = resource.Body('LastModified') etag = resource.Body('ETag') content_length = resource.Body('Size', type=int) storage_class = resource.Body('StorageClass') content_md5 = resource.Header('Content-MD5', type=str) #: private, public-read, public-read-write, authenticated-read #: bucket-owner-read, bucket-owner-full-control acl = resource.Header('x-amz-acl') object_storage_class = resource.Header('x-amz-storage-class') container = resource.URI('container') def __init__(self, data=None, **attrs): super(_base.BaseResource, self).__init__(**attrs) self.data = data def _translate_response(self, response, has_body=True, error_message=None): """Given a KSA response, inflate this instance with its data This method updates attributes that correspond to headers and body on this instance and clears the dirty set. """ exceptions.raise_from_response(response, error_message=response.text) _logger.debug(response.text) if response: if has_body: # TODO(agoncharov): do nothing so far. Generally need # to parse different responses pass @classmethod def list(cls, session, paginated=False, endpoint_override=None, headers=None, requests_auth=None, **params): if not cls.allow_list: raise exceptions.MethodNotSupported(cls, "list") cls._query_mapping._validate(params, base_path=cls.base_path) query_params = cls._query_mapping._transpose(params) uri = cls.base_path % params # Build additional arguments to the GET call get_args = cls._prepare_override_args( endpoint_override=endpoint_override, additional_headers=headers) while uri: response = session.get(uri, params=query_params.copy(), requests_auth=requests_auth, **get_args) uri = None next_params = {} root = ET.fromstring(response.content) if root.tag != ET.QName(cls.OBS_NS, 'ListBucketResult'): _logger.warn('Namespace in the response does not match ' 'expectation') cls.OBS_NS = root.tag.split('}', 1)[0][1:] for element in root: if element.tag == ET.QName(cls.OBS_NS, cls.resource_key): # Convert XML part into dict dict_raw_resource = cls.etree_to_dict(element) # extract resource data dict_resource = dict_raw_resource[cls.resource_key] value = cls.existing(**dict_resource) yield value elif element.tag == ET.QName(cls.OBS_NS, 'NextMarker'): next_params['marker'] = element.text if 'marker' in next_params: uri = cls.base_path % params query_params.update(next_params) return def create(self, session, prepend_key=True, endpoint_override=None, headers=None, requests_auth=None): if not self.allow_create: raise exceptions.MethodNotSupported(self, 'create') session = self._get_session(session) if not self.content_md5 and self.data: md5 = hashlib.md5() md5.update(str.encode(self.data)) self.content_md5 = base64.b64encode(md5.digest()).decode() request = self._prepare_request(requires_id=True, prepend_key=prepend_key) req_args = self._prepare_override_args( endpoint_override=endpoint_override, request_headers=request.headers, additional_headers=headers, requests_auth=requests_auth) response = session.put(request.url, data=self.data, **req_args) self._translate_response(response) return self def download(self, session, filename=None, endpoint_override=None, requests_auth=None): session = self._get_session(session) request = self._prepare_request(requires_id=True) req_args = self._prepare_override_args( endpoint_override=endpoint_override, request_headers=request.headers, requests_auth=requests_auth) response = session.get(request.url, **req_args) self._translate_response(response) _logger.debug(response.content) with open(filename, 'wb') as f: f.write(response.content) return
class Subscription(resource.Resource): # FIXME(anyone): The name string of `location` field of Zaqar API response # is lower case. That is inconsistent with the guide from API-WG. This is # a workaround for this issue. location = resource.Header("location") resources_key = 'subscriptions' base_path = '/queues/%(queue_name)s/subscriptions' # capabilities allow_create = True allow_list = True allow_fetch = True allow_delete = True # Properties #: The value in seconds indicating how long the subscription has existed. age = resource.Body("age") #: Alternate id of the subscription. This key is used in response of #: subscription create API to return id of subscription created. subscription_id = resource.Body("subscription_id", alternate_id=True) #: The extra metadata for the subscription. The value must be a dict. #: If the subscriber is `mailto`. The options can contain `from` and #: `subject` to indicate the email's author and title. options = resource.Body("options", type=dict) #: The queue name which the subscription is registered on. source = resource.Body("source") #: The destination of the message. Two kinds of subscribers are supported: #: http/https and email. The http/https subscriber should start with #: `http/https`. The email subscriber should start with `mailto`. subscriber = resource.Body("subscriber") #: Number of seconds the subscription remains alive? The ttl value must #: be great than 60 seconds. The default value is 3600 seconds. ttl = resource.Body("ttl") #: The queue name which the subscription is registered on. queue_name = resource.URI("queue_name") #: The ID to identify the client accessing Zaqar API. Must be specified #: in header for each API request. client_id = resource.Header("Client-ID") #: The ID to identify the project. Must be provided when keystone #: authentication is not enabled in Zaqar service. project_id = resource.Header("X-PROJECT-ID") def create(self, session, prepend_key=True, base_path=None): request = self._prepare_request(requires_id=False, prepend_key=prepend_key, base_path=base_path) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.post(request.url, json=request.body, headers=request.headers) self._translate_response(response) return self @classmethod def list(cls, session, paginated=True, base_path=None, **params): """This method is a generator which yields subscription objects. This is almost the copy of list method of resource.Resource class. The only difference is the request header now includes `Client-ID` and `X-PROJECT-ID` fields which are required by Zaqar v2 API. """ more_data = True if base_path is None: base_path = cls.base_path uri = base_path % params headers = { "Client-ID": params.get('client_id', None) or str(uuid.uuid4()), "X-PROJECT-ID": params.get('project_id', None) or session.get_project_id() } query_params = cls._query_mapping._transpose(params, cls) while more_data: resp = session.get(uri, headers=headers, params=query_params) resp = resp.json() resp = resp[cls.resources_key] if not resp: more_data = False yielded = 0 new_marker = None for data in resp: value = cls.existing(**data) new_marker = value.id yielded += 1 yield value if not paginated: return if "limit" in query_params and yielded < query_params["limit"]: return query_params["limit"] = yielded query_params["marker"] = new_marker def fetch(self, session, requires_id=True, base_path=None, error_message=None): request = self._prepare_request(requires_id=requires_id, base_path=base_path) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.get(request.url, headers=request.headers) self._translate_response(response) return self def delete(self, session): request = self._prepare_request() headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.delete(request.url, headers=request.headers) self._translate_response(response, has_body=False) return self
class Message(resource.Resource): # FIXME(anyone): The name string of `location` field of Zaqar API response # is lower case. That is inconsistent with the guide from API-WG. This is # a workaround for this issue. location = resource.Header("location") resources_key = 'messages' base_path = '/queues/%(queue_name)s/messages' # capabilities allow_create = True allow_list = True allow_fetch = True allow_delete = True _query_mapping = resource.QueryParameters("echo", "include_claimed") # Properties #: The value in second to specify how long the message has been #: posted to the queue. age = resource.Body("age") #: A dictionary specifies an arbitrary document that constitutes the #: body of the message being sent. body = resource.Body("body") #: An uri string describe the location of the message resource. href = resource.Body("href") #: The value in seconds to specify how long the server waits before #: marking the message as expired and removing it from the queue. ttl = resource.Body("ttl") #: The name of target queue message is post to or got from. queue_name = resource.URI("queue_name") #: The ID to identify the client accessing Zaqar API. Must be specified #: in header for each API request. client_id = resource.Header("Client-ID") #: The ID to identify the project accessing Zaqar API. Must be specified #: in case keystone auth is not enabled in Zaqar service. project_id = resource.Header("X-PROJECT-ID") def post(self, session, messages): request = self._prepare_request(requires_id=False, prepend_key=True) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) request.body = {'messages': messages} response = session.post(request.url, json=request.body, headers=request.headers) return response.json()['resources'] @classmethod def list(cls, session, paginated=True, base_path=None, **params): """This method is a generator which yields message objects. This is almost the copy of list method of resource.Resource class. The only difference is the request header now includes `Client-ID` and `X-PROJECT-ID` fields which are required by Zaqar v2 API. """ more_data = True if base_path is None: base_path = cls.base_path uri = base_path % params headers = { "Client-ID": params.get('client_id', None) or str(uuid.uuid4()), "X-PROJECT-ID": params.get('project_id', None ) or session.get_project_id() } query_params = cls._query_mapping._transpose(params, cls) while more_data: resp = session.get(uri, headers=headers, params=query_params) resp = resp.json() resp = resp[cls.resources_key] if not resp: more_data = False yielded = 0 new_marker = None for data in resp: value = cls.existing(**data) new_marker = value.id yielded += 1 yield value if not paginated: return if "limit" in query_params and yielded < query_params["limit"]: return query_params["limit"] = yielded query_params["marker"] = new_marker def fetch(self, session, requires_id=True, base_path=None, error_message=None): request = self._prepare_request(requires_id=requires_id, base_path=base_path) headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) response = session.get(request.url, headers=headers) self._translate_response(response) return self def delete(self, session): request = self._prepare_request() headers = { "Client-ID": self.client_id or str(uuid.uuid4()), "X-PROJECT-ID": self.project_id or session.get_project_id() } request.headers.update(headers) # For Zaqar v2 API requires client to specify claim_id as query # parameter when deleting a message that has been claimed, we # rebuild the request URI if claim_id is not None. if self.claim_id: request.url += '?claim_id=%s' % self.claim_id response = session.delete(request.url, headers=headers) self._translate_response(response, has_body=False) return self
class Container(_base.BaseResource): resources_key = 'Buckets' resource_key = 'Bucket' allow_get = True allow_head = True allow_list = True allow_create = True allow_delete = True create_method = 'PUT' base_path = '/' # all requests (except create) will default to requires_id = None requires_id = None name = resource.Body('Name', alternate_id=True, alias='id') creation_date = resource.Body('CreationDate') storage_acl = resource.Header('x-amz-acl') storage_class = resource.Header('x-default-storage-class') def _translate_response(self, response, has_body=True, error_message=None): """Given a KSA response, inflate this instance with its data This method updates attributes that correspond to headers and body on this instance and clears the dirty set. """ exceptions.raise_from_response(response, error_message=response.text) if response: if has_body: # TODO(agoncharov): do nothing so far. Generally need # to parse different responses pass def _prepare_request(self, requires_id=None, prepend_key=False): """Prepare a request to be sent to the server Create operations don't require an ID, but all others do, so only try to append an ID when it's needed with requires_id. Create and update operations sometimes require their bodies to be contained within an dict -- if the instance contains a resource_key and prepend_key=True, the body will be wrapped in a dict with that key. Return a _Request object that contains the constructed URI as well a body and headers that are ready to send. Only dirty body and header contents will be returned. """ if requires_id is None: requires_id = self.requires_id body = None # body = self._body.dirty # if prepend_key and self.resource_key is not None: # body = {self.resource_key: body} # if self.name: # body['Bucket'] = self.name base_path = '/' headers = {} uri = base_path % self._uri.attributes if requires_id: if self.id is None: raise exceptions.InvalidRequest( "Request requires an ID but none was found") uri = utils.urljoin(uri, self.id) return resource._Request(uri, body, headers) @classmethod def list(cls, session, paginated=False, endpoint_override=None, headers=None, requests_auth=None, **params): if not cls.allow_list: raise exceptions.MethodNotSupported(cls, "list") cls._query_mapping._validate(params, base_path=cls.base_path) query_params = cls._query_mapping._transpose(params, cls) response = session.get( session.get_endpoint(), params=query_params.copy(), requests_auth=requests_auth ) root = ET.fromstring(response.content) if root.tag != ET.QName(cls.OBS_NS, 'ListAllMyBucketsResult'): _logger.warn('Namespace in the response does not match ' 'expectation') cls.OBS_NS = root.tag.split('}', 1)[0][1:] for elements in root: if elements.tag == ET.QName(cls.OBS_NS, cls.resources_key): for el in elements: if el.tag == ET.QName(cls.OBS_NS, cls.resource_key): # Convert XML part into dict dict_raw_resource = cls.etree_to_dict(el) # extract resource data dict_resource = dict_raw_resource[cls.resource_key] value = cls.existing(**dict_resource) yield value return def create(self, session, prepend_key=True, endpoint_override=None, headers=None, requests_auth=None): if not self.allow_create: raise exceptions.MethodNotSupported(self, "create") session = self._get_session(session) request = self._prepare_request() req_args = self._prepare_override_args( endpoint_override=endpoint_override, request_headers=request.headers, additional_headers=headers, requests_auth=requests_auth) response = session.put(request.url, data=request.body, **req_args) self._translate_response(response) return self
class Object(_base.BaseResource): _custom_metadata_prefix = "x-amz-meta-" base_path = '/' allow_create = True allow_get = True allow_commit = True allow_delete = True allow_list = True allow_head = True resources_key = '' resource_key = 'Contents' _query_mapping = resource.QueryParameters( 'prefix', 'delimiter', 'limit', prefix='prefix', delimiter='delimiter', limit='max-keys' ) # Data to be passed during a POST call to create an object on the server. data = None # URL parameters #: The unique name for the container. container = resource.URI("container") #: The unique name for the object. name = resource.Body('Key', alternate_id=True) #: The date and time that the object was created or the last #: time that the metadata was changed. last_modified = resource.Body('LastModified') #: size of the response body. Instead it contains the size of #: the object, in bytes. content_length = resource.Body('Size', type=int) # Headers for requests #: private, public-read, public-read-write, authenticated-read #: bucket-owner-read, bucket-owner-full-control acl = resource.Header('x-amz-acl') accept_ranges = resource.Header('Accept-Ranges') #: The MD5 digest string of the message body is calculated according #: to the RFC 1864 standard. That is, calculate the 128-bit binary array #: (the message header data encrypted with MD5) first, #: and then use Base 64 encoding to convert the binary data to #: a character string. content_md5 = resource.Header('Content-MD5', type=str) #: Indicates the content type of a requested resource, for example, #: text/plain. content_type = resource.Header('Content-Type', type=str) #: Indicates the hash value of an object. #: The entity tag (ETag) only reflects changes to the contents #: of an object, not its metadata. etag = resource.Header('ETag', type=str) #: Indicates the value created by OBS to uniquely identify a request. #: OBS uses this value to troubleshoot faults. request_id = resource.Header('x-amz-request-id', type=str) #: Indicates a special token that helps OBS troubleshoot faults. request_id_2 = resource.Header('x-amz-id-2', type=str) #: Indicates that SSE-KMS is used. #: Example: x-amz-server-side-encryption:aws:kms sse = resource.Header('x-amz-server-side-encryption') #: Indicates the master key ID. This header is used in SSE-KMS mode. #: If the customer does not provide the master key, #: the default master key will be used. sse_key_id = resource.Header('x-amz-server-side-encryption-aws-kms-key-id') #: Indicates a decryption algorithm. The header is used in SSE-C mode. #: Constraints: This header must be used together with #: x-amz-server-side-encryption-customer-key and #: x-amz-server-side-encryption-customer-key-MD5. sse_algorithm = resource.Header( 'x-amz-server-side-encryption-customer-algorithm' ) #: Indicates a key used to decrypt objects. #: The header is used in SSE-C mode. #: Constraints: This header is a base64-encoded 256-bit or 512-bit key and #: must be used together with # x-amz-server-side-encryption-customer-algorithm and # x-amz-server-side-encryption-customer-key-MD5 sse_key = resource.Header('x-amz-server-side-encryption-customer-key') #: Indicates the MD5 value of a key used to decrypt objects. #: The header is used in SSE-C mode. #: The MD5 value is used to check whether any error #: occurs during the transmission of the key. #: Constraints: This header is a base64-encoded 128-bit MD5 value and #: must be used together with #: x-amz-server-side-encryption-customer-algorithm and #: x-amz-server-side-encryption-customer-key. sse_key_md5 = resource.Header( 'x-amz-server-side-encryption-customer-key-MD5' ) #: When creating an object, you can add this header in the request #: to set the storage class of the object. If you do not add this header, #: the object will use the default storage class of the bucket. #: Note: The storage class can be STANDARD (OBS Standard), #: STANDARD_IA (OBS Warm), or GLACIER (OBS Cold). #: Note that the three storage class values are case-sensitive. storage_class = resource.Header('x-amz-storage-class') #: Server name server = resource.Header('Server', type=str) #: If a bucket is configured as a website, redirects requests #: for this object to another object in the same bucket or to #: an external URL. #: OBS stores the value of this header in the object metadata. website_redirect = resource.Header('x-amz-website-redirect-location') #: Obtains the specified range bytes of an object. #: The value is a range starting from 0 to maximum object length minus one. #: If the range is invalid, all object data is returned. range = resource.Header("range", type=str) #: Returns the object only if it has been modified since #: the time specified by this header, #: otherwise 304 Not Modified is returned. if_modified_since = resource.Header("if-modified-since", type=str) #: Returns the object only if it has not been modified since #: the time specified by this header, #: otherwise 412 Precondition Failed is returned. #: http://www.ietf.org/rfc/rfc2616.txt. if_unmodified_since = resource.Header("if-unmodified-since", type=str) #: Returns the object only if its ETag is the same #: as the one specified by this header, #: otherwise 412 Precondition Failed is returned. #: http://www.ietf.org/rfc/rfc2616.txt. if_match = resource.Header("if-match", type=list) #: Returns the object only if its ETag is different from the one #: specified by this header, #: otherwise 304 Not Modified is returned. if_none_match = resource.Header("if-none-match", type=list) #: Indicates an origin specified by a pre-request. #: Generally, it is a domain name origin = resource.Header("Origin", type=bool) def __init__(self, data=None, **attrs): super(_base.BaseResource, self).__init__(**attrs) self.data = data def _translate_response(self, response, has_body=True, error_message=None): """Given a KSA response, inflate this instance with its data This method updates attributes that correspond to headers and body on this instance and clears the dirty set. """ exceptions.raise_from_response(response, error_message=response.text) _logger.debug(response.text) if response: if has_body: # TODO(agoncharov): do nothing so far. Generally need # to parse different responses pass headers = self._consume_header_attrs(response.headers) self._header.attributes.update(headers) self._header.clean() self._update_location() dict.update(self, self.to_dict()) @classmethod def list(cls, session, paginated=False, endpoint_override=None, headers=None, requests_auth=None, **params): if not cls.allow_list: raise exceptions.MethodNotSupported(cls, "list") cls._query_mapping._validate(params, base_path=cls.base_path) query_params = cls._query_mapping._transpose(params, cls) uri = cls.base_path % params # Build additional arguments to the GET call get_args = cls._prepare_override_args( endpoint_override=endpoint_override, additional_headers=headers) while uri: response = session.get( uri, params=query_params.copy(), requests_auth=requests_auth, **get_args ) uri = None next_params = {} root = ET.fromstring(response.content) if root.tag != ET.QName(cls.OBS_NS, 'ListBucketResult'): _logger.warn('Namespace in the response does not match ' 'expectation') cls.OBS_NS = root.tag.split('}', 1)[0][1:] for element in root: if element.tag == ET.QName(cls.OBS_NS, cls.resource_key): # Convert XML part into dict dict_raw_resource = cls.etree_to_dict(element) # extract resource data dict_resource = dict_raw_resource[cls.resource_key] value = cls.existing(**dict_resource) yield value elif element.tag == ET.QName(cls.OBS_NS, 'NextMarker'): next_params['marker'] = element.text if 'marker' in next_params: uri = cls.base_path % params query_params.update(next_params) return def create(self, session, prepend_key=True, endpoint_override=None, headers=None, requests_auth=None): if not self.allow_create: raise exceptions.MethodNotSupported(self, 'create') session = self._get_session(session) if not self.content_md5 and self.data: md5 = hashlib.md5() md5.update(str.encode(self.data)) self.content_md5 = base64.b64encode(md5.digest()).decode() request = self._prepare_request( requires_id=True, prepend_key=prepend_key) req_args = self._prepare_override_args( endpoint_override=endpoint_override, request_headers=request.headers, additional_headers=headers, requests_auth=requests_auth) response = session.put( request.url, data=self.data, **req_args) self._translate_response(response) return self def download(self, session, filename=None, endpoint_override=None, requests_auth=None): session = self._get_session(session) request = self._prepare_request(requires_id=True) req_args = self._prepare_override_args( endpoint_override=endpoint_override, request_headers=request.headers, requests_auth=requests_auth) response = session.get( request.url, **req_args) self._translate_response(response) _logger.debug(response.content) with open(filename, 'wb') as f: f.write(response.content) return