def test_create_instance_when_not_exists(self, mock_get_conn, mock_project_id):
mock_get_conn.return_value.get_instance.side_effect = [
NotFound("Instance not found"),
cloud_memcache.Instance(name=TEST_NAME),
]
mock_get_conn.return_value.create_instance.return_value.result.return_value = cloud_memcache.Instance(
name=TEST_NAME
)
result = self.hook.create_instance( # pylint: disable=no-value-for-parameter
location=TEST_LOCATION,
instance_id=TEST_INSTANCE_ID,
instance=cloud_memcache.Instance(name=TEST_NAME),
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
)
mock_get_conn.return_value.get_instance.has_calls(
[
mock.call(name=TEST_NAME, retry=TEST_RETRY, timeout=TEST_TIMEOUT, metadata=TEST_METADATA),
mock.call(name=TEST_NAME, retry=TEST_RETRY, timeout=TEST_TIMEOUT, metadata=TEST_METADATA),
]
)
mock_get_conn.return_value.create_instance.assert_called_once_with(
resource=cloud_memcache.Instance(
name=TEST_NAME,
labels={"airflow-version": "v" + version.version.replace(".", "-").replace("+", "-")},
),
instance_id=TEST_INSTANCE_ID,
metadata=TEST_METADATA,
parent=TEST_PARENT_DEFAULT_PROJECT_ID,
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
)
self.assertEqual(cloud_memcache.Instance(name=TEST_NAME), result)
def test_create_instance_flattened():
client = CloudMemcacheClient(
credentials=credentials.AnonymousCredentials())
# Mock the actual call within the gRPC stub, and fake the request.
with mock.patch.object(type(client._transport.create_instance),
"__call__") as call:
# Designate an appropriate return value for the call.
call.return_value = operations_pb2.Operation(name="operations/op")
# Call the method with a truthy value for each flattened field,
# using the keyword arguments to the method.
response = client.create_instance(
parent="parent_value",
instance_id="instance_id_value",
resource=cloud_memcache.Instance(name="name_value"),
)
# Establish that the underlying call was made with the expected
# request object values.
assert len(call.mock_calls) == 1
_, args, _ = call.mock_calls[0]
assert args[0].parent == "parent_value"
assert args[0].instance_id == "instance_id_value"
assert args[0].resource == cloud_memcache.Instance(name="name_value")
def test_create_instance_when_exists(self, mock_get_conn, mock_project_id):
mock_get_conn.return_value.get_instance.return_value = cloud_memcache.Instance(name=TEST_NAME)
result = self.hook.create_instance( # pylint: disable=no-value-for-parameter
location=TEST_LOCATION,
instance_id=TEST_INSTANCE_ID,
instance=cloud_memcache.Instance(name=TEST_NAME),
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
)
mock_get_conn.return_value.get_instance.assert_called_once_with(
name=TEST_NAME_DEFAULT_PROJECT_ID, retry=TEST_RETRY, timeout=TEST_TIMEOUT, metadata=TEST_METADATA
)
self.assertEqual(cloud_memcache.Instance(name=TEST_NAME), result)
def test_assert_valid_hook_call(self, mock_hook):
mock_hook.return_value.get_instance.return_value = cloud_memcache.Instance(
)
task = CloudMemorystoreMemcachedGetInstanceOperator(
task_id=TEST_TASK_ID,
location=TEST_LOCATION,
instance=TEST_INSTANCE_NAME,
project_id=TEST_PROJECT_ID,
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
gcp_conn_id=TEST_GCP_CONN_ID,
impersonation_chain=TEST_IMPERSONATION_CHAIN,
)
task.execute(mock.MagicMock())
mock_hook.assert_called_once_with(
gcp_conn_id=TEST_GCP_CONN_ID,
impersonation_chain=TEST_IMPERSONATION_CHAIN,
)
mock_hook.return_value.get_instance.assert_called_once_with(
location=TEST_LOCATION,
instance=TEST_INSTANCE_NAME,
project_id=TEST_PROJECT_ID,
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
)
def test_update_instance(self, mock_get_conn, mock_project_id):
self.hook.update_instance( # pylint: disable=no-value-for-parameter
update_mask=TEST_UPDATE_MASK_MEMCACHED,
instance=cloud_memcache.Instance(name=TEST_NAME),
location=TEST_LOCATION,
instance_id=TEST_INSTANCE_ID,
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
)
mock_get_conn.return_value.update_instance.assert_called_once_with(
update_mask=TEST_UPDATE_MASK_MEMCACHED,
resource=cloud_memcache.Instance(name=TEST_NAME_DEFAULT_PROJECT_ID),
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
)
def update_instance(
self,
update_mask: Union[Dict, FieldMask],
instance: Union[Dict, cloud_memcache.Instance],
project_id: str,
location: Optional[str] = None,
instance_id: Optional[str] = None,
retry: Optional[Retry] = None,
timeout: Optional[float] = None,
metadata: Sequence[Tuple[str, str]] = (),
):
"""
Updates the metadata and configuration of a specific Memcached instance.
:param update_mask: Required. Mask of fields to update. At least one path must be supplied in this
field. The elements of the repeated paths field may only include these fields from ``Instance``:
- ``displayName``
If a dict is provided, it must be of the same form as the protobuf message
:class:`~google.protobuf.field_mask_pb2.FieldMask`)
Union[Dict, google.protobuf.field_mask_pb2.FieldMask]
:param instance: Required. Update description. Only fields specified in ``update_mask`` are updated.
If a dict is provided, it must be of the same form as the protobuf message
:class:`~google.cloud.memcache_v1beta2.types.cloud_memcache.Instance`
:param location: The location of the Cloud Memorystore instance (for example europe-west1)
:param instance_id: The logical name of the Memcached instance in the customer project.
:param project_id: Project ID of the project that contains the instance. If set
to None or missing, the default project_id from the Google Cloud connection is used.
:param retry: A retry object used to retry requests. If ``None`` is specified, requests will not be
retried.
:param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
``retry`` is specified, the timeout applies to each individual attempt.
:param metadata: Additional metadata that is provided to the method.
"""
client = self.get_conn()
metadata = metadata or ()
if isinstance(instance, dict):
instance = cloud_memcache.Instance(instance)
elif not isinstance(instance, cloud_memcache.Instance):
raise AirflowException(
"instance is not instance of Instance type or python dict")
if location and instance_id:
name = CloudMemcacheClient.instance_path(project_id, location,
instance_id)
instance.name = name
self.log.info("Updating instances: %s", instance.name)
result = client.update_instance(update_mask=update_mask,
resource=instance,
retry=retry,
timeout=timeout,
metadata=metadata or ())
result.result()
self.log.info("Instance updated: %s", instance.name)
def test_update_instance_flattened_error():
client = CloudMemcacheClient(credentials=credentials.AnonymousCredentials())
# Attempting to call a method with both a request object and flattened
# fields is an error.
with pytest.raises(ValueError):
client.update_instance(
cloud_memcache.UpdateInstanceRequest(),
update_mask=field_mask.FieldMask(paths=["paths_value"]),
resource=cloud_memcache.Instance(name="name_value"),
)
def test_create_instance_flattened_error():
client = CloudMemcacheClient(credentials=credentials.AnonymousCredentials())
# Attempting to call a method with both a request object and flattened
# fields is an error.
with pytest.raises(ValueError):
client.create_instance(
cloud_memcache.CreateInstanceRequest(),
parent="parent_value",
instance_id="instance_id_value",
resource=cloud_memcache.Instance(name="name_value"),
)
def test_list_instances_pages():
client = CloudMemcacheClient(credentials=credentials.AnonymousCredentials)
# Mock the actual call within the gRPC stub, and fake the request.
with mock.patch.object(type(client._transport.list_instances), "__call__") as call:
# Set the response to a series of pages.
call.side_effect = (
cloud_memcache.ListInstancesResponse(
resources=[
cloud_memcache.Instance(),
cloud_memcache.Instance(),
cloud_memcache.Instance(),
],
next_page_token="abc",
),
cloud_memcache.ListInstancesResponse(resources=[], next_page_token="def"),
cloud_memcache.ListInstancesResponse(
resources=[cloud_memcache.Instance()], next_page_token="ghi"
),
cloud_memcache.ListInstancesResponse(
resources=[cloud_memcache.Instance(), cloud_memcache.Instance()]
),
RuntimeError,
)
pages = list(client.list_instances(request={}).pages)
for page, token in zip(pages, ["abc", "def", "ghi", ""]):
assert page.raw_page.next_page_token == token
def test_list_instances_pager():
client = CloudMemcacheClient(credentials=credentials.AnonymousCredentials)
# Mock the actual call within the gRPC stub, and fake the request.
with mock.patch.object(type(client._transport.list_instances), "__call__") as call:
# Set the response to a series of pages.
call.side_effect = (
cloud_memcache.ListInstancesResponse(
resources=[
cloud_memcache.Instance(),
cloud_memcache.Instance(),
cloud_memcache.Instance(),
],
next_page_token="abc",
),
cloud_memcache.ListInstancesResponse(resources=[], next_page_token="def"),
cloud_memcache.ListInstancesResponse(
resources=[cloud_memcache.Instance()], next_page_token="ghi"
),
cloud_memcache.ListInstancesResponse(
resources=[cloud_memcache.Instance(), cloud_memcache.Instance()]
),
RuntimeError,
)
results = [i for i in client.list_instances(request={})]
assert len(results) == 6
assert all(isinstance(i, cloud_memcache.Instance) for i in results)
def test_proto_functions(self):
instance_dict = {
'name': 'test_name',
'node_count': 1,
'node_config': {'cpu_count': 1, 'memory_size_mb': 1024},
}
instance = cloud_memcache.Instance(instance_dict)
instance_dict_result = self.hook.proto_message_to_dict(instance)
self.assertEqual(instance_dict_result["name"], instance_dict["name"])
self.assertEqual(
instance_dict_result["nodeConfig"]["cpuCount"], instance_dict["node_config"]["cpu_count"]
)
self.assertEqual(
instance_dict_result["nodeConfig"]["memorySizeMb"], instance_dict["node_config"]["memory_size_mb"]
)
def test_get_instance(transport: str = "grpc"):
client = CloudMemcacheClient(
credentials=credentials.AnonymousCredentials(), transport=transport
)
# Everything is optional in proto3 as far as the runtime is concerned,
# and we are mocking out the actual API, so just send an empty request.
request = cloud_memcache.GetInstanceRequest()
# Mock the actual call within the gRPC stub, and fake the request.
with mock.patch.object(type(client._transport.get_instance), "__call__") as call:
# Designate an appropriate return value for the call.
call.return_value = cloud_memcache.Instance(
name="name_value",
display_name="display_name_value",
authorized_network="authorized_network_value",
zones=["zones_value"],
node_count=1070,
memcache_version=cloud_memcache.MemcacheVersion.MEMCACHE_1_5,
state=cloud_memcache.Instance.State.CREATING,
memcache_full_version="memcache_full_version_value",
discovery_endpoint="discovery_endpoint_value",
)
response = client.get_instance(request)
# Establish that the underlying gRPC stub method was called.
assert len(call.mock_calls) == 1
_, args, _ = call.mock_calls[0]
assert args[0] == request
# Establish that the response is the type that we expect.
assert isinstance(response, cloud_memcache.Instance)
assert response.name == "name_value"
assert response.display_name == "display_name_value"
assert response.authorized_network == "authorized_network_value"
assert response.zones == ["zones_value"]
assert response.node_count == 1070
assert response.memcache_version == cloud_memcache.MemcacheVersion.MEMCACHE_1_5
assert response.state == cloud_memcache.Instance.State.CREATING
assert response.memcache_full_version == "memcache_full_version_value"
assert response.discovery_endpoint == "discovery_endpoint_value"
def test_get_instance_field_headers():
client = CloudMemcacheClient(credentials=credentials.AnonymousCredentials())
# Any value that is part of the HTTP/1.1 URI should be sent as
# a field header. Set these to a non-empty value.
request = cloud_memcache.GetInstanceRequest(name="name/value")
# Mock the actual call within the gRPC stub, and fake the request.
with mock.patch.object(type(client._transport.get_instance), "__call__") as call:
call.return_value = cloud_memcache.Instance()
client.get_instance(request)
# Establish that the underlying gRPC stub method was called.
assert len(call.mock_calls) == 1
_, args, _ = call.mock_calls[0]
assert args[0] == request
# Establish that the field header was sent.
_, _, kw = call.mock_calls[0]
assert ("x-goog-request-params", "name=name/value") in kw["metadata"]
def create_instance(
self,
location: str,
instance_id: str,
instance: Union[Dict, cloud_memcache.Instance],
project_id: str,
retry: Optional[Retry] = None,
timeout: Optional[float] = None,
metadata: Optional[Sequence[Tuple[str, str]]] = None,
):
"""
Creates a Memcached instance based on the specified tier and memory size.
By default, the instance is accessible from the project's `default network
<https://cloud.google.com/compute/docs/networks-and-firewalls#networks>`__.
:param location: The location of the Cloud Memorystore instance (for example europe-west1)
:type location: str
:param instance_id: Required. The logical name of the Memcached instance in the customer project
with the following restrictions:
- Must contain only lowercase letters, numbers, and hyphens.
- Must start with a letter.
- Must be between 1-40 characters.
- Must end with a number or a letter.
- Must be unique within the customer project / location
:type instance_id: str
:param instance: Required. A Memcached [Instance] resource
If a dict is provided, it must be of the same form as the protobuf message
:class:`~google.cloud.memcache_v1beta2.types.cloud_memcache.Instance`
:type instance: Union[Dict, google.cloud.memcache_v1beta2.types.cloud_memcache.Instance]
:param project_id: Project ID of the project that contains the instance. If set
to None or missing, the default project_id from the GCP connection is used.
:type project_id: str
:param retry: A retry object used to retry requests. If ``None`` is specified, requests will not be
retried.
:type retry: google.api_core.retry.Retry
:param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
``retry`` is specified, the timeout applies to each individual attempt.
:type timeout: float
:param metadata: Additional metadata that is provided to the method.
:type metadata: Sequence[Tuple[str, str]]
"""
client = self.get_conn()
metadata = metadata or ()
parent = path_template.expand(
"projects/{project}/locations/{location}",
project=project_id,
location=location)
instance_name = CloudMemcacheClient.instance_path(
project_id, location, instance_id)
try:
instance = client.get_instance(name=instance_name,
retry=retry,
timeout=timeout,
metadata=metadata)
self.log.info("Instance exists. Skipping creation.")
return instance
except NotFound:
self.log.info("Instance not exists.")
if isinstance(instance, dict):
instance = cloud_memcache.Instance(instance)
elif not isinstance(instance, cloud_memcache.Instance):
raise AirflowException(
"instance is not instance of Instance type or python dict")
self._append_label(instance, "airflow-version", "v" + version.version)
result = client.create_instance(
parent=parent,
instance_id=instance_id,
resource=instance,
retry=retry,
timeout=timeout,
metadata=metadata or (),
)
result.result()
self.log.info("Instance created.")
return client.get_instance(
name=instance_name,
retry=retry,
timeout=timeout,
metadata=metadata or (),
)
