def _parse_question(self, question):
"""Parse a question in the Google Cloud Pub/Sub or Google Cloud Run format.
:param dict|Message question:
:return (dict, str, bool, packaging.version.Version|None):
"""
try:
# Parse question directly from Pub/Sub or Dataflow.
data = json.loads(question.data.decode())
# Acknowledge it if it's directly from Pub/Sub
if hasattr(question, "ack"):
question.ack()
except Exception:
# Parse question from Google Cloud Run.
data = json.loads(base64.b64decode(question["data"]).decode("utf-8").strip())
logger.info("%r received a question.", self)
question_uuid = get_nested_attribute(question, "attributes.question_uuid")
forward_logs = bool(int(get_nested_attribute(question, "attributes.forward_logs")))
try:
parent_sdk_version = get_nested_attribute(question, "attributes.octue_sdk_version")
except AttributeError:
parent_sdk_version = None
if parent_sdk_version:
parent_sdk_version = packaging.version.parse(parent_sdk_version)
return data, question_uuid, forward_logs, parent_sdk_version
def satisfies(self, raise_error_if_filter_is_invalid=True, **kwargs):
"""Check that the instance satisfies the given filter for the given filter value. The filter should be provided
as a single keyword argument such as `name__first__equals="Joe"`
:param bool raise_error_if_filter_is_invalid:
:param {str: any} kwargs: a single keyword argument whose key is the name of the filter and whose value is the value to filter for
:raise ValueError: if more than one keyword argument is received
:raise AttributeError: if the instance doesn't have the attribute specified in the filter name (only if `raise_error_if_filter_is_invalid` is `True`)
:raise octue.exceptions.InvalidInputException: if the filter name is invalid or there are no filters for the type of attribute specified in the filter name
:return mixed:
"""
if len(kwargs) != 1:
raise ValueError(f"The satisfies method only takes one keyword argument; received {kwargs!r}.")
filter_name, filter_value = list(kwargs.items())[0]
try:
attribute_name, filter_action = self._split_filter_name(filter_name)
try:
attribute = get_nested_attribute(self, attribute_name)
except AttributeError as error:
if raise_error_if_filter_is_invalid:
raise error
return False
filter_ = self._get_filter(attribute, filter_action)
return filter_(attribute, filter_value)
except exceptions.InvalidInputException as error:
return self._try_equals_filter_shortcut(filter_name, filter_value, error)
def order_by(self, attribute_name, check_start_value=None, check_constant_increment=None, reverse=False):
"""Order the `Filterable`s in the container by an attribute with the given name, returning them as a new
`FilterList` regardless of the type of filter container begun with (`FilterSet`s and `FilterDict`s are
inherently orderless).
:param str attribute_name: name of attribute (optionally nested) to order by e.g. "a", "a.b", "a.b.c"
:param any check_start_value: if provided, check that the first item in the ordered container has the given start value for the attribute ordered by
:param int|float|None check_constant_increment: if given, check that the ordered-by attribute of each of the items in the ordered container increases by the given value when progressing along the sequence
:param bool reverse: if True, reverse the ordering
:raise octue.exceptions.InvalidInputException: if an attribute with the given name doesn't exist on any of the container's members
:return FilterList:
"""
attribute_name = ".".join(attribute_name.split("__"))
try:
results = FilterList(
sorted(self, key=lambda item: get_nested_attribute(item, attribute_name), reverse=reverse)
)
except AttributeError:
raise exceptions.InvalidInputException(
f"An attribute named {attribute_name!r} does not exist on one or more members of {self!r}."
)
if check_start_value is not None:
if get_nested_attribute(results[0], attribute_name) != check_start_value:
raise exceptions.BrokenSequenceException(
f"The attribute {attribute_name!r} of the first item of {results!r} does equal the given start "
f"value {check_start_value!r}."
)
if check_constant_increment is not None:
required_increment = check_constant_increment
for i in range(len(results) - 1):
actual_increment = get_nested_attribute(results[i + 1], attribute_name) - get_nested_attribute(
results[i], attribute_name
)
if actual_increment != required_increment:
raise exceptions.BrokenSequenceException(
f"The attributes {attribute_name!r} of the items of {results!r} do not increase by a constant "
f"increment of {required_increment}."
)
return results
def order_by(self, attribute_name, reverse=False):
"""Order the instance by the given attribute_name, returning the instance's elements as a new FilterList.
:param str attribute_name: name of attribute (optionally nested) to order by e.g. "a", "a.b", "a.b.c"
:param bool reverse: if True, reverse the ordering
:raise octue.exceptions.InvalidInputException: if an attribute with the given name doesn't exist on any of the FilterDict's values
:return FilterList:
"""
try:
return FilterList(
sorted(self.items(), key=lambda item: get_nested_attribute(item[1], attribute_name), reverse=reverse)
)
except AttributeError:
raise exceptions.InvalidInputException(
f"An attribute named {attribute_name!r} does not exist on one or more members of {self!r}."
)
def answer_question(question, project_name):
"""Answer a question sent to an app deployed in Google Cloud.
:param dict|tuple|apache_beam.io.gcp.pubsub.PubsubMessage question:
:param str project_name:
:return None:
"""
service_configuration, app_configuration = load_service_and_app_configuration(
DEFAULT_SERVICE_CONFIGURATION_PATH)
service_id = os.environ.get(
"SERVICE_ID") or service_configuration.service_id
service = Service(
service_id=service_id,
backend=GCPPubSubBackend(project_name=project_name),
)
question_uuid = get_nested_attribute(question, "attributes.question_uuid")
answer_topic = service.instantiate_answer_topic(question_uuid)
try:
service.name = service_configuration.name
runner = Runner(
app_src=service_configuration.app_source_path,
twine=service_configuration.twine_path,
configuration_values=app_configuration.configuration_values,
configuration_manifest=app_configuration.configuration_manifest,
children=app_configuration.children,
output_location=app_configuration.output_location,
project_name=project_name,
service_id=service_id,
)
service.run_function = functools.partial(runner.run)
service.answer(question, answer_topic=answer_topic)
logger.info(
"Analysis successfully run and response sent for question %r.",
question_uuid)
# Forward any errors in the deployment configuration (errors in the analysis are already forwarded by the service).
except BaseException as error: # noqa
service.send_exception(topic=answer_topic)
logger.exception(error)
def _try_equals_filter_shortcut(self, filter_name, filter_value, error):
"""Try to use the equals filter shortcut e.g. `a=7` instead of `a__equals=7` or `a__b=7` instead of
`a__b__equals=7`. Raise the error if this is not applicable (e.g. the filter name is just wrong).
:param str filter_name:
:param mixed filter_value:
:param Exception error: the error to raise if the equals filter shortcut is not applicable
:raise Exception: if the equals filter shortcut is not applicable
:return mixed:
"""
possible_attribute_name = ".".join(filter_name.split("__"))
if has_nested_attribute(self, possible_attribute_name):
attribute = get_nested_attribute(self, possible_attribute_name)
filter_ = self._get_filter(attribute, "equals")
return filter_(attribute, filter_value)
raise error
def test_get_nested_dictionary_attribute(self):
"""Test that nested attributes ending in a dictionary key can be accessed."""
inner_mock = Mock(b={"hello": "world"})
outer_mock = Mock(a=inner_mock)
self.assertEqual(get_nested_attribute(instance=outer_mock, nested_attribute_name="a.b.hello"), "world")
def test_get_nested_attribute(self):
"""Test that nested attributes can be accessed."""
inner_mock = Mock(b=3)
outer_mock = Mock(a=inner_mock)
self.assertEqual(get_nested_attribute(instance=outer_mock, nested_attribute_name="a.b"), 3)