def test_get_value():
l = [1, 2, 3]
assert utils.get_value(1, l) == 2
class MyInt(int):
pass
assert utils.get_value(MyInt(1), l) == 2
def get_attribute(self, obj, attr, default):
if attr == "sla_miss":
# Object is a tuple of task_instance and slamiss
# and the get_value expects a dict with key, value
# corresponding to the attr.
slamiss_instance = {"sla_miss": obj[1]}
return get_value(slamiss_instance, attr, default)
return get_value(obj[0], attr, default)
def test_get_value():
lst = [1, 2, 3]
assert utils.get_value(lst, 1) == 2
class MyInt(int):
pass
assert utils.get_value(lst, MyInt(1)) == 2
def test_get_value():
l = [1,2,3]
assert utils.get_value(1, l) == 2
class MyInt(int):
pass
assert utils.get_value(MyInt(1), l) == 2
def dumpCards(self, data, **kwargs):
def getString(obj):
if isinstance(obj, str):
return obj
return utils.get_value(obj, "raw")
return {
"items": list(map(getString, utils.get_value(data, "items"))),
"collection": utils.get_value(data, "collection"),
"collection_scale": utils.get_value(data, "collection_scale")
}
def get_resource_linkage(self, value):
if self.many:
resource_object = [{
'type': self.type_,
'id': _stringify(get_value(self.id_field, each, each))
} for each in value]
else:
resource_object = {
'type': self.type_,
'id': _stringify(get_value(self.id_field, value, value))
}
return resource_object
def get_resource_linkage(self, value):
if self.many:
resource_object = [{
'type':
self.type_,
'id':
_stringify(get_value(self.id_field, each, each))
} for each in value]
else:
resource_object = {
'type': self.type_,
'id': _stringify(get_value(self.id_field, value, value))
}
return resource_object
def _serialize(self, value, key, obj):
"""Output the URL for the endpoint, given the kwargs passed to
``__init__``.
"""
param_values = {}
for name, attr_tpl in iteritems(self.params):
attr_name = _tpl(str(attr_tpl))
if attr_name:
attribute_value = utils.get_value(attr_name,
obj,
default=missing)
if attribute_value is not missing:
param_values[name] = attribute_value
else:
err = AttributeError('{attr_name!r} is not a valid '
'attribute of {obj!r}'.format(
attr_name=attr_name, obj=obj))
if has_forced_error:
raise ForcedError(err)
else:
raise err
else:
param_values[name] = attr_tpl
try:
return url_for(self.endpoint, **param_values)
except BuildError as err: # Make sure BuildErrors are raised
if has_forced_error:
raise ForcedError(err)
else:
raise err
def _serialize(self, value, key, obj):
"""Output the URL for the endpoint, given the kwargs passed to
``__init__``.
"""
param_values = {}
for name, attr_tpl in iteritems(self.params):
attr_name = _tpl(str(attr_tpl))
if attr_name:
attribute_value = utils.get_value(attr_name, obj, default=missing)
if attribute_value is not missing:
param_values[name] = attribute_value
else:
err = AttributeError(
'{attr_name!r} is not a valid '
'attribute of {obj!r}'.format(attr_name=attr_name, obj=obj)
)
if has_forced_error:
raise ForcedError(err)
else:
raise err
else:
param_values[name] = attr_tpl
try:
return url_for(self.endpoint, **param_values)
except BuildError as err: # Make sure BuildErrors are raised
if has_forced_error:
raise ForcedError(err)
else:
raise err
def get_attribute(self, obj, attr, default):
"""Overwritten marshmallow function"""
task_instance_attr = ['task_id', 'execution_date', 'dag_id']
if attr in task_instance_attr:
obj = obj[0] # As object is a tuple of task_instance and dag_run_id
return get_value(obj, attr, default)
return obj[1]
def get_attribute(self, attr, obj, default):
field = self.fields.get(attr)
if field:
metadata = field.metadata
if (metadata and metadata.get('is_multidict', False) and
isinstance(obj, dict) and hasattr(obj, 'getlist')):
return obj.getlist(attr)
return get_value(attr, obj, default)
def add_resource_linkage(self, value):
def stringify(value):
if value is not None:
return str(value)
return value
if self.many:
included_data = [{
'type': self.type_,
'id': stringify(get_value(self.id_field, each, each))
} for each in value]
else:
included_data = {
'type': self.type_,
'id': stringify(get_value(self.id_field, value, value))
}
return included_data
def get_parameters(obj: prefect.Flow, context: dict) -> Set:
if isinstance(obj, prefect.Flow):
return {
p
for p in obj.tasks if isinstance(p, prefect.core.task.Parameter)
}
else:
return utils.get_value(obj, "parameters")
def get_querystring_params(event, querystring_schema):
qs_schema = querystring_schema()
qs_params = get_value(event, 'queryStringParameters')
try:
params = {'params': qs_schema.load(qs_params)}
except marshmallow.ValidationError as e:
raise exceptions.UnprocessableException(e.messages)
return params
def get_attribute(self, obj: typing.Any, attr: str, default: typing.Any):
"""Defines how to pull values from an object to serialize.
.. versionadded:: 2.0.0
.. versionchanged:: 3.0.0a1
Changed position of ``obj`` and ``attr``.
"""
return get_value(obj, attr, default)
def get_tasks(obj: prefect.Flow, context: dict) -> List:
if isinstance(obj, prefect.Flow):
tasks = obj.tasks
else:
tasks = utils.get_value(obj, "tasks")
if tasks is missing:
return tasks
return list(sorted(tasks, key=lambda t: t.slug))
def get_parameters(obj: prefect.Flow, context: dict) -> List:
if isinstance(obj, prefect.Flow):
params = obj.parameters()
else:
params = utils.get_value(obj, "parameters")
if params is missing:
return params
return sorted(params, key=lambda p: p.slug)
def get_min_max(sender, path):
# NOTE can't filter for project when using wildcard index data.$**
# https://docs.mongodb.com/manual/core/index-wildcard/#wildcard-index-query-sort-support
field = f"{path}{delimiter}value"
key = f"{field}__type".replace(delimiter, "__")
q = {key: "number"} # NOTE need a query to trigger wildcard IXSCAN
qs = sender.objects(**q).only(field).order_by(field)
values = [get_value(doc, field) for doc in qs]
values = [v for v in values if not isinstance(v, _Missing)]
return (values[0], values[-1]) if len(values) else (None, None)
def get_url_with_params(api_url, event, path_param_schema):
param_schema = path_param_schema()
event_params = get_value(event, 'pathParameters', {})
try:
params_data = param_schema.load(event_params)
except marshmallow.ValidationError as e:
raise exceptions.UnprocessableException(e.messages)
for param, value in params_data.items():
api_url = api_url.replace(f'{{{param}}}', value)
return api_url
def _render_link_tpl(self, obj, context):
link = self.link_tpl
for bound_attr in set(re.findall(r'<\w*>', link)):
attr_name = bound_attr.strip('<>')
value = get_value(attr_name, obj)
if value is missing:
value = context.get(attr_name, missing)
if value is missing:
raise AttributeError(
'{} is not a valid attribute of object: {},'
' context: {}'.format(bound_attr, obj, context))
link = link.replace(bound_attr, str(value))
return link
def get_attribute(self, obj, attr, default):
"""Override how attributes are retrieved when dumping.
NOTE: We have to access by attribute because although we are loading
from an external pure dict, but we are dumping from a data-layer
object whose fields should be accessed by attributes and not
keys. Access by key runs into FilesManager key access protection
and raises.
"""
if attr == "files":
return getattr(obj, attr, default)
else:
return get_value(obj, attr, default)
def callapi(request_schema: Schema = None,
response_schema: Schema = None,
path_param_schema: Schema = None,
querystring_schema: Schema = None,
api_url: str = None,
http_method: HttpMethod = None,
request_headers: dict = None,
event: dict = None):
"""
:param request_schema: marshmallow schema to be used request body
:param response_schema: marshmallow schema to be used response body
:param path_param_schema: marshmallow schema with values to be replaced in api_url param
:param querystring_schema: marshmallow schema to be used querystring
:param api_url: url with address api, placeholders {xxx} to be replaced to path_params_schema values
:param http_method: method for api request
:param request_headers: request headers to request
:param event: original event from api gateway proxy integration
:return:
"""
payload = None
params = {'params': None}
text_body = get_value(event, 'body')
if path_param_schema:
api_url = get_url_with_params(api_url, event, path_param_schema)
if querystring_schema:
params = get_querystring_params(event, querystring_schema)
if request_schema:
params = get_json_body_payload(request_schema, text_body)
logger.info(f'calling url:{api_url}')
logger.info('with headers:')
logger.info(pformat(request_headers))
res = http_method(api_url, **params, headers=request_headers)
logger.info(f'request : {res.request}')
logger.info(f'response text: {res.text}')
try:
res.raise_for_status()
except requests.exceptions.RequestException as e:
if res.status_code == 404:
raise exceptions.NotFoundException()
raise exceptions.UnprocessableException(res.text)
if response_schema:
result = response_schema().dump(res.json())
logger.info('response:')
logger.info(pformat(result))
return result
def get_parameters(obj: prefect.Flow, context: dict) -> List:
if isinstance(obj, prefect.Flow):
params = {
p
for p in obj.tasks
if isinstance(p, prefect.core.parameter.Parameter)
}
else:
params = utils.get_value(obj, "parameters")
if params is missing:
return params
return sorted(params, key=lambda p: p.slug)
def get_edges(obj: prefect.Flow, context: dict) -> List:
if isinstance(obj, prefect.Flow):
edges = obj.edges
else:
edges = utils.get_value(obj, "edges")
if edges is missing:
return edges
return list(
sorted(
edges,
key=lambda e: (e.upstream_task.slug, e.downstream_task.slug),
))
def resolve_params(obj, params, default=missing):
"""Given a dictionary of keyword arguments, return the same dictionary except with
values enclosed in `< >` resolved to attributes on `obj`.
"""
param_values = {}
for name, attr_tpl in params.items():
attr_name = tpl(str(attr_tpl))
if attr_name:
attribute_value = get_value(obj, attr_name, default=default)
if attribute_value is not missing:
param_values[name] = attribute_value
else:
raise AttributeError("{attr_name!r} is not a valid "
"attribute of {obj!r}".format(
attr_name=attr_name, obj=obj))
else:
param_values[name] = attr_tpl
return param_values
def resolve_params(obj, params):
"""Given a dictionary of keyword arguments, return the same dictionary except with
values enclosed in `< >` resolved to attributes on `obj`.
"""
param_values = {}
for name, attr_tpl in iteritems(params):
attr_name = tpl(str(attr_tpl))
if attr_name:
attribute_value = get_value(attr_name, obj, default=missing)
if attribute_value is not missing:
param_values[name] = attribute_value
else:
raise AttributeError(
'{attr_name!r} is not a valid '
'attribute of {obj!r}'.format(attr_name=attr_name, obj=obj)
)
else:
param_values[name] = attr_tpl
return param_values
def get_value_or_raise(attr, obj):
value = get_value(attr, obj)
if value is missing:
raise AttributeError('{0!r} has no attribute {1!r}'.format(obj, attr))
return value
def get_attribute(self, attr, obj, default):
"""Defines how to pull values from an object to serialize.
.. versionadded:: 2.0.0
"""
return utils.get_value(attr, obj, default)
def test_get_value_from_dict():
d = dict(items=["foo", "bar"], keys=["baz", "quux"])
assert utils.get_value(d, "items") == ["foo", "bar"]
assert utils.get_value(d, "keys") == ["baz", "quux"]
def test_get_value_for_nested_object():
tri = Triangle(p1=PointClass(1, 2), p2=PointNT(3, 4), p3={"x": 5, "y": 6})
assert utils.get_value(tri, "p1.x") == 1
assert utils.get_value(tri, "p2.x") == 3
assert utils.get_value(tri, "p3.x") == 5
def test_get_value_from_namedtuple_with_default():
p = PointNT(x=42, y=None)
# Default is only returned if key is not found
assert utils.get_value(p, "z", default=123) == 123
# since 'y' is an attribute, None is returned instead of the default
assert utils.get_value(p, "y", default=123) is None
def test_get_value_for_nested_object():
tri = Triangle(p1=PointClass(1, 2), p2=PointNT(3, 4), p3={'x': 5, 'y': 6})
assert utils.get_value('p1.x', tri) == 1
assert utils.get_value('p2.x', tri) == 3
assert utils.get_value('p3.x', tri) == 5
def get_attribute(self, attr, obj, default):
"""Defines how to pull values from an object to serialize.
.. versionadded:: 2.0.0
"""
return utils.get_value(attr, obj, default)
def test_get_value_from_dict():
d = dict(items=['foo', 'bar'], keys=['baz', 'quux'])
assert utils.get_value(d, 'items') == ['foo', 'bar']
assert utils.get_value(d, 'keys') == ['baz', 'quux']
def test_get_value_for_nested_object():
tri = Triangle(p1=PointClass(1, 2), p2=PointNT(3, 4), p3={'x': 5, 'y': 6})
assert utils.get_value(tri, 'p1.x') == 1
assert utils.get_value(tri, 'p2.x') == 3
assert utils.get_value(tri, 'p3.x') == 5
def test_get_value_from_object(obj):
result = utils.get_value(obj, 'x')
assert result == 24
result2 = utils.get_value(obj, 'y')
assert result2 == 42
def _field_for_data(self, data):
condition = get_value(self.when, data)
try:
return self.then[condition]
except KeyError:
self.fail('unknown_condition', value=condition, field=self.when)
def test_get_value(obj):
result = utils.get_value('x', obj)
assert result == 24
result2 = utils.get_value('y', obj)
assert result2 == 42
def test_get_value_default():
p = PointClass(x=42, y=None)
# Default is only returned if key is not found
assert utils.get_value('z', p, default=123) == 123
# since 'y' is an attribute, None is returned instead of the default
assert utils.get_value('y', p, default=123) is None
def test_get_value_from_object(obj):
assert utils.get_value(obj, "x") == 24
assert utils.get_value(obj, "y") == 42
def get_reference_tasks(obj: prefect.Flow, context: dict) -> Set:
if isinstance(obj, prefect.Flow):
return obj._reference_tasks
else:
return utils.get_value(obj, "reference_tasks")
def test_get_value_from_dict():
d = dict(items=['foo', 'bar'], keys=['baz', 'quux'])
assert utils.get_value('items', d) == ['foo', 'bar']
assert utils.get_value('keys', d) == ['baz', 'quux']