def test_task_runner_validates_cached_state_inputs_if_task_has_caching_and_uses_task_handler(
client, ):
class Handler(ResultHandler):
def read(self, val):
return 1337
@prefect.task(
cache_for=datetime.timedelta(minutes=1),
cache_validator=all_inputs,
result_handler=Handler(),
)
def cached_task(x):
return 42
dull_state = Cached(
cached_result_expiration=datetime.datetime.utcnow() +
datetime.timedelta(minutes=2),
result=SafeResult("-1", JSONResultHandler()),
)
state = Cached(
cached_result_expiration=datetime.datetime.utcnow() +
datetime.timedelta(minutes=2),
result=SafeResult("99", JSONResultHandler()),
cached_inputs={
"x": SafeResult("2", result_handler=JSONResultHandler())
},
)
client.get_latest_cached_states = MagicMock(
return_value=[dull_state, state])
res = CloudTaskRunner(task=cached_task).check_task_is_cached(
Pending(), inputs={"x": Result(2, result_handler=JSONResultHandler())})
assert client.get_latest_cached_states.called
assert res.is_successful()
assert res.is_cached()
assert res.result == 1337
async def test_set_task_run_state_with_safe_result(self, run_query,
task_run_id):
result = SafeResult("10", result_handler=JSONResultHandler())
state = Success(result=result)
result = await run_query(
query=self.mutation,
variables=dict(input=dict(states=[
dict(task_run_id=task_run_id, state=state.serialize())
])),
)
tr = await models.TaskRun.where(
id=result.data.set_task_run_states.states[0].id
).first({"state", "version"})
assert tr.version == 2
assert tr.state == "Success"
def test_serialize_and_deserialize_on_mixed_cached_state():
safe_dct = SafeResult(dict(hi=5, bye=6),
result_handler=JSONResultHandler())
now = pendulum.now("utc")
state = Cached(
cached_inputs=dict(x=Result(2), p=Result("p")),
result=safe_dct,
cached_result_expiration=now,
)
serialized = state.serialize()
new_state = State.deserialize(serialized)
assert isinstance(new_state, Cached)
assert new_state.color == state.color
assert new_state.result == dict(hi=5, bye=6)
assert new_state.cached_result_expiration == state.cached_result_expiration
assert new_state.cached_inputs == dict.fromkeys(["x", "p"], NoResult)
def test_set_task_run_state_serializes(monkeypatch):
response = {"data": {"setTaskRunState": None}}
post = MagicMock(return_value=MagicMock(json=MagicMock(
return_value=response)))
monkeypatch.setattr("requests.post", post)
with set_temporary_config({
"cloud.graphql": "http://my-cloud.foo",
"cloud.auth_token": "secret_token"
}):
client = Client()
res = SafeResult(lambda: None, result_handler=None)
with pytest.raises(marshmallow.exceptions.ValidationError) as exc:
result = client.set_task_run_state(task_run_id="76-salt",
version=0,
state=Pending(result=res))
def test_set_task_run_state_serializes(patch_post):
response = {"data": {"set_task_run_states": {"states": [{"status": "SUCCESS"}]}}}
post = patch_post(response)
with set_temporary_config(
{
"cloud.api": "http://my-cloud.foo",
"cloud.auth_token": "secret_token",
"backend": "cloud",
}
):
client = Client()
res = SafeResult(lambda: None, result_handler=None)
with pytest.raises(marshmallow.exceptions.ValidationError):
client.set_task_run_state(
task_run_id="76-salt", version=0, state=Pending(result=res)
)
def test_task_failure_with_upstream_secrets_doesnt_store_secret_value_and_recompute_if_necessary(
client, ):
@prefect.task(max_retries=2, retry_delay=timedelta(seconds=100))
def is_p_three(p):
if p == 3:
raise ValueError("No thank you.")
return p
with prefect.Flow("test", result_handler=JSONResultHandler()) as f:
p = prefect.tasks.secrets.Secret("p")
res = is_p_three(p)
with prefect.context(secrets=dict(p=3)):
state = CloudFlowRunner(flow=f).run(return_tasks=[res])
assert state.is_running()
assert isinstance(state.result[res], Retrying)
exp_res = Result(3, result_handler=SecretResultHandler(p))
assert not state.result[res].cached_inputs["p"] == exp_res
exp_res.store_safe_value()
assert state.result[res].cached_inputs["p"] == exp_res
## here we set the result of the secret to a saferesult, ensuring
## it will get converted to a "true" result;
## we expect that the upstream value will actually get recomputed from context
## through the SecretResultHandler
safe = SafeResult("p", result_handler=SecretResultHandler(p))
state.result[p] = Success(result=safe)
state.result[res].start_time = pendulum.now("utc")
state.result[res].cached_inputs = dict(p=safe)
with prefect.context(secrets=dict(p=4)):
new_state = CloudFlowRunner(flow=f).run(return_tasks=[res],
task_states=state.result)
assert new_state.is_successful()
assert new_state.result[res].result == 4
def test_to_result_returns_self_for_no_results(self):
assert NoResult.to_result() is NoResult
def test_to_result_returns_hydrated_result_for_safe(self):
s = SafeResult("3", result_handler=JSONResultHandler())
res = s.to_result()
assert isinstance(res, Result)
assert res.value == 3
assert res.safe_value is s
assert res.result_handler is s.result_handler
@pytest.mark.parametrize(
"obj",
[
Result(3),
Result(object, result_handler=LocalResultHandler()),
NoResult,
SafeResult("3", result_handler=JSONResultHandler()),
],
)
def test_everything_is_pickleable_after_init(obj):
assert cloudpickle.loads(cloudpickle.dumps(obj)) == obj
def test_results_are_pickleable_with_their_safe_values():
res = Result(3, result_handler=JSONResultHandler())
res.store_safe_value()
assert cloudpickle.loads(cloudpickle.dumps(res)) == res
def test_safe_results_are_same(self):
r = SafeResult("3", result_handler=JSONResultHandler())
s = SafeResult("3", result_handler=JSONResultHandler())
assert r == s
def test_safe_result_disallows_none_result_handler_at_deserialization_time():
schema = SafeResultSchema()
r = SafeResult(value=None, result_handler=None)
serialized = schema.dump(r)
with pytest.raises(marshmallow.exceptions.ValidationError):
schema.load(serialized)
def test_value_raises_error_on_dump_if_not_valid_json():
r = SafeResult(
value={"x": {"y": {"z": lambda: 1}}}, result_handler=JSONResultHandler()
)
with pytest.raises(marshmallow.exceptions.ValidationError):
StateResultSchema().dump(r)
def test_basic_safe_result_repr():
r = SafeResult(2, result_handler=JSONResultHandler())
assert repr(r) == "<SafeResult: 2>"
def test_storing_happens_once(self):
r = Result(value=4, result_handler=JSONResultHandler())
safe_value = SafeResult(value="123", result_handler=JSONResultHandler())
r.safe_value = safe_value
r.store_safe_value()
assert r.safe_value is safe_value
<<<<<<< HEAD
res1 = results.PrefectResult(value=1)
res2 = results.PrefectResult(value={"z": 2})
res3 = results.PrefectResult(location=json.dumps(dict(x=1, y={"z": 2})))
naive_dt = datetime.datetime(2020, 1, 1)
utc_dt = pendulum.datetime(2020, 1, 1)
cached_state = state.Cached(
hashed_inputs=dict(x="foo", y="bar"),
result=res3,
cached_result_expiration=utc_dt,
)
cached_state_naive = state.Cached(
hashed_inputs=dict(x="foo", y="bar"),
result=res3,
=======
res1 = SafeResult(1, result_handler=JSONResultHandler())
res2 = SafeResult({"z": 2}, result_handler=JSONResultHandler())
res3 = SafeResult(dict(x=1, y={"z": 2}), result_handler=JSONResultHandler())
naive_dt = datetime.datetime(2020, 1, 1)
utc_dt = pendulum.datetime(2020, 1, 1)
complex_result = {"x": res1, "y": res2}
cached_state = state.Cached(
cached_inputs=complex_result,
result=res3,
cached_parameters={"x": 1, "y": {"z": 2}},
cached_result_expiration=utc_dt,
)
cached_state_naive = state.Cached(
cached_inputs=complex_result,
result=res3,
cached_parameters={"x": 1, "y": {"z": 2}},
def test_store_safe_value_for_safe_results(self):
r = SafeResult(value=4, result_handler=JSONResultHandler())
output = r.store_safe_value()
assert output is None
assert isinstance(r.safe_value, SafeResult)
assert r.value == 4
def test_serialize_state_with_safe_result(cls):
res = SafeResult(value="1", result_handler=JSONResultHandler())
def test_result_allows_none_value():
schema = SafeResultSchema()
r = SafeResult(value=None, result_handler=JSONResultHandler())
handled = schema.load(schema.dump(r))
assert handled.value is None
assert handled.result_handler == JSONResultHandler()
def test_safe_result_disallows_none_result_handler_at_deserialization_time():
schema = SafeResultSchema()
r = SafeResult(value=None, result_handler=None)
serialized = schema.dump(r)
with pytest.raises(marshmallow.exceptions.ValidationError):
schema.load(serialized)
@pytest.mark.parametrize(
"obj", [SafeResult(value=19, result_handler=JSONResultHandler()), NoResult]
)
def test_state_result_schema_chooses_schema(obj):
schema = StateResultSchema()
assert type(schema.load(schema.dump(obj))) == type(obj)
def test_value_raises_error_on_dump_if_not_valid_json():
r = SafeResult(
value={"x": {"y": {"z": lambda: 1}}}, result_handler=JSONResultHandler()
)
with pytest.raises(marshmallow.exceptions.ValidationError):
StateResultSchema().dump(r)
def test_result_must_be_valid_json():
res = SafeResult({"x": {"y": {"z": 1}}}, result_handler=JSONResultHandler())
def test_result_raises_error_on_dump_if_not_valid_json():
res = SafeResult({"x": {"y": {"z": lambda: 1}}}, result_handler=JSONResultHandler())
def test_result_must_be_valid_json():
res = SafeResult({"x": {"y": {"z": 1}}}, result_handler=JSONResultHandler())
s = state.Success(result=res)
serialized = StateSchema().dump(s)
assert serialized["_result"]["value"] == s.result
def test_safe_results_with_different_handlers_are_not_same(self):
r = SafeResult("3", result_handler=JSONResultHandler())
s = SafeResult("3", result_handler=LocalResultHandler())
assert r != s
def test_roundtrip_for_results():
r = SafeResult(value={"y": 4}, result_handler=JSONResultHandler())
s = SafeResultSchema().load(SafeResultSchema().dump(r))
assert r == s
def test_safe_results_to_results_remain_the_same(self):
r = SafeResult("3", result_handler=JSONResultHandler())
s = SafeResult("3", result_handler=JSONResultHandler())
assert r.to_result() == s.to_result()
def test_safe_result_serializes_result_handlers():
r = SafeResult(value=3, result_handler=JSONResultHandler())
handled = SafeResultSchema().dump(r)
assert handled["result_handler"]["type"] == "JSONResultHandler"
def test_safe_result_inits_with_both_args(self):
res = SafeResult(value="3", result_handler=JSONResultHandler())
assert res.value == "3"
assert res.result_handler == JSONResultHandler()
assert res.safe_value is res
def test_result_allows_none_value():
schema = SafeResultSchema()
r = SafeResult(value=None, result_handler=JSONResultHandler())
handled = schema.load(schema.dump(r))
assert handled.value is None
assert handled.result_handler == JSONResultHandler()
def test_basic_safe_result_serializes():
res = SafeResult("3", result_handler=JSONResultHandler())
serialized = SafeResultSchema().dump(res)
assert serialized["__version__"] == prefect.__version__
assert serialized["value"] == "3"
assert serialized["result_handler"]["type"] == "JSONResultHandler"
def test_result_raises_error_on_dump_if_not_valid_json():
res = SafeResult({"x": {"y": {"z": lambda: 1}}}, result_handler=JSONResultHandler())
s = state.Success(result=res)
with pytest.raises(marshmallow.exceptions.ValidationError):
StateSchema().dump(s)
