def test_jira_notifier_pulls_creds_from_secret(monkeypatch):
client = MagicMock()
jira = MagicMock(client=client)
monkeypatch.setattr("jira.JIRA", jira)
state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(
secrets=dict(
JIRASECRETS={
"JIRAUSER": "******",
"JIRATOKEN": "",
"JIRASERVER": "https://foo/bar",
}
)
):
jira_notifier(
Task(),
"",
state,
options={"project": "TEST", "issuetype": {"name": "Task"}},
)
with pytest.raises(ValueError, match="JIRASECRETS"):
jira_notifier(
Task(),
"",
state,
options={"project": "TEST", "issuetype": {"name": "Task"}},
)
kwargs = jira.call_args[1]
assert kwargs == {
"basic_auth": ("Bob", ""),
"options": {"server": "https://foo/bar"},
}
def test_copying_then_setting_tags_doesnt_leak_backwards():
with Flow(name="test"):
t1 = Task()
with tasks.tags("init-tag"):
t2 = t1.copy()
assert t2.tags == {"init-tag"}
assert t1.tags == set()
def test_modify_task_changes_hash():
f = Flow(name="test")
t = Task()
f.add_task(t)
hash1 = f.generate_local_task_ids()
# this is not an attribute referenced in task.serialize(), so it should not affect the id
t.new_attribute = "hi"
hash2 = f.generate_local_task_ids()
# this is an attribute referenced in task.serialize(), so it should affect the id
t.slug = "hi"
hash3 = f.generate_local_task_ids()
assert hash1 == hash2
assert hash1 != hash3
def test_no_raise_on_normal_flow(self):
@task
def up():
pass
@task
def down(x):
pass
with Flow("THIS IS A TEST") as flow:
result = down(x=up, upstream_tasks=[Task(), Task()])
assert healthchecks.result_handler_check([flow]) is None
def test_slack_notifier_returns_new_state_and_old_state_is_ignored(monkeypatch):
ok = MagicMock(ok=True)
monkeypatch.setattr(requests, "post", ok)
new_state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(SLACK_WEBHOOK_URL="")):
assert slack_notifier(Task(), "", new_state) is new_state
def test_can_push_with_tags(self):
self.tag_repo(['1.0.0', '1', '1.0', '1.0.0+20200228.blue-ivory'])
checkout = SemanticCheckoutTask(upstream_repos=dict(
abc=f'sgr://{remote_name}/abc/1234?tag=1', ), )
workspaces = checkout.run()
push = PushRepoTask(workspaces=workspaces, )
self.repo.commit()
runner = TaskRunner(task=push)
tags_edge = Edge(Task(), push, key='sgr_tags')
tag_state = Success(result=ConstantResult(value=dict(
abc=['foo', 'bar', 'tag1_w_upstream'])))
with raise_on_exception():
with prefect.context():
state = runner.run(upstream_states={tags_edge: tag_state})
if state.is_failed():
print(state)
self.fail()
self.assertCountEqual(
self.repo.head.get_tags(),
['HEAD', 'foo', 'bar', 'tag1_w_upstream'])
def test_gmail_notifier_ignores_ignore_states(monkeypatch):
all_states = [
Running,
Pending,
Finished,
Failed,
TriggerFailed,
Cached,
Scheduled,
Retrying,
Success,
Skipped,
]
smtp = MagicMock()
sendmail = MagicMock()
smtp.SMTP_SSL.return_value.sendmail = sendmail
monkeypatch.setattr(prefect.utilities.notifications.notifications,
"smtplib", smtp)
for state in all_states:
s = state()
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(
secrets=dict(EMAIL_USERNAME="", EMAIL_PASSWORD="")):
returned = gmail_notifier(Task(), "", s, ignore_states=[State])
assert returned is s
assert sendmail.called is False
def test_clean_cru_elec_demand_run(clean_cru_elec_demand: Task) -> None:
"""Test run output matches expected.
Args:
clean_cru_elec_demand (Task): An instance of Task CleanCRUElecDemand
"""
expected_output = pd.DataFrame(
{
"id": pd.Series([1392, 1392, 1392, 1392], dtype="int16"),
"demand": pd.Series([0.14, 0.138, 0.14, 0.138], dtype="float32"),
"datetime": pd.Series(
[
"2009-07-15 01:30:00",
"2009-07-15 02:00:00",
"2009-07-15 01:30:00",
"2009-07-15 02:00:00",
],
dtype="datetime64[ns]",
),
},
)
output = clean_cru_elec_demand.run().compute().reset_index(drop=True)
assert_frame_equal(output, expected_output)
def test_jira_notifier_is_curried_and_uses_only_states(monkeypatch, state):
state = state()
client = MagicMock()
jiraMock = MagicMock(client=client)
monkeypatch.setattr(
"prefect.utilities.notifications.jira_notification.JIRA", jiraMock)
handler = jira_notifier(only_states=[TriggerFailed])
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(
JIRASECRETS={
"JIRAUSER": "******",
"JIRATOKEN": "",
"JIRASERVER": ""
})):
returned = handler(
Task(),
"",
state,
options={
"project": "TEST",
"issuetype": {
"name": "Task"
}
},
)
assert returned is state
assert jiraMock.called is isinstance(state, TriggerFailed)
def test_can_semantic_bump_prerelease(self):
semantic_bump = SemanticBumpTask()
runner = TaskRunner(task=semantic_bump)
edge = Edge(Task(), semantic_bump, key='workspaces')
upstream_state = Success(result=ConstantResult(value=dict(abc=dict(
repo_uri='sgr:///abc/1234?tag=1-hourly',
version=Version('1.0.1-hourly.4+2021-03-08.zip-fur'),
))))
date = datetime.utcnow()
flow_run_name = 'testflow1'
with raise_on_exception():
with prefect.context(date=date, flow_run_name=flow_run_name):
state = runner.run(upstream_states={edge: upstream_state})
if state.is_failed():
print(state)
self.fail()
self.assertEqual(
state.result,
dict(abc=[
'1-hourly', '1.0-hourly',
f'1.0.1-hourly.5+{date:%Y-%m-%dT%H}.{date:%M}.{flow_run_name}'
]))
def test_slack_notifier_returns_new_state_and_old_state_is_ignored(
monkeypatch):
ok = MagicMock(ok=True)
monkeypatch.setattr(prefect.utilities.notifications.requests, "post", ok)
new_state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
assert slack_notifier(Task(), "", new_state) is new_state
def test_formatter_formats_states_with_exception_message(state):
orig = slack_message_formatter(Task(),
state(result=ZeroDivisionError("Nope")))
expected = "```ZeroDivisionError('Nope'"
expected += ")```" if sys.version_info >= (3, 7) else ",)```"
assert orig["attachments"][0]["fields"][0]["value"] == expected
assert json.loads(json.dumps(orig)) == orig
def test_jira_notifier_ignores_ignore_states(monkeypatch):
all_states = [
Running,
Pending,
Finished,
Failed,
TriggerFailed,
Cached,
Scheduled,
Retrying,
Success,
Skipped,
]
client = MagicMock()
jiraMock = MagicMock(client=client)
monkeypatch.setattr("jira.JIRA", jiraMock)
for state in all_states:
s = state()
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(
secrets=dict(
JIRASECRETS={"JIRAUSER": "******", "JIRATOKEN": "", "JIRASERVER": ""}
)
):
returned = jira_notifier(
Task(),
"",
s,
ignore_states=[State],
options={"project": "TEST", "issuetype": {"name": "Task"}},
)
assert returned is s
assert jiraMock.called is False
def test_setting_tags_then_calling_copies_tags():
with tasks.tags("init-tag"):
t1 = Task()
with Flow(name="test"):
t2 = t1()
assert t2.tags == {"init-tag"}
def test_slack_notifier_pulls_url_from_secret(monkeypatch):
post = MagicMock(ok=True)
monkeypatch.setattr("requests.post", post)
state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
with pytest.raises(ValueError, match="SLACK_WEBHOOK_URL"):
slack_notifier(Task(), "", state)
with prefect.context(secrets=dict(SLACK_WEBHOOK_URL="https://foo/bar")):
slack_notifier(Task(), "", state)
assert post.call_args[0][0] == "https://foo/bar"
with prefect.context(secrets=dict(TOP_SECRET='"42"')):
slack_notifier(Task(), "", state, webhook_secret="TOP_SECRET")
assert post.call_args[0][0] == "42"
def test_slack_notifier_is_curried_and_uses_only_states(monkeypatch, state):
state = state()
ok = MagicMock(ok=True)
monkeypatch.setattr(prefect.utilities.notifications.requests, "post", ok)
handler = slack_notifier(only_states=[TriggerFailed])
with set_temporary_config({"cloud.use_local_secrets": True}):
returned = handler(Task(), "", state)
assert returned is state
assert ok.called is isinstance(state, TriggerFailed)
def test_context_manager_for_setting_tags():
"""
Test setting Task tags with a context manager, including:
- top level
- nested
- nested with explicit tags
"""
with tasks.tags("1", "2"):
t1 = Task()
assert t1.tags == set(["1", "2"])
with tasks.tags("3", "4"):
t2 = Task()
assert t2.tags == set(["1", "2", "3", "4"])
t3 = Task(tags=["5"])
assert t3.tags == set(["1", "2", "3", "4", "5"])
def test_slack_notifier_is_curried_and_ignores_ignore_states(
monkeypatch, state):
state = state()
ok = MagicMock(ok=True)
monkeypatch.setattr(prefect.utilities.notifications.requests, "post", ok)
handler = slack_notifier(ignore_states=[Finished])
with set_temporary_config({"cloud.use_local_secrets": True}):
returned = handler(Task(), "", state)
assert returned is state
assert ok.called is not state.is_finished()
def test_slack_notifier_is_curried_and_ignores_ignore_states(monkeypatch, state):
state = state()
ok = MagicMock(ok=True)
monkeypatch.setattr(requests, "post", ok)
handler = slack_notifier(ignore_states=[Finished])
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(SLACK_WEBHOOK_URL="")):
returned = handler(Task(), "", state)
assert returned is state
assert ok.called is not state.is_finished()
def test_slack_notifier_is_curried_and_uses_only_states(monkeypatch, state):
state = state()
ok = MagicMock(ok=True)
monkeypatch.setattr(requests, "post", ok)
handler = slack_notifier(only_states=[TriggerFailed])
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(SLACK_WEBHOOK_URL="")):
returned = handler(Task(), "", state)
assert returned is state
assert ok.called is isinstance(state, TriggerFailed)
def test_gmail_notifier_is_curried_and_uses_only_states(monkeypatch, state):
state = state()
smtp = MagicMock()
sendmail = MagicMock()
smtp.SMTP_SSL.return_value.sendmail = sendmail
monkeypatch.setattr(prefect.utilities.notifications, "smtplib", smtp)
monkeypatch.setattr(prefect.config.cloud, "use_local_secrets", True)
handler = gmail_notifier(only_states=[TriggerFailed])
returned = handler(Task(), "", state)
assert returned is state
assert sendmail.called is isinstance(state, TriggerFailed)
def test_slack_notifier_uses_proxies(monkeypatch):
post = MagicMock(ok=True)
monkeypatch.setattr(requests, "post", post)
state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(SLACK_WEBHOOK_URL="")):
slack_notifier(Task(),
"",
state,
proxies={"http": "some.proxy.I.P"})
assert post.call_args[1]["proxies"] == {"http": "some.proxy.I.P"}
def test_gmail_notifier_is_curried_and_uses_only_states(monkeypatch, state):
state = state()
smtp = MagicMock()
sendmail = MagicMock()
smtp.SMTP_SSL.return_value.sendmail = sendmail
monkeypatch.setattr(prefect.utilities.notifications, "smtplib", smtp)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(EMAIL_USERNAME="", EMAIL_PASSWORD="")):
handler = gmail_notifier(only_states=[TriggerFailed])
returned = handler(Task(), "", state)
assert returned is state
assert sendmail.called is isinstance(state, TriggerFailed)
def test_deserialize_flow(self):
f = Flow(name="test")
f.add_task(Task())
f.add_task(Parameter("x"))
env = LocalEnvironment()
serialized = env.serialize_flow_to_bytes(f)
deserialized = env.deserialize_flow_from_bytes(serialized)
assert isinstance(deserialized, Flow)
assert len(deserialized.tasks) == 2
assert {p.name for p in deserialized.parameters()} == {"x"}
def test_gmail_notifier_sends_simple_email(monkeypatch):
smtp = MagicMock()
sendmail = MagicMock()
smtp.SMTP_SSL.return_value.sendmail = sendmail
monkeypatch.setattr(prefect.utilities.notifications, "smtplib", smtp)
s = Failed("optional message...")
monkeypatch.setattr(prefect.config.cloud, "use_local_secrets", True)
with prefect.context(secrets=dict(EMAIL_USERNAME="******", EMAIL_PASSWORD=1234)):
returned = gmail_notifier(Task(name="dud"), "", s)
assert returned is s
email_from, to, body = sendmail.call_args[0]
assert email_from == "*****@*****.**"
assert to == "alice"
assert "Failed" in body
assert "optional message" in body
assert s.color in body
def test_every_state_gets_a_unique_color():
all_states = [
Running,
Pending,
Finished,
Failed,
TriggerFailed,
Cached,
Scheduled,
Retrying,
Success,
Skipped,
]
colors = set()
for state in all_states:
color = slack_message_formatter(Task(), state())["attachments"][0]["color"]
colors.add(color)
assert len(colors) == len(all_states)
def test_version_to_date(self):
version_to_date = VersionToDateTask()
runner = TaskRunner(task=version_to_date)
edge = Edge(Task(), version_to_date, key='version')
upstream_state = Success(result=ConstantResult(
value=Version('1.0.0+2021-03-03T00.stinky-fish')))
with raise_on_exception():
with prefect.context():
state = runner.run(upstream_states={edge: upstream_state})
if state.is_failed():
print(state)
self.fail()
self.assertEqual(state.result, pendulum.parse('2021-03-03T00'))
def test_jira_notifier_returns_new_state_and_old_state_is_ignored(monkeypatch):
client = MagicMock()
jira = MagicMock(client=client)
monkeypatch.setattr("jira.JIRA", jira)
new_state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(
secrets=dict(
JIRASECRETS={"JIRAUSER": "******", "JIRATOKEN": "123", "JIRASERVER": ""}
)
):
assert (
jira_notifier(
Task(),
"",
new_state,
options={"project": "TEST", "issuetype": {"name": "Task"}},
)
is new_state
)
def test_jira_notifier_is_curried_and_ignores_ignore_states(monkeypatch, state):
state = state()
client = MagicMock()
jiraMock = MagicMock(client=client)
monkeypatch.setattr("jira.JIRA", jiraMock)
handler = jira_notifier(ignore_states=[Finished])
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(
secrets=dict(
JIRASECRETS={"JIRAUSER": "******", "JIRATOKEN": "", "JIRASERVER": ""}
)
):
returned = handler(
Task(),
"",
state,
options={"project": "TEST", "issuetype": {"name": "Task"}},
)
assert returned is state
assert jiraMock.called is not state.is_finished()
def test_ten_different_tasks():
"""
x1
x2
...
x10
Ten non-identical and independent tasks
"""
f = Flow(name="test")
for i in range(1, 11):
f.add_task(Task(name=str(i)))
steps = f.generate_local_task_ids(_debug_steps=True)
# tasks are immediately identifiable
assert count_unique_ids(steps[1]) == 10
# no further processing
assert steps[1] == steps[2] == steps[3] == steps[4] == steps[5]