def test_email_dashboard_report_fails(screenshot_mock, email_mock,
create_report_email_dashboard):
"""
ExecuteReport Command: Test dashboard email report schedule notification fails
"""
# setup screenshot mock
from smtplib import SMTPException
screenshot = read_fixture("sample.png")
screenshot_mock.return_value = screenshot
email_mock.side_effect = SMTPException("Could not connect to SMTP XPTO")
with pytest.raises(ReportScheduleNotificationError):
AsyncExecuteReportScheduleCommand(create_report_email_dashboard.id,
datetime.utcnow()).run()
assert_log(ReportState.ERROR,
error_message="Could not connect to SMTP XPTO")
def test_slack_token_callable_chart_report(
screenshot_mock, slack_client_mock_class, create_report_slack_chart
):
"""
ExecuteReport Command: Test chart slack alert (slack token callable)
"""
slack_client_mock_class.return_value = Mock()
app.config["SLACK_API_TOKEN"] = Mock(return_value="cool_code")
# setup screenshot mock
screenshot = read_fixture("sample.png")
screenshot_mock.return_value = screenshot
with freeze_time("2020-01-01T00:00:00Z"):
AsyncExecuteReportScheduleCommand(
test_id, create_report_slack_chart.id, datetime.utcnow()
).run()
app.config["SLACK_API_TOKEN"].assert_called_once()
assert slack_client_mock_class.called_with(token="cool_code", proxy="")
assert_log(ReportState.SUCCESS)
def mock_NLU_for(httpx_mock, dataset):
with override_settings({
"NLU_TAGGER_URL": NLU_URL,
"NLU_CLASSIFIER_URL": CLASSIF_URL,
"PJ_ES": ES_URL
}):
nlu_json = read_fixture(f"fixtures/autocomplete/nlu/{dataset}.json")
httpx_mock.post(NLU_URL).respond(json=nlu_json)
for q, data in FIXTURE_CLASSIF.items():
httpx_mock.post(CLASSIF_URL,
json={
"text": q,
"domain": "poi",
"language": "fr",
"count": 10
}).respond(json=data)
yield nlu_json
def test_slack_chart_alert(screenshot_mock, email_mock, create_alert_email_chart):
"""
ExecuteReport Command: Test chart slack alert
"""
# setup screenshot mock
screenshot = read_fixture("sample.png")
screenshot_mock.return_value = screenshot
with freeze_time("2020-01-01T00:00:00Z"):
AsyncExecuteReportScheduleCommand(
create_alert_email_chart.id, datetime.utcnow()
).run()
notification_targets = get_target_from_report_schedule(create_alert_email_chart)
# Assert the email smtp address
assert email_mock.call_args[0][0] == notification_targets[0]
# Assert the email inline screenshot
smtp_images = email_mock.call_args[1]["images"]
assert smtp_images[list(smtp_images.keys())[0]] == screenshot
# Assert logs are correct
assert_log(ReportState.SUCCESS)
def test_slack_chart_report_schedule(screenshot_mock, file_upload_mock,
create_report_slack_chart):
"""
ExecuteReport Command: Test chart slack report schedule
"""
# setup screenshot mock
screenshot = read_fixture("sample.png")
screenshot_mock.return_value = screenshot
with freeze_time("2020-01-01T00:00:00Z"):
AsyncExecuteReportScheduleCommand(create_report_slack_chart.id,
datetime.utcnow()).run()
notification_targets = get_target_from_report_schedule(
create_report_slack_chart)
assert file_upload_mock.call_args[1][
"channels"] == notification_targets[0]
assert file_upload_mock.call_args[1]["file"] == screenshot
# Assert logs are correct
assert_log(ReportState.SUCCESS)
def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class):
# Test functionality for chrome driver which does not support
# element snapshots
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
type(element).screenshot_as_png = PropertyMock(side_effect=WebDriverException)
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_id.return_value = element
driver.find_element_by_class_name.return_value = element
driver.screenshot.return_value = read_fixture("sample.png")
schedule = (
db.session.query(DashboardEmailSchedule)
.filter_by(id=self.dashboard_schedule)
.one()
)
deliver_dashboard(
schedule.dashboard_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_called_once()
send_email_smtp.assert_called_once()
self.assertEqual(send_email_smtp.call_args[0][0], self.RECIPIENTS)
self.assertEqual(
list(send_email_smtp.call_args[1]["images"].values())[0],
driver.screenshot.return_value,
)
def test_deliver_dashboard_as_attachment(
self, mtime, send_email_smtp, driver_class
):
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_id.return_value = element
driver.find_element_by_class_name.return_value = element
element.screenshot_as_png = read_fixture("sample.png")
schedule = (
db.session.query(DashboardEmailSchedule)
.filter_by(id=self.dashboard_schedule)
.one()
)
schedule.delivery_type = EmailDeliveryType.attachment
deliver_dashboard(
schedule.dashboard_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_not_called()
send_email_smtp.assert_called_once()
self.assertIsNone(send_email_smtp.call_args[1]["images"])
self.assertEqual(
send_email_smtp.call_args[1]["data"]["screenshot"],
element.screenshot_as_png,
)
def test_deliver_email_options(self, mtime, send_email_smtp, driver_class):
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_class_name.return_value = element
element.screenshot_as_png = read_fixture("sample.png")
schedule = (
db.session.query(DashboardEmailSchedule)
.filter_by(id=self.dashboard_schedule)
.one()
)
# Send individual mails to the group
schedule.deliver_as_group = False
# Set a bcc email address
app.config["EMAIL_REPORT_BCC_ADDRESS"] = self.BCC
deliver_dashboard(
schedule.dashboard_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_not_called()
self.assertEqual(send_email_smtp.call_count, 2)
self.assertEqual(send_email_smtp.call_args[1]["bcc"], self.BCC)
from superset.reports.commands.log_prune import AsyncPruneReportScheduleLogCommand
from superset.utils.core import get_example_database
from tests.fixtures.birth_names_dashboard import load_birth_names_dashboard_with_slices
from tests.fixtures.world_bank_dashboard import (
load_world_bank_dashboard_with_slices_module_scope,
)
from tests.reports.utils import insert_report_schedule
from tests.test_app import app
from tests.utils import read_fixture
pytestmark = pytest.mark.usefixtures(
"load_world_bank_dashboard_with_slices_module_scope"
)
TEST_ID = str(uuid4())
CSV_FILE = read_fixture("trends.csv")
SCREENSHOT_FILE = read_fixture("sample.png")
OWNER_EMAIL = "*****@*****.**"
def get_target_from_report_schedule(report_schedule: ReportSchedule) -> List[str]:
return [
json.loads(recipient.recipient_config_json)["target"]
for recipient in report_schedule.recipients
]
def get_error_logs_query(report_schedule: ReportSchedule) -> BaseQuery:
return (
db.session.query(ReportExecutionLog)
.filter(
class TestSchedules(SupersetTestCase):
RECIPIENTS = "[email protected], [email protected]"
BCC = "*****@*****.**"
CSV = read_fixture("trends.csv")
@classmethod
def setUpClass(cls):
with app.app_context():
cls.common_data = dict(
active=True,
crontab="* * * * *",
recipients=cls.RECIPIENTS,
deliver_as_group=True,
delivery_type=EmailDeliveryType.inline,
)
# Pick up a sample slice and dashboard
slce = db.session.query(Slice).filter_by(
slice_name="Participants").one()
dashboard = (db.session.query(Dashboard).filter_by(
dashboard_title="World Bank's Data").one())
dashboard_schedule = DashboardEmailSchedule(**cls.common_data)
dashboard_schedule.dashboard_id = dashboard.id
dashboard_schedule.user_id = 1
db.session.add(dashboard_schedule)
slice_schedule = SliceEmailSchedule(**cls.common_data)
slice_schedule.slice_id = slce.id
slice_schedule.user_id = 1
slice_schedule.email_format = SliceEmailReportFormat.data
slice_schedule.slack_channel = "#test_channel"
db.session.add(slice_schedule)
db.session.commit()
cls.slice_schedule = slice_schedule.id
cls.dashboard_schedule = dashboard_schedule.id
@classmethod
def tearDownClass(cls):
with app.app_context():
db.session.query(SliceEmailSchedule).filter_by(
id=cls.slice_schedule).delete()
db.session.query(DashboardEmailSchedule).filter_by(
id=cls.dashboard_schedule).delete()
db.session.commit()
def test_crontab_scheduler(self):
crontab = "* * * * *"
start_at = datetime.now().replace(microsecond=0, second=0, minute=0)
stop_at = start_at + timedelta(seconds=3600)
# Fire off the task every minute
schedules = list(
next_schedules(crontab, start_at, stop_at, resolution=0))
self.assertEqual(schedules[0], start_at)
self.assertEqual(schedules[-1], stop_at - timedelta(seconds=60))
self.assertEqual(len(schedules), 60)
# Fire off the task every 10 minutes, controlled via resolution
schedules = list(
next_schedules(crontab, start_at, stop_at, resolution=10 * 60))
self.assertEqual(schedules[0], start_at)
self.assertEqual(schedules[-1], stop_at - timedelta(seconds=10 * 60))
self.assertEqual(len(schedules), 6)
# Fire off the task every 12 minutes, controlled via resolution
schedules = list(
next_schedules(crontab, start_at, stop_at, resolution=12 * 60))
self.assertEqual(schedules[0], start_at)
self.assertEqual(schedules[-1], stop_at - timedelta(seconds=12 * 60))
self.assertEqual(len(schedules), 5)
def test_wider_schedules(self):
crontab = "*/15 2,10 * * *"
for hour in range(0, 24):
start_at = datetime.now().replace(microsecond=0,
second=0,
minute=0,
hour=hour)
stop_at = start_at + timedelta(seconds=3600)
schedules = list(
next_schedules(crontab, start_at, stop_at, resolution=0))
if hour in (2, 10):
self.assertEqual(len(schedules), 4)
else:
self.assertEqual(len(schedules), 0)
def test_complex_schedule(self):
# Run the job on every Friday of March and May
# On these days, run the job at
# 5:10 pm
# 5:11 pm
# 5:12 pm
# 5:13 pm
# 5:14 pm
# 5:15 pm
# 5:25 pm
# 5:28 pm
# 5:31 pm
# 5:34 pm
# 5:37 pm
# 5:40 pm
crontab = "10-15,25-40/3 17 * 3,5 5"
start_at = datetime.strptime("2018/01/01", "%Y/%m/%d")
stop_at = datetime.strptime("2018/12/31", "%Y/%m/%d")
schedules = list(
next_schedules(crontab, start_at, stop_at, resolution=60))
self.assertEqual(len(schedules), 108)
fmt = "%Y-%m-%d %H:%M:%S"
self.assertEqual(schedules[0],
datetime.strptime("2018-03-02 17:10:00", fmt))
self.assertEqual(schedules[-1],
datetime.strptime("2018-05-25 17:40:00", fmt))
self.assertEqual(schedules[59],
datetime.strptime("2018-03-30 17:40:00", fmt))
self.assertEqual(schedules[60],
datetime.strptime("2018-05-04 17:10:00", fmt))
@patch("superset.tasks.schedules.firefox.webdriver.WebDriver")
def test_create_driver(self, mock_driver_class):
mock_driver = Mock()
mock_driver_class.return_value = mock_driver
mock_driver.find_elements_by_id.side_effect = [True, False]
create_webdriver()
mock_driver.add_cookie.assert_called_once()
@patch("superset.tasks.schedules.firefox.webdriver.WebDriver")
@patch("superset.tasks.schedules.send_email_smtp")
@patch("superset.tasks.schedules.time")
def test_deliver_dashboard_inline(self, mtime, send_email_smtp,
driver_class):
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_class_name.return_value = element
element.screenshot_as_png = read_fixture("sample.png")
schedule = (db.session.query(DashboardEmailSchedule).filter_by(
id=self.dashboard_schedule).one())
deliver_dashboard(
schedule.dashboard_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_not_called()
send_email_smtp.assert_called_once()
@patch("superset.tasks.schedules.firefox.webdriver.WebDriver")
@patch("superset.tasks.schedules.send_email_smtp")
@patch("superset.tasks.schedules.time")
def test_deliver_dashboard_as_attachment(self, mtime, send_email_smtp,
driver_class):
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_id.return_value = element
driver.find_element_by_class_name.return_value = element
element.screenshot_as_png = read_fixture("sample.png")
schedule = (db.session.query(DashboardEmailSchedule).filter_by(
id=self.dashboard_schedule).one())
schedule.delivery_type = EmailDeliveryType.attachment
deliver_dashboard(
schedule.dashboard_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_not_called()
send_email_smtp.assert_called_once()
self.assertIsNone(send_email_smtp.call_args[1]["images"])
self.assertEqual(
send_email_smtp.call_args[1]["data"]["screenshot"],
element.screenshot_as_png,
)
@patch("superset.tasks.schedules.firefox.webdriver.WebDriver")
@patch("superset.tasks.schedules.send_email_smtp")
@patch("superset.tasks.schedules.time")
def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class):
# Test functionality for chrome driver which does not support
# element snapshots
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
type(element).screenshot_as_png = PropertyMock(
side_effect=WebDriverException)
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_id.return_value = element
driver.find_element_by_class_name.return_value = element
driver.screenshot.return_value = read_fixture("sample.png")
schedule = (db.session.query(DashboardEmailSchedule).filter_by(
id=self.dashboard_schedule).one())
deliver_dashboard(
schedule.dashboard_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_called_once()
send_email_smtp.assert_called_once()
self.assertEqual(send_email_smtp.call_args[0][0], self.RECIPIENTS)
self.assertEqual(
list(send_email_smtp.call_args[1]["images"].values())[0],
driver.screenshot.return_value,
)
@patch("superset.tasks.schedules.firefox.webdriver.WebDriver")
@patch("superset.tasks.schedules.send_email_smtp")
@patch("superset.tasks.schedules.time")
def test_deliver_email_options(self, mtime, send_email_smtp, driver_class):
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_class_name.return_value = element
element.screenshot_as_png = read_fixture("sample.png")
schedule = (db.session.query(DashboardEmailSchedule).filter_by(
id=self.dashboard_schedule).one())
# Send individual mails to the group
schedule.deliver_as_group = False
# Set a bcc email address
app.config["EMAIL_REPORT_BCC_ADDRESS"] = self.BCC
deliver_dashboard(
schedule.dashboard_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_not_called()
self.assertEqual(send_email_smtp.call_count, 2)
self.assertEqual(send_email_smtp.call_args[1]["bcc"], self.BCC)
@patch("superset.tasks.slack_util.WebClient.files_upload")
@patch("superset.tasks.schedules.firefox.webdriver.WebDriver")
@patch("superset.tasks.schedules.send_email_smtp")
@patch("superset.tasks.schedules.time")
def test_deliver_slice_inline_image(self, mtime, send_email_smtp,
driver_class, files_upload):
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_class_name.return_value = element
element.screenshot_as_png = read_fixture("sample.png")
schedule = (db.session.query(SliceEmailSchedule).filter_by(
id=self.slice_schedule).one())
schedule.email_format = SliceEmailReportFormat.visualization
schedule.delivery_format = EmailDeliveryType.inline
deliver_slice(
schedule.slice_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.email_format,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_not_called()
send_email_smtp.assert_called_once()
self.assertEqual(
list(send_email_smtp.call_args[1]["images"].values())[0],
element.screenshot_as_png,
)
self.assertEqual(
files_upload.call_args[1],
{
"channels": "#test_channel",
"file": element.screenshot_as_png,
"initial_comment":
f"\n *Participants*\n\n <http://0.0.0.0:8080/superset/slice/{schedule.slice_id}/|Explore in Superset>\n ",
"title": "[Report] Participants",
},
)
@patch("superset.tasks.slack_util.WebClient.files_upload")
@patch("superset.tasks.schedules.firefox.webdriver.WebDriver")
@patch("superset.tasks.schedules.send_email_smtp")
@patch("superset.tasks.schedules.time")
def test_deliver_slice_attachment(self, mtime, send_email_smtp,
driver_class, files_upload):
element = Mock()
driver = Mock()
mtime.sleep.return_value = None
driver_class.return_value = driver
# Ensure that we are able to login with the driver
driver.find_elements_by_id.side_effect = [True, False]
driver.find_element_by_class_name.return_value = element
element.screenshot_as_png = read_fixture("sample.png")
schedule = (db.session.query(SliceEmailSchedule).filter_by(
id=self.slice_schedule).one())
schedule.email_format = SliceEmailReportFormat.visualization
schedule.delivery_type = EmailDeliveryType.attachment
deliver_slice(
schedule.slice_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.email_format,
schedule.deliver_as_group,
)
mtime.sleep.assert_called_once()
driver.screenshot.assert_not_called()
send_email_smtp.assert_called_once()
self.assertEqual(
send_email_smtp.call_args[1]["data"]["screenshot"],
element.screenshot_as_png,
)
self.assertEqual(
files_upload.call_args[1],
{
"channels": "#test_channel",
"file": element.screenshot_as_png,
"initial_comment":
f"\n *Participants*\n\n <http://0.0.0.0:8080/superset/slice/{schedule.slice_id}/|Explore in Superset>\n ",
"title": "[Report] Participants",
},
)
@patch("superset.tasks.slack_util.WebClient.files_upload")
@patch("superset.tasks.schedules.urllib.request.OpenerDirector.open")
@patch("superset.tasks.schedules.urllib.request.urlopen")
@patch("superset.tasks.schedules.send_email_smtp")
def test_deliver_slice_csv_attachment(self, send_email_smtp, mock_open,
mock_urlopen, files_upload):
response = Mock()
mock_open.return_value = response
mock_urlopen.return_value = response
mock_urlopen.return_value.getcode.return_value = 200
response.read.return_value = self.CSV
schedule = (db.session.query(SliceEmailSchedule).filter_by(
id=self.slice_schedule).one())
schedule.email_format = SliceEmailReportFormat.data
schedule.delivery_type = EmailDeliveryType.attachment
deliver_slice(
schedule.slice_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.email_format,
schedule.deliver_as_group,
)
send_email_smtp.assert_called_once()
file_name = __("%(name)s.csv", name=schedule.slice.slice_name)
self.assertEqual(send_email_smtp.call_args[1]["data"][file_name],
self.CSV)
self.assertEqual(
files_upload.call_args[1],
{
"channels": "#test_channel",
"file": self.CSV,
"initial_comment":
f"\n *Participants*\n\n <http://0.0.0.0:8080/superset/slice/{schedule.slice_id}/|Explore in Superset>\n ",
"title": "[Report] Participants",
},
)
@patch("superset.tasks.slack_util.WebClient.files_upload")
@patch("superset.tasks.schedules.urllib.request.urlopen")
@patch("superset.tasks.schedules.urllib.request.OpenerDirector.open")
@patch("superset.tasks.schedules.send_email_smtp")
def test_deliver_slice_csv_inline(self, send_email_smtp, mock_open,
mock_urlopen, files_upload):
response = Mock()
mock_open.return_value = response
mock_urlopen.return_value = response
mock_urlopen.return_value.getcode.return_value = 200
response.read.return_value = self.CSV
schedule = (db.session.query(SliceEmailSchedule).filter_by(
id=self.slice_schedule).one())
schedule.email_format = SliceEmailReportFormat.data
schedule.delivery_type = EmailDeliveryType.inline
deliver_slice(
schedule.slice_id,
schedule.recipients,
schedule.slack_channel,
schedule.delivery_type,
schedule.email_format,
schedule.deliver_as_group,
)
send_email_smtp.assert_called_once()
self.assertIsNone(send_email_smtp.call_args[1]["data"])
self.assertTrue("<table " in send_email_smtp.call_args[0][2])
self.assertEqual(
files_upload.call_args[1],
{
"channels": "#test_channel",
"file": self.CSV,
"initial_comment":
f"\n *Participants*\n\n <http://0.0.0.0:8080/superset/slice/{schedule.slice_id}/|Explore in Superset>\n ",
"title": "[Report] Participants",
},
)
import re
import pytest
from httpx import Response
from tests.utils import override_settings, read_fixture
BASE_URL = "http://qwant.bragi"
NLU_URL = "http://qwant.nlu/"
CLASSIF_URL = "http://qwant.classif"
ES_URL = "http://qwant.es"
FIXTURE_AUTOCOMPLETE = read_fixture(
"fixtures/autocomplete/pavillon_paris.json")
FIXTURE_CLASSIF = {
"pharmacie": read_fixture("fixtures/autocomplete/classif_pharmacy.json"),
"bank": read_fixture("fixtures/autocomplete/classif_bank.json"),
}
def mock_NLU_for(httpx_mock, dataset):
with override_settings({
"NLU_TAGGER_URL": NLU_URL,
"NLU_CLASSIFIER_URL": CLASSIF_URL,
"PJ_ES": ES_URL
}):
nlu_json = read_fixture(f"fixtures/autocomplete/nlu/{dataset}.json")
httpx_mock.post(NLU_URL).respond(json=nlu_json)
for q, data in FIXTURE_CLASSIF.items():
httpx_mock.post(CLASSIF_URL,
json={