def cache_chart_thumbnail(
url: str,
digest: str,
force: bool = False,
window_size: Optional[WindowSize] = None,
thumb_size: Optional[WindowSize] = None,
) -> None:
with app.app_context(): # type: ignore
if not thumbnail_cache:
logger.warning("No cache set, refusing to compute")
return None
logger.info("Caching chart: %s", url)
screenshot = ChartScreenshot(url, digest)
with session_scope(nullpool=True) as session:
user = security_manager.get_user_by_username(
current_app.config["THUMBNAIL_SELENIUM_USER"], session=session
)
screenshot.compute_and_cache(
user=user,
cache=thumbnail_cache,
force=force,
window_size=window_size,
thumb_size=thumb_size,
)
return None
def get_sql_results( # pylint: disable=too-many-arguments
ctask: Task,
query_id: int,
rendered_query: str,
return_results: bool = True,
store_results: bool = False,
user_name: Optional[str] = None,
start_time: Optional[float] = None,
expand_data: bool = False,
log_params: Optional[Dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
"""Executes the sql query returns the results."""
with session_scope(not ctask.request.called_directly) as session:
try:
return execute_sql_statements(
query_id,
rendered_query,
return_results,
store_results,
user_name,
session=session,
start_time=start_time,
expand_data=expand_data,
log_params=log_params,
)
except Exception as ex: # pylint: disable=broad-except
logger.debug("Query %d: %s", query_id, ex)
stats_logger.incr("error_sqllab_unhandled")
query = get_query(query_id, session)
return handle_query_error(str(ex), query, session)
def scheduler() -> None:
"""
Celery beat main scheduler for reports
"""
if not is_feature_enabled("ALERT_REPORTS"):
return
with session_scope(nullpool=True) as session:
active_schedules = ReportScheduleDAO.find_active(session)
for active_schedule in active_schedules:
for schedule in cron_schedule_window(
active_schedule.crontab, active_schedule.timezone
):
logger.info(
"Scheduling alert %s eta: %s", active_schedule.name, schedule
)
async_options = {"eta": schedule}
if (
active_schedule.working_timeout is not None
and app.config["ALERT_REPORTS_WORKING_TIME_OUT_KILL"]
):
async_options["time_limit"] = (
active_schedule.working_timeout
+ app.config["ALERT_REPORTS_WORKING_TIME_OUT_LAG"]
)
async_options["soft_time_limit"] = (
active_schedule.working_timeout
+ app.config["ALERT_REPORTS_WORKING_SOFT_TIME_OUT_LAG"]
)
execute.apply_async((active_schedule.id, schedule,), **async_options)
def schedule_alerts() -> None:
"""Celery beat job meant to be invoked every minute to check alerts"""
resolution = 0
now = datetime.utcnow()
start_at = now - timedelta(
seconds=300) # process any missed tasks in the past few minutes
stop_at = now + timedelta(seconds=1)
with session_scope(nullpool=True) as session:
schedule_window(ScheduleType.alert, start_at, stop_at, resolution,
session)
def run(self) -> None:
with session_scope(nullpool=True) as session:
self.validate()
for report_schedule in session.query(ReportSchedule).all():
from_date = datetime.utcnow() - timedelta(
days=report_schedule.log_retention)
ReportScheduleDAO.bulk_delete_logs(report_schedule,
from_date,
session=session,
commit=False)
def scheduler() -> None:
"""
Celery beat main scheduler for reports
"""
with session_scope(nullpool=True) as session:
active_schedules = ReportScheduleDAO.find_active(session)
for active_schedule in active_schedules:
for schedule in cron_schedule_window(active_schedule.crontab):
logger.info(
"Scheduling alert %s eta: %s", active_schedule.name, schedule
)
execute.apply_async((active_schedule.id, schedule,), eta=schedule)
def run(self) -> None:
with session_scope(nullpool=True) as session:
try:
self.validate(session=session)
if not self._model:
raise ReportScheduleExecuteUnexpectedError()
ReportScheduleStateMachine(session, self._model,
self._scheduled_dttm).run()
except CommandException as ex:
raise ex
except Exception as ex:
raise ReportScheduleUnexpectedError(str(ex))
def alert() -> None:
"""Run the alert scheduler loop"""
# this command is just for testing purposes
from superset.models.schedules import ScheduleType
from superset.tasks.schedules import schedule_window
click.secho("Processing one alert loop", fg="green")
with session_scope(nullpool=True) as session:
schedule_window(
report_type=ScheduleType.alert,
start_at=datetime.now() - timedelta(1000),
stop_at=datetime.now(),
resolution=6000,
session=session,
)
def schedule_hourly() -> None:
""" Celery beat job meant to be invoked hourly """
if not config["ENABLE_SCHEDULED_EMAIL_REPORTS"]:
logger.info("Scheduled email reports not enabled in config")
return
resolution = config["EMAIL_REPORTS_CRON_RESOLUTION"] * 60
# Get the top of the hour
start_at = datetime.now(tzlocal()).replace(microsecond=0, second=0, minute=0)
stop_at = start_at + timedelta(seconds=3600)
with session_scope(nullpool=True) as session:
schedule_window(ScheduleType.dashboard, start_at, stop_at, resolution, session)
schedule_window(ScheduleType.slice, start_at, stop_at, resolution, session)
def cache_dashboard_thumbnail(url: str,
digest: str,
force: bool = False,
thumb_size: Optional[WindowSize] = None) -> None:
if not thumbnail_cache:
logging.warning("No cache set, refusing to compute")
return
logger.info("Caching dashboard: %s", url)
screenshot = DashboardScreenshot(url, digest)
with session_scope(nullpool=True) as session:
user = security_manager.get_user_by_username(
current_app.config["THUMBNAIL_SELENIUM_USER"], session=session)
screenshot.compute_and_cache(
user=user,
cache=thumbnail_cache,
force=force,
thumb_size=thumb_size,
)
def schedule_email_report(
_task: Task,
report_type: ScheduleType,
schedule_id: int,
recipients: Optional[str] = None,
slack_channel: Optional[str] = None,
) -> None:
model_cls = get_scheduler_model(report_type)
with session_scope(nullpool=True) as session:
schedule = session.query(model_cls).get(schedule_id)
# The user may have disabled the schedule. If so, ignore this
if not schedule or not schedule.active:
logger.info("Ignoring deactivated schedule")
return
recipients = recipients or schedule.recipients
slack_channel = slack_channel or schedule.slack_channel
logger.info(
"Starting report for slack: %s and recipients: %s.",
slack_channel,
recipients,
)
if report_type == ScheduleType.dashboard:
deliver_dashboard(
schedule.dashboard_id,
recipients,
slack_channel,
schedule.delivery_type,
schedule.deliver_as_group,
)
elif report_type == ScheduleType.slice:
deliver_slice(
schedule.slice_id,
recipients,
slack_channel,
schedule.delivery_type,
schedule.email_format,
schedule.deliver_as_group,
session,
)
else:
raise RuntimeError("Unknown report type")
def schedule_alert_query(
_task: Task,
report_type: ScheduleType,
schedule_id: int,
recipients: Optional[str] = None,
slack_channel: Optional[str] = None,
) -> None:
model_cls = get_scheduler_model(report_type)
with session_scope(nullpool=True) as session:
schedule = session.query(model_cls).get(schedule_id)
# The user may have disabled the schedule. If so, ignore this
if not schedule or not schedule.active:
logger.info("Ignoring deactivated alert")
return
if report_type == ScheduleType.alert:
evaluate_alert(schedule.id, schedule.label, session, recipients,
slack_channel)
else:
raise RuntimeError("Unknown report type")
def run(self) -> None:
with session_scope(nullpool=True) as session:
try:
start_dttm = datetime.utcnow()
self.validate(session=session)
if not self._model:
raise ReportScheduleExecuteUnexpectedError()
self.set_state_and_log(session, start_dttm, ReportLogState.WORKING)
# If it's an alert check if the alert is triggered
if self._model.type == ReportScheduleType.ALERT:
if not AlertCommand(self._model).run():
self.set_state_and_log(session, start_dttm, ReportLogState.NOOP)
return
self._send(self._model)
# Log, state and TS
self.set_state_and_log(session, start_dttm, ReportLogState.SUCCESS)
except ReportScheduleAlertGracePeriodError as ex:
self.set_state_and_log(
session, start_dttm, ReportLogState.NOOP, error_message=str(ex)
)
except ReportSchedulePreviousWorkingError as ex:
self.create_log(
session,
start_dttm,
datetime.utcnow(),
state=ReportLogState.ERROR,
error_message=str(ex),
)
session.commit()
raise
except CommandException as ex:
self.set_state_and_log(
session, start_dttm, ReportLogState.ERROR, error_message=str(ex)
)
# We want to actually commit the state and log inside the scope
session.commit()
raise
def run(self) -> None:
with session_scope(nullpool=True) as session:
self.validate()
prune_errors = []
for report_schedule in session.query(ReportSchedule).all():
if report_schedule.log_retention is not None:
from_date = datetime.utcnow() - timedelta(
days=report_schedule.log_retention
)
try:
row_count = ReportScheduleDAO.bulk_delete_logs(
report_schedule, from_date, session=session, commit=False
)
logger.info(
"Deleted %s logs for report schedule id: %s",
str(row_count),
str(report_schedule.id),
)
except DAODeleteFailedError as ex:
prune_errors.append(str(ex))
if prune_errors:
raise ReportSchedulePruneLogError(";".join(prune_errors))
def get_sql_results( # pylint: disable=too-many-arguments
ctask: Task,
query_id: int,
rendered_query: str,
return_results: bool = True,
store_results: bool = False,
user_name: Optional[str] = None,
start_time: Optional[float] = None,
expand_data: bool = False,
log_params: Optional[Dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
"""Executes the sql query returns the results."""
with session_scope(not ctask.request.called_directly) as session:
try:
return execute_sql_statements(
query_id,
rendered_query,
return_results,
store_results,
user_name,
session=session,
start_time=start_time,
expand_data=expand_data,
log_params=log_params,
)
except SoftTimeLimitExceeded as ex:
logger.warning("Query %d: Time limit exceeded", query_id)
logger.debug("Query %d: %s", query_id, ex)
raise SqlLabTimeoutException(
_("SQL Lab timeout. This environment's policy is to kill queries "
"after {} seconds.".format(SQLLAB_TIMEOUT)))
except Exception as ex: # pylint: disable=broad-except
logger.debug("Query %d: %s", query_id, ex)
stats_logger.incr("error_sqllab_unhandled")
query = get_query(query_id, session)
return handle_query_error(str(ex), query, session)
def deliver_dashboard( # pylint: disable=too-many-locals
dashboard_id: int,
recipients: Optional[str],
slack_channel: Optional[str],
delivery_type: EmailDeliveryType,
deliver_as_group: bool,
) -> None:
"""
Given a schedule, delivery the dashboard as an email report
"""
with session_scope(nullpool=True) as session:
dashboard = session.query(Dashboard).filter_by(id=dashboard_id).one()
dashboard_url = _get_url_path("Superset.dashboard",
dashboard_id_or_slug=dashboard.id)
dashboard_url_user_friendly = _get_url_path(
"Superset.dashboard",
user_friendly=True,
dashboard_id_or_slug=dashboard.id)
# Create a driver, fetch the page, wait for the page to render
driver = create_webdriver(session)
window = config["WEBDRIVER_WINDOW"]["dashboard"]
driver.set_window_size(*window)
driver.get(dashboard_url)
time.sleep(EMAIL_PAGE_RENDER_WAIT)
# Set up a function to retry once for the element.
# This is buggy in certain selenium versions with firefox driver
get_element = getattr(driver, "find_element_by_class_name")
element = retry_call(
get_element,
fargs=["grid-container"],
max_tries=2,
interval=EMAIL_PAGE_RENDER_WAIT,
)
try:
screenshot = element.screenshot_as_png
except WebDriverException:
# Some webdrivers do not support screenshots for elements.
# In such cases, take a screenshot of the entire page.
screenshot = driver.screenshot()
finally:
destroy_webdriver(driver)
# Generate the email body and attachments
report_content = _generate_report_content(
delivery_type,
screenshot,
dashboard.dashboard_title,
dashboard_url_user_friendly,
)
subject = __(
"%(prefix)s %(title)s",
prefix=config["EMAIL_REPORTS_SUBJECT_PREFIX"],
title=dashboard.dashboard_title,
)
if recipients:
_deliver_email(
recipients,
deliver_as_group,
subject,
report_content.body,
report_content.data,
report_content.images,
)
if slack_channel:
deliver_slack_msg(
slack_channel,
subject,
report_content.slack_message,
report_content.slack_attachment,
)
