def cache_chart_thumbnail( url: str, digest: str, force: bool = False, window_size: Optional[WindowSize] = None, thumb_size: Optional[WindowSize] = None, ) -> None: with app.app_context(): # type: ignore if not thumbnail_cache: logger.warning("No cache set, refusing to compute") return None logger.info("Caching chart: %s", url) screenshot = ChartScreenshot(url, digest) with session_scope(nullpool=True) as session: user = security_manager.get_user_by_username( current_app.config["THUMBNAIL_SELENIUM_USER"], session=session ) screenshot.compute_and_cache( user=user, cache=thumbnail_cache, force=force, window_size=window_size, thumb_size=thumb_size, ) return None
def get_sql_results( # pylint: disable=too-many-arguments ctask: Task, query_id: int, rendered_query: str, return_results: bool = True, store_results: bool = False, user_name: Optional[str] = None, start_time: Optional[float] = None, expand_data: bool = False, log_params: Optional[Dict[str, Any]] = None, ) -> Optional[Dict[str, Any]]: """Executes the sql query returns the results.""" with session_scope(not ctask.request.called_directly) as session: try: return execute_sql_statements( query_id, rendered_query, return_results, store_results, user_name, session=session, start_time=start_time, expand_data=expand_data, log_params=log_params, ) except Exception as ex: # pylint: disable=broad-except logger.debug("Query %d: %s", query_id, ex) stats_logger.incr("error_sqllab_unhandled") query = get_query(query_id, session) return handle_query_error(str(ex), query, session)
def scheduler() -> None: """ Celery beat main scheduler for reports """ if not is_feature_enabled("ALERT_REPORTS"): return with session_scope(nullpool=True) as session: active_schedules = ReportScheduleDAO.find_active(session) for active_schedule in active_schedules: for schedule in cron_schedule_window( active_schedule.crontab, active_schedule.timezone ): logger.info( "Scheduling alert %s eta: %s", active_schedule.name, schedule ) async_options = {"eta": schedule} if ( active_schedule.working_timeout is not None and app.config["ALERT_REPORTS_WORKING_TIME_OUT_KILL"] ): async_options["time_limit"] = ( active_schedule.working_timeout + app.config["ALERT_REPORTS_WORKING_TIME_OUT_LAG"] ) async_options["soft_time_limit"] = ( active_schedule.working_timeout + app.config["ALERT_REPORTS_WORKING_SOFT_TIME_OUT_LAG"] ) execute.apply_async((active_schedule.id, schedule,), **async_options)
def schedule_alerts() -> None: """Celery beat job meant to be invoked every minute to check alerts""" resolution = 0 now = datetime.utcnow() start_at = now - timedelta( seconds=300) # process any missed tasks in the past few minutes stop_at = now + timedelta(seconds=1) with session_scope(nullpool=True) as session: schedule_window(ScheduleType.alert, start_at, stop_at, resolution, session)
def run(self) -> None: with session_scope(nullpool=True) as session: self.validate() for report_schedule in session.query(ReportSchedule).all(): from_date = datetime.utcnow() - timedelta( days=report_schedule.log_retention) ReportScheduleDAO.bulk_delete_logs(report_schedule, from_date, session=session, commit=False)
def scheduler() -> None: """ Celery beat main scheduler for reports """ with session_scope(nullpool=True) as session: active_schedules = ReportScheduleDAO.find_active(session) for active_schedule in active_schedules: for schedule in cron_schedule_window(active_schedule.crontab): logger.info( "Scheduling alert %s eta: %s", active_schedule.name, schedule ) execute.apply_async((active_schedule.id, schedule,), eta=schedule)
def run(self) -> None: with session_scope(nullpool=True) as session: try: self.validate(session=session) if not self._model: raise ReportScheduleExecuteUnexpectedError() ReportScheduleStateMachine(session, self._model, self._scheduled_dttm).run() except CommandException as ex: raise ex except Exception as ex: raise ReportScheduleUnexpectedError(str(ex))
def alert() -> None: """Run the alert scheduler loop""" # this command is just for testing purposes from superset.models.schedules import ScheduleType from superset.tasks.schedules import schedule_window click.secho("Processing one alert loop", fg="green") with session_scope(nullpool=True) as session: schedule_window( report_type=ScheduleType.alert, start_at=datetime.now() - timedelta(1000), stop_at=datetime.now(), resolution=6000, session=session, )
def schedule_hourly() -> None: """ Celery beat job meant to be invoked hourly """ if not config["ENABLE_SCHEDULED_EMAIL_REPORTS"]: logger.info("Scheduled email reports not enabled in config") return resolution = config["EMAIL_REPORTS_CRON_RESOLUTION"] * 60 # Get the top of the hour start_at = datetime.now(tzlocal()).replace(microsecond=0, second=0, minute=0) stop_at = start_at + timedelta(seconds=3600) with session_scope(nullpool=True) as session: schedule_window(ScheduleType.dashboard, start_at, stop_at, resolution, session) schedule_window(ScheduleType.slice, start_at, stop_at, resolution, session)
def cache_dashboard_thumbnail(url: str, digest: str, force: bool = False, thumb_size: Optional[WindowSize] = None) -> None: if not thumbnail_cache: logging.warning("No cache set, refusing to compute") return logger.info("Caching dashboard: %s", url) screenshot = DashboardScreenshot(url, digest) with session_scope(nullpool=True) as session: user = security_manager.get_user_by_username( current_app.config["THUMBNAIL_SELENIUM_USER"], session=session) screenshot.compute_and_cache( user=user, cache=thumbnail_cache, force=force, thumb_size=thumb_size, )
def schedule_email_report( _task: Task, report_type: ScheduleType, schedule_id: int, recipients: Optional[str] = None, slack_channel: Optional[str] = None, ) -> None: model_cls = get_scheduler_model(report_type) with session_scope(nullpool=True) as session: schedule = session.query(model_cls).get(schedule_id) # The user may have disabled the schedule. If so, ignore this if not schedule or not schedule.active: logger.info("Ignoring deactivated schedule") return recipients = recipients or schedule.recipients slack_channel = slack_channel or schedule.slack_channel logger.info( "Starting report for slack: %s and recipients: %s.", slack_channel, recipients, ) if report_type == ScheduleType.dashboard: deliver_dashboard( schedule.dashboard_id, recipients, slack_channel, schedule.delivery_type, schedule.deliver_as_group, ) elif report_type == ScheduleType.slice: deliver_slice( schedule.slice_id, recipients, slack_channel, schedule.delivery_type, schedule.email_format, schedule.deliver_as_group, session, ) else: raise RuntimeError("Unknown report type")
def schedule_alert_query( _task: Task, report_type: ScheduleType, schedule_id: int, recipients: Optional[str] = None, slack_channel: Optional[str] = None, ) -> None: model_cls = get_scheduler_model(report_type) with session_scope(nullpool=True) as session: schedule = session.query(model_cls).get(schedule_id) # The user may have disabled the schedule. If so, ignore this if not schedule or not schedule.active: logger.info("Ignoring deactivated alert") return if report_type == ScheduleType.alert: evaluate_alert(schedule.id, schedule.label, session, recipients, slack_channel) else: raise RuntimeError("Unknown report type")
def run(self) -> None: with session_scope(nullpool=True) as session: try: start_dttm = datetime.utcnow() self.validate(session=session) if not self._model: raise ReportScheduleExecuteUnexpectedError() self.set_state_and_log(session, start_dttm, ReportLogState.WORKING) # If it's an alert check if the alert is triggered if self._model.type == ReportScheduleType.ALERT: if not AlertCommand(self._model).run(): self.set_state_and_log(session, start_dttm, ReportLogState.NOOP) return self._send(self._model) # Log, state and TS self.set_state_and_log(session, start_dttm, ReportLogState.SUCCESS) except ReportScheduleAlertGracePeriodError as ex: self.set_state_and_log( session, start_dttm, ReportLogState.NOOP, error_message=str(ex) ) except ReportSchedulePreviousWorkingError as ex: self.create_log( session, start_dttm, datetime.utcnow(), state=ReportLogState.ERROR, error_message=str(ex), ) session.commit() raise except CommandException as ex: self.set_state_and_log( session, start_dttm, ReportLogState.ERROR, error_message=str(ex) ) # We want to actually commit the state and log inside the scope session.commit() raise
def run(self) -> None: with session_scope(nullpool=True) as session: self.validate() prune_errors = [] for report_schedule in session.query(ReportSchedule).all(): if report_schedule.log_retention is not None: from_date = datetime.utcnow() - timedelta( days=report_schedule.log_retention ) try: row_count = ReportScheduleDAO.bulk_delete_logs( report_schedule, from_date, session=session, commit=False ) logger.info( "Deleted %s logs for report schedule id: %s", str(row_count), str(report_schedule.id), ) except DAODeleteFailedError as ex: prune_errors.append(str(ex)) if prune_errors: raise ReportSchedulePruneLogError(";".join(prune_errors))
def get_sql_results( # pylint: disable=too-many-arguments ctask: Task, query_id: int, rendered_query: str, return_results: bool = True, store_results: bool = False, user_name: Optional[str] = None, start_time: Optional[float] = None, expand_data: bool = False, log_params: Optional[Dict[str, Any]] = None, ) -> Optional[Dict[str, Any]]: """Executes the sql query returns the results.""" with session_scope(not ctask.request.called_directly) as session: try: return execute_sql_statements( query_id, rendered_query, return_results, store_results, user_name, session=session, start_time=start_time, expand_data=expand_data, log_params=log_params, ) except SoftTimeLimitExceeded as ex: logger.warning("Query %d: Time limit exceeded", query_id) logger.debug("Query %d: %s", query_id, ex) raise SqlLabTimeoutException( _("SQL Lab timeout. This environment's policy is to kill queries " "after {} seconds.".format(SQLLAB_TIMEOUT))) except Exception as ex: # pylint: disable=broad-except logger.debug("Query %d: %s", query_id, ex) stats_logger.incr("error_sqllab_unhandled") query = get_query(query_id, session) return handle_query_error(str(ex), query, session)
def deliver_dashboard( # pylint: disable=too-many-locals dashboard_id: int, recipients: Optional[str], slack_channel: Optional[str], delivery_type: EmailDeliveryType, deliver_as_group: bool, ) -> None: """ Given a schedule, delivery the dashboard as an email report """ with session_scope(nullpool=True) as session: dashboard = session.query(Dashboard).filter_by(id=dashboard_id).one() dashboard_url = _get_url_path("Superset.dashboard", dashboard_id_or_slug=dashboard.id) dashboard_url_user_friendly = _get_url_path( "Superset.dashboard", user_friendly=True, dashboard_id_or_slug=dashboard.id) # Create a driver, fetch the page, wait for the page to render driver = create_webdriver(session) window = config["WEBDRIVER_WINDOW"]["dashboard"] driver.set_window_size(*window) driver.get(dashboard_url) time.sleep(EMAIL_PAGE_RENDER_WAIT) # Set up a function to retry once for the element. # This is buggy in certain selenium versions with firefox driver get_element = getattr(driver, "find_element_by_class_name") element = retry_call( get_element, fargs=["grid-container"], max_tries=2, interval=EMAIL_PAGE_RENDER_WAIT, ) try: screenshot = element.screenshot_as_png except WebDriverException: # Some webdrivers do not support screenshots for elements. # In such cases, take a screenshot of the entire page. screenshot = driver.screenshot() finally: destroy_webdriver(driver) # Generate the email body and attachments report_content = _generate_report_content( delivery_type, screenshot, dashboard.dashboard_title, dashboard_url_user_friendly, ) subject = __( "%(prefix)s %(title)s", prefix=config["EMAIL_REPORTS_SUBJECT_PREFIX"], title=dashboard.dashboard_title, ) if recipients: _deliver_email( recipients, deliver_as_group, subject, report_content.body, report_content.data, report_content.images, ) if slack_channel: deliver_slack_msg( slack_channel, subject, report_content.slack_message, report_content.slack_attachment, )