from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool from superset import app, dataframe, db, results_backend, security_manager from superset.models.sql_lab import Query from superset.sql_parse import SupersetQuery from superset.utils import ( get_celery_app, json_iso_dttm_ser, now_as_float, QueryStatus, zlib_compress, ) config = app.config celery_app = get_celery_app(config) stats_logger = app.config.get('STATS_LOGGER') SQLLAB_TIMEOUT = config.get('SQLLAB_ASYNC_TIME_LIMIT_SEC', 600) class SqlLabException(Exception): pass def get_query(query_id, session, retry_count=5): """attemps to get the query and retry if it cannot""" query = None attempt = 0 while not query and attempt < retry_count: try: query = session.query(Query).filter_by(id=query_id).one()
from datetime import datetime import logging from subprocess import Popen from sys import stdout from colorama import Fore, Style from flask_migrate import MigrateCommand from flask_script import Manager from pathlib2 import Path import yaml from superset import app, data, db, dict_import_export_util, security_manager, utils config = app.config celery_app = utils.get_celery_app(config) manager = Manager(app) manager.add_command('db', MigrateCommand) @manager.command def init(): """Inits the Superset application""" utils.get_or_create_main_db() security_manager.sync_role_definitions() @manager.option( '-d', '--debug', action='store_true', help='Start the web server in debug mode')
from datetime import datetime import logging from subprocess import Popen from sys import stdout from colorama import Fore, Style from flask_migrate import MigrateCommand from flask_script import Manager from pathlib2 import Path import yaml from superset import app, db, dict_import_export_util, security, utils config = app.config celery_app = utils.get_celery_app(config) manager = Manager(app) manager.add_command('db', MigrateCommand) @manager.command def init(): """Inits the Superset application""" security.sync_role_definitions() @manager.option( '-d', '--debug', action='store_true', help='Start the web server in debug mode') @manager.option(
from celery.exceptions import SoftTimeLimitExceeded from contextlib2 import contextmanager import numpy as np import pandas as pd import sqlalchemy from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool from superset import app, dataframe, db, results_backend, security_manager, utils from superset.models.sql_lab import Query from superset.sql_parse import SupersetQuery from superset.utils import get_celery_app, QueryStatus config = app.config celery_app = get_celery_app(config) stats_logger = app.config.get('STATS_LOGGER') SQLLAB_TIMEOUT = config.get('SQLLAB_ASYNC_TIME_LIMIT_SEC', 600) class SqlLabException(Exception): pass def dedup(l, suffix='__'): """De-duplicates a list of string by suffixing a counter Always returns the same number of entries as provided, and always returns unique values. >>> print(','.join(dedup(['foo', 'bar', 'bar', 'bar'])))
def __init__(self, app): self._app = app self._db_session = None def get_db(self): if self._db_session is None: self._db_session = SQLA(app) return self._db_session app_manager = AppManager(app) # override migrations directory migrate.directory = BIT_APP_DIR + '/migrations' app.extensions['migrate'].directory = BIT_APP_DIR + '/migrations' celery_app = utils.get_celery_app(app) @manager.command def beat(): cbeat = celery_beat.beat(app=celery_app) cbeat.run() if __name__ == "__main__": manager.run()
# system from logging import getLogger from datetime import datetime from celery import shared_task # superset from superset import db from superset import app # from superset import create_app # noqa from superset.utils import get_celery_app # locale from bit.models import EtlTable # app, db, migrate = create_app() celery_app = get_celery_app(app) logger = getLogger(__name__) from celery.signals import worker_process_init @worker_process_init.connect def on_fork_close_session(**kwargs): if db.session is not None: db.session.close() db.engine.dispose() # @shared_task @celery_app.task(ignore_results=True) def run_etl():