from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import NullPool
from superset import app, dataframe, db, results_backend, security_manager
from superset.models.sql_lab import Query
from superset.sql_parse import SupersetQuery
from superset.utils import (
get_celery_app,
json_iso_dttm_ser,
now_as_float,
QueryStatus,
zlib_compress,
)
config = app.config
celery_app = get_celery_app(config)
stats_logger = app.config.get('STATS_LOGGER')
SQLLAB_TIMEOUT = config.get('SQLLAB_ASYNC_TIME_LIMIT_SEC', 600)
class SqlLabException(Exception):
pass
def get_query(query_id, session, retry_count=5):
"""attemps to get the query and retry if it cannot"""
query = None
attempt = 0
while not query and attempt < retry_count:
try:
query = session.query(Query).filter_by(id=query_id).one()
from datetime import datetime
import logging
from subprocess import Popen
from sys import stdout
from colorama import Fore, Style
from flask_migrate import MigrateCommand
from flask_script import Manager
from pathlib2 import Path
import yaml
from superset import app, data, db, dict_import_export_util, security_manager, utils
config = app.config
celery_app = utils.get_celery_app(config)
manager = Manager(app)
manager.add_command('db', MigrateCommand)
@manager.command
def init():
"""Inits the Superset application"""
utils.get_or_create_main_db()
security_manager.sync_role_definitions()
@manager.option(
'-d', '--debug', action='store_true',
help='Start the web server in debug mode')
from datetime import datetime
import logging
from subprocess import Popen
from sys import stdout
from colorama import Fore, Style
from flask_migrate import MigrateCommand
from flask_script import Manager
from pathlib2 import Path
import yaml
from superset import app, db, dict_import_export_util, security, utils
config = app.config
celery_app = utils.get_celery_app(config)
manager = Manager(app)
manager.add_command('db', MigrateCommand)
@manager.command
def init():
"""Inits the Superset application"""
security.sync_role_definitions()
@manager.option(
'-d', '--debug', action='store_true',
help='Start the web server in debug mode')
@manager.option(
from celery.exceptions import SoftTimeLimitExceeded
from contextlib2 import contextmanager
import numpy as np
import pandas as pd
import sqlalchemy
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import NullPool
from superset import app, dataframe, db, results_backend, security_manager, utils
from superset.models.sql_lab import Query
from superset.sql_parse import SupersetQuery
from superset.utils import get_celery_app, QueryStatus
config = app.config
celery_app = get_celery_app(config)
stats_logger = app.config.get('STATS_LOGGER')
SQLLAB_TIMEOUT = config.get('SQLLAB_ASYNC_TIME_LIMIT_SEC', 600)
class SqlLabException(Exception):
pass
def dedup(l, suffix='__'):
"""De-duplicates a list of string by suffixing a counter
Always returns the same number of entries as provided, and always returns
unique values.
>>> print(','.join(dedup(['foo', 'bar', 'bar', 'bar'])))
def __init__(self, app):
self._app = app
self._db_session = None
def get_db(self):
if self._db_session is None:
self._db_session = SQLA(app)
return self._db_session
app_manager = AppManager(app)
# override migrations directory
migrate.directory = BIT_APP_DIR + '/migrations'
app.extensions['migrate'].directory = BIT_APP_DIR + '/migrations'
celery_app = utils.get_celery_app(app)
@manager.command
def beat():
cbeat = celery_beat.beat(app=celery_app)
cbeat.run()
if __name__ == "__main__":
manager.run()
# system
from logging import getLogger
from datetime import datetime
from celery import shared_task
# superset
from superset import db
from superset import app
# from superset import create_app # noqa
from superset.utils import get_celery_app
# locale
from bit.models import EtlTable
# app, db, migrate = create_app()
celery_app = get_celery_app(app)
logger = getLogger(__name__)
from celery.signals import worker_process_init
@worker_process_init.connect
def on_fork_close_session(**kwargs):
if db.session is not None:
db.session.close()
db.engine.dispose()
# @shared_task
@celery_app.task(ignore_results=True)
def run_etl():
