Ejemplo n.º 1
0
def after_init_apps(sender):
    from uliweb import orm
    from uliweb.core.SimpleFrame import __app_alias__
    
    orm.set_debug_query(uliweb.settings.ORM.DEBUG_LOG)
    orm.set_auto_create(uliweb.settings.ORM.AUTO_CREATE)
    orm.get_connection(uliweb.settings.ORM.CONNECTION, **uliweb.settings.ORM.CONNECTION_ARGS)

    if 'MODELS' in uliweb.settings:
        for name, path in uliweb.settings.MODELS.items():
            for k, v in __app_alias__.iteritems():
                if path.startswith(k):
                    path = v + path[len(k):]
                    break
            orm.set_model(path, name)
Ejemplo n.º 2
0
def run_migrations_online():
    """Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context.
    
    """
    from uliweb.manage import make_simple_application
    from uliweb import orm, settings

    #    engine = engine_from_config(
    #                config.get_section(config.config_ini_section),
    #                prefix='sqlalchemy.',
    #                poolclass=pool.NullPool)

    name = config.get_main_option("engine_name")
    make_simple_application(project_dir='.')
    target_metadata = orm.get_metadata(name)
    connection = orm.get_connection(name).connect()
    #    connection = engine.connect()

    context.configure(
        connection=connection,
        target_metadata=target_metadata,
        compare_server_default=True,
        include_object=uliweb_include_object,
        #                compare_server_default=uliweb_compare_server_default,
    )

    try:
        with context.begin_transaction():
            context.run_migrations()
    finally:
        connection.close()
Ejemplo n.º 3
0
 def handle(self, options, global_options, *args):
     from uliweb.utils.common import extract_dirs
     from uliweb.core.template import template_file
     from uliweb.manage import make_simple_application
     from uliweb import settings
     from sqlalchemy import create_engine, MetaData, Table
     from shutil import rmtree
     from uliweb.orm import get_connection, engine_manager
     
     alembic_path = os.path.join(global_options.project, 'alembic', options.engine).replace('\\', '/')
     #delete alembic path
     if os.path.exists(alembic_path):
         rmtree(alembic_path, True)
     extract_dirs('uliweb.contrib.orm', 'templates/alembic', alembic_path, 
         verbose=global_options.verbose, replace=True)
     make_simple_application(project_dir=global_options.project,
         settings_file=global_options.settings,
         local_settings_file=global_options.local_settings)
     ini_file = os.path.join(alembic_path, 'alembic.ini')
     text = template_file(ini_file, 
         {'connection':engine_manager[options.engine].options.connection_string, 
         'engine_name':options.engine,
         'script_location':alembic_path})
     
     with open(ini_file, 'w') as f:
         f.write(text)
         
     #drop old alembic_version table
     db = get_connection(engine_name=options.engine)
     metadata = MetaData(db)
     if db.dialect.has_table(db.connect(), 'alembic_version'):
         version = Table('alembic_version', metadata, autoload=True) 
         version.drop()
Ejemplo n.º 4
0
    def handle(self, options, global_options, *args):
        from uliweb import orm
        
        if args:
            message = """This command will delete all data of [%s] before loading, 
are you sure to load data""" % ','.join(args)
        else:
            message = """This command will delete whole database before loading, 
are you sure to load data"""

        get_answer(message)

        if not os.path.exists(options.dir):
            os.makedirs(options.dir)
        
        engine = get_engine(global_options.apps_dir)
        con = orm.get_connection(engine)

        for name, t in get_tables(global_options.apps_dir, args, engine=engine, settings_file=global_options.settings, local_settings_file=global_options.local_settings).items():
            if global_options.verbose:
                print 'Loading %s...' % name
            try:
                con.begin()
                filename = os.path.join(options.dir, name+'.txt')
                if options.text:
                    format = 'txt'
                else:
                    format = None
                load_table(t, filename, con, delimiter=options.delimiter, 
                    format=format, encoding=options.encoding)
                con.commit()
            except:
                log.exception("There are something wrong when loading table [%s]" % name)
                con.rollback()
Ejemplo n.º 5
0
    def handle(self, options, global_options, *args):
        from uliweb.core.SimpleFrame import get_app_dir, Dispatcher
        from uliweb import orm

        app = Dispatcher(project_dir=global_options.project, start=False)

        if not args:
            apps_list = self.get_apps(global_options)
        else:
            apps_list = args
        
        con = orm.get_connection()
        
        for p in apps_list:
            if not is_pyfile_exist(get_app_dir(p), 'dbinit'):
                continue
            m = '%s.dbinit' % p
            try:
                if global_options.verbose:
                    print "Processing %s..." % m
                con.begin()
                mod = __import__(m, {}, {}, [''])
                con.commit()
            except ImportError:
                con.rollback()
                log.exception("There are something wrong when importing module [%s]" % m)
Ejemplo n.º 6
0
 def handle(self, options, global_options, *args):
     from uliweb.manage import make_simple_application
     from uliweb import orm
     from getpass import getpass
     
     app = make_simple_application(apps_dir=global_options.apps_dir, 
         settings_file=global_options.settings, local_settings_file=global_options.local_settings)
     db = orm.get_connection()
     
     username = ''
     while not username:
         username = raw_input("Please enter the super user's name: ")
     email = ''
     while not email:
         email = raw_input("Please enter the email of [%s]: " % username)
         
     password = ''
     while not password:
         password = getpass("Please enter the password for [%s(%s)]: " % (username, email))
     repassword = ''
     while not repassword:
         repassword = getpass("Please enter the password again: ")
     
     if password != repassword:
         print "The password is not matched, can't create super user!"
         return
     
     orm.set_dispatch_send(False)
     
     User = orm.get_model('user', options.engine)
     user = User(username=username, email=email)
     user.set_password(password)
     user.is_superuser = True
     user.save()
Ejemplo n.º 7
0
def patch_mysqldb():
    from uliweb.orm import get_connection

    db = get_connection()
    if db.dialect.driver == 'mysqldb':
        from MySQLdb.cursors import BaseCursor
        setattr(BaseCursor, '_warning_check', _warning_check)
Ejemplo n.º 8
0
def run_migrations_online():
    """Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context.
    
    """
    from uliweb.manage import make_simple_application
    from uliweb import orm, settings

#    engine = engine_from_config(
#                config.get_section(config.config_ini_section), 
#                prefix='sqlalchemy.', 
#                poolclass=pool.NullPool)

    name = config.get_main_option("engine_name")
    make_simple_application(project_dir='.')
    target_metadata = orm.get_metadata(name)
    connection = orm.get_connection(engine_name=name).connect()
#    connection = engine.connect()
    
    context.configure(
                connection=connection, 
                target_metadata=target_metadata,
                compare_server_default=True,
                include_object=uliweb_include_object,
#                compare_server_default=uliweb_compare_server_default,
                )

    try:
        with context.begin_transaction():
            context.run_migrations()
    finally:
        connection.close()
Ejemplo n.º 9
0
 def handle(self, options, global_options, *args):
     from uliweb.core.SimpleFrame import Dispatcher
     from uliweb import orm
     from getpass import getpass
     
     app = Dispatcher(project_dir=global_options.project, start=False)
     orm.set_auto_create(True)
     db = orm.get_connection(app.settings.ORM.CONNECTION)
     
     username = ''
     while not username:
         username = raw_input("Please enter the super user's name: ")
     email = ''
     while not email:
         email = raw_input("Please enter the email of [%s]: " % username)
         
     password = ''
     while not password:
         password = getpass("Please enter the password for [%s(%s)]: " % (username, email))
     repassword = ''
     while not repassword:
         repassword = getpass("Please enter the password again: ")
     
     if password != repassword:
         print "The password is not matched, can't create super user!"
         return
     
     orm.set_dispatch_send(False)
     
     User = orm.get_model('user')
     user = User(username=username, email=email)
     user.set_password(password)
     user.is_superuser = True
     user.save()
Ejemplo n.º 10
0
 def handle(self, options, global_options, *args):
     from uliweb.utils.common import extract_dirs
     from uliweb.core.template import template_file
     from uliweb.manage import make_simple_application
     from uliweb import settings
     from sqlalchemy import create_engine, MetaData, Table
     from shutil import rmtree
     from uliweb.orm import get_connection, engine_manager
     
     alembic_path = os.path.join(global_options.project, 'alembic', options.engine).replace('\\', '/')
     #delete alembic path
     if os.path.exists(alembic_path):
         rmtree(alembic_path, True)
     extract_dirs('uliweb.contrib.orm', 'templates/alembic', alembic_path, 
         verbose=global_options.verbose, replace=True)
     make_simple_application(project_dir=global_options.project,
         settings_file=global_options.settings,
         local_settings_file=global_options.local_settings)
     ini_file = os.path.join(alembic_path, 'alembic.ini')
     text = template_file(ini_file, 
         {'connection':engine_manager[options.engine].options.connection_string, 
         'engine_name':options.engine,
         'script_location':alembic_path})
     
     with open(ini_file, 'w') as f:
         f.write(text)
         
     #drop old alembic_version table
     db = get_connection(options.engine)
     metadata = MetaData(db)
     if db.dialect.has_table(db.connect(), 'alembic_version'):
         version = Table('alembic_version', metadata, autoload=True) 
         version.drop()
Ejemplo n.º 11
0
def patch_mysqldb():
    from uliweb.orm import get_connection

    db = get_connection()
    if db.dialect.driver == 'mysqldb':
        from MySQLdb.cursors import BaseCursor
        setattr(BaseCursor, '_warning_check', _warning_check)
Ejemplo n.º 12
0
def load_table_file(table,
                    filename,
                    max_workers=4,
                    message='',
                    bulk=0,
                    engine=None,
                    delete=True):
    from uliweb.orm import get_connection

    queue = Queue()
    result_queue = Queue()

    engine = engine or get_connection()
    if delete:
        table.drop(engine, checkfirst=True)
        table.create(engine)

    fields = []
    with open(filename) as f:
        line = f.readline()
        fields = line.split()

    workers = []
    workers.append(
        PutWorker(filename=filename,
                  queue=queue,
                  max_workers=max_workers,
                  name='PutWorker'))
    for i in range(max_workers):
        workers.append(
            InsertWorker(table=table,
                         fields=fields,
                         queue=queue,
                         result_queue=result_queue,
                         name='InsertWorker',
                         bulk=bulk))
    manager = Manager(workers)

    b = time()
    log.info('Starting! {}'.format(message))

    #drop table
    # T = reflect_table(table)
    # engine = get_connection()
    # Model = functions.get_model(table)
    # Model.drop(checkfirst=True)
    # T.drop(engine, checkfirst=True)
    # T.create(engine)
    # Model.table.create()
    manager.start()
    manager.join()
    t = 0
    for i in range(max_workers):
        if not result_queue.empty():
            t += result_queue.get()
        else:
            break

    log.info('Finished! Total {} records, time used: {}'.format(t, time() - b))
Ejemplo n.º 13
0
 def process_response(self, request, response):
     try:
         return response
     finally:
         CommitAll(close=True)
         if self.settings.ORM.CONNECTION_TYPE == 'short':
             db = get_connection()
             db.dispose()
Ejemplo n.º 14
0
 def process_response(self, request, response):
     try:
         return response
     finally:
         CommitAll(close=True)
         if self.settings.ORM.CONNECTION_TYPE == 'short':
             db = get_connection()
             db.dispose()
Ejemplo n.º 15
0
def get_tables(apps_dir, apps=None, engine=None, import_models=False, settings_file='settings.ini', local_settings_file='local_settings.ini'):
    from uliweb.core.SimpleFrame import get_apps, get_app_dir
    from uliweb import orm
    from sqlalchemy import create_engine
    from StringIO import StringIO
    
    if not engine:
        engine = get_engine(apps_dir)
    
    _engine = engine[:engine.find('://')+3]
    
    buf = StringIO()
    
    con = create_engine(_engine, strategy='mock', executor=lambda s, p='': buf.write(str(s) + p))
    db = orm.get_connection(con)
    
    if import_models:
        apps = get_apps(apps_dir, settings_file=settings_file, local_settings_file=local_settings_file)
        if apps:
            apps_list = apps
        else:
            apps_list = apps[:]
        models = []
        for p in apps_list:
            if p not in apps:
                log.error('Error: Appname %s is not a valid app' % p)
                continue
            if not is_pyfile_exist(get_app_dir(p), 'models'):
                continue
            m = '%s.models' % p
            try:
                mod = __import__(m, {}, {}, [''])
                models.append(mod)
            except ImportError:
                log.exception("There are something wrong when importing module [%s]" % m)
        
    else:
        old_models = orm.__models__.keys()
        try:
            for tablename, m in orm.__models__.items():
                orm.get_model(tablename)
        except:
            print "Problems to models like:", list(set(old_models) ^ set(orm.__models__.keys()))
            raise
            
    if apps:
        tables = {}
        for tablename, m in db.metadata.tables.iteritems():
            if hasattr(m, '__appname__') and m.__appname__ in apps:
                tables[tablename] = db.metadata.tables[tablename]
    else:
        tables = db.metadata.tables
                
    return tables
Ejemplo n.º 16
0
def get_engine(options, global_options):
    from uliweb.manage import make_simple_application
    settings = {'ORM/DEBUG_LOG':False, 'ORM/AUTO_CREATE':False, 'ORM/AUTO_DOTRANSACTION':False}
    app = make_simple_application(apps_dir=global_options.apps_dir, 
        settings_file=global_options.settings, 
        local_settings_file=global_options.local_settings,
        default_settings=settings)
    #because set_auto_set_model will be invoked in orm initicalization, so
    #below setting will be executed after Dispatcher started
    set_auto_set_model(True)
    engine_name = options.engine
    engine = get_connection(engine_name=engine_name)
    return engine
Ejemplo n.º 17
0
def get_engine(options, global_options):
    from uliweb.manage import make_simple_application
    settings = {'ORM/DEBUG_LOG':False, 'ORM/AUTO_CREATE':False, 'ORM/AUTO_DOTRANSACTION':False}
    app = make_simple_application(apps_dir=global_options.apps_dir, 
        settings_file=global_options.settings, 
        local_settings_file=global_options.local_settings,
        default_settings=settings)
    #because set_auto_set_model will be invoked in orm initicalization, so
    #below setting will be executed after Dispatcher started
    set_auto_set_model(True)
    engine_name = options.engine
    engine = get_connection(engine_name=engine_name)
    return engine
Ejemplo n.º 18
0
    def process_response(self, request, response):
        from uliweb import response as res
        try:
            return response
        finally:
            CommitAll(close=True)
            if self.settings.ORM.CONNECTION_TYPE == 'short':
                db = get_connection()
                db.dispose()

            #add post_commit process
            if hasattr(response, 'post_commit') and response.post_commit:
                response.post_commit()

            if hasattr(res, 'post_commit') and res.post_commit:
                res.post_commit()
Ejemplo n.º 19
0
 def process_response(self, request, response):
     from uliweb import response as res
     try:
         return response
     finally:
         CommitAll(close=True)
         if self.settings.ORM.CONNECTION_TYPE == 'short':
             db = get_connection()
             db.dispose()
             
         #add post_commit process
         if hasattr(response, 'post_commit') and response.post_commit:
             response.post_commit()
             
         if hasattr(res, 'post_commit') and res.post_commit:
             res.post_commit()
Ejemplo n.º 20
0
def load_table_file(table, filename, max_workers=4, message='', bulk=0, engine=None, delete=True):
    from uliweb.orm import get_connection

    queue = Queue()
    result_queue = Queue()

    engine = engine or get_connection()
    if delete:
        table.drop(engine, checkfirst=True)
        table.create(engine)

    fields = []
    with open(filename) as f:
        line = f.readline()
        fields = line.split()


    workers = []
    workers.append(PutWorker(filename=filename, queue=queue,
                             max_workers=max_workers, name='PutWorker'))
    for i in range(max_workers):
        workers.append(InsertWorker(table=table, fields=fields, queue=queue,
                                    result_queue=result_queue,
                                    name='InsertWorker', bulk=bulk))
    manager = Manager(workers)

    b = time()
    log.info('Starting! {}'.format(message))

    #drop table
    # T = reflect_table(table)
    # engine = get_connection()
    # Model = functions.get_model(table)
    # Model.drop(checkfirst=True)
    # T.drop(engine, checkfirst=True)
    # T.create(engine)
    # Model.table.create()
    manager.start()
    manager.join()
    t = 0
    for i in range(max_workers):
        if not result_queue.empty():
            t += result_queue.get()
        else:
            break

    log.info('Finished! Total {} records, time used: {}'.format(t, time()-b))
Ejemplo n.º 21
0
    def handle(self, options, global_options, *args):
        from uliweb import orm
        
        if args:
            message = """This command will delete all data of [%s] before loading, 
are you sure to load data""" % ','.join(args)
        else:
            print "Failed! You should pass one or more tables name."
            sys.exit(1)

        get_answer(message)

        if not os.path.exists(options.dir):
            os.makedirs(options.dir)
        
        engine = get_engine(global_options.apps_dir)
        con = orm.get_connection(engine)

        for name in args:
            m = orm.get_model(name)
            if not m:
                print "Error! Can't find the table %s...Skipped!" % name
                continue
            t = m.table
            if global_options.verbose:
                print 'Loading %s...' % name
            try:
                con.begin()
                filename = os.path.join(options.dir, name+'.txt')
                if options.text:
                    format = 'txt'
                else:
                    format = None
                load_table(t, filename, con, delimiter=options.delimiter, 
                    format=format, encoding=options.encoding)
                con.commit()
            except:
                log.exception("There are something wrong when loading table [%s]" % name)
                con.rollback()
Ejemplo n.º 22
0
    def handle(self, options, global_options, *args):
        from uliweb import orm
        
        if len(args) != 2:
            print self.print_help(self.prog_name, 'loadtablefile')
            sys.exit(1)
            
        if args:
            message = """Do you want to delete all data of [%s] before loading, if you choose N, the data will not be deleted""" % args[0]
        else:
            print "Failed! You should pass one or more tables name."
            sys.exit(1)

        ans = get_answer(message, answers='Yn', quit='q')

        engine = get_engine(global_options.apps_dir)
        con = orm.get_connection(engine)

        name = args[0]
        m = orm.get_model(name)
        if not m:
            print "Error! Can't find the table %s...Skipped!" % name

        t = m.table
        if global_options.verbose:
            print 'Loading %s...' % name
        try:
            con.begin()
            if options.text:
                format = 'txt'
            else:
                format = None
            load_table(t, args[1], con, delimiter=options.delimiter, 
                format=format, encoding=options.encoding, delete=ans=='Y')
            con.commit()
        except:
            log.exception("There are something wrong when loading table [%s]" % name)
            con.rollback()
Ejemplo n.º 23
0
    def handle(self, options, global_options, *args):
        from uliweb.manage import make_simple_application
        from uliweb import orm
        from getpass import getpass

        app = make_simple_application(
            apps_dir=global_options.apps_dir,
            settings_file=global_options.settings,
            local_settings_file=global_options.local_settings)
        db = orm.get_connection()

        username = ''
        while not username:
            username = raw_input("Please enter the super user's name: ")
        email = ''
        while not email:
            email = raw_input("Please enter the email of [%s]: " % username)

        password = ''
        while not password:
            password = getpass("Please enter the password for [%s(%s)]: " %
                               (username, email))
        repassword = ''
        while not repassword:
            repassword = getpass("Please enter the password again: ")

        if password != repassword:
            print "The password is not matched, can't create super user!"
            return

        orm.set_dispatch_send(False)

        User = orm.get_model('user', options.engine)
        user = User(username=username, email=email)
        user.set_password(password)
        user.is_superuser = True
        user.save()
Ejemplo n.º 24
0
 def process_exception(self, request, exception):
     RollbackAll(close=True)
     if self.settings.ORM.CONNECTION_TYPE == 'short':
         db = get_connection()
         db.dispose()
Ejemplo n.º 25
0
 def process_request(self, request):
     from sqlalchemy.pool import NullPool
     self.db = get_connection(settings.ORM.CONNECTION, poolclass=NullPool,
         **settings.ORM.CONNECTION_ARGS)
Ejemplo n.º 26
0
 def process_exception(self, request, exception):
     RollbackAll(close=True)
     if self.settings.ORM.CONNECTION_TYPE == 'short':
         db = get_connection()
         db.dispose()