def after_init_apps(sender):
from uliweb import orm
from uliweb.core.SimpleFrame import __app_alias__
orm.set_debug_query(uliweb.settings.ORM.DEBUG_LOG)
orm.set_auto_create(uliweb.settings.ORM.AUTO_CREATE)
orm.get_connection(uliweb.settings.ORM.CONNECTION, **uliweb.settings.ORM.CONNECTION_ARGS)
if 'MODELS' in uliweb.settings:
for name, path in uliweb.settings.MODELS.items():
for k, v in __app_alias__.iteritems():
if path.startswith(k):
path = v + path[len(k):]
break
orm.set_model(path, name)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
from uliweb.manage import make_simple_application
from uliweb import orm, settings
# engine = engine_from_config(
# config.get_section(config.config_ini_section),
# prefix='sqlalchemy.',
# poolclass=pool.NullPool)
name = config.get_main_option("engine_name")
make_simple_application(project_dir='.')
target_metadata = orm.get_metadata(name)
connection = orm.get_connection(name).connect()
# connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_server_default=True,
include_object=uliweb_include_object,
# compare_server_default=uliweb_compare_server_default,
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def handle(self, options, global_options, *args):
from uliweb.utils.common import extract_dirs
from uliweb.core.template import template_file
from uliweb.manage import make_simple_application
from uliweb import settings
from sqlalchemy import create_engine, MetaData, Table
from shutil import rmtree
from uliweb.orm import get_connection, engine_manager
alembic_path = os.path.join(global_options.project, 'alembic', options.engine).replace('\\', '/')
#delete alembic path
if os.path.exists(alembic_path):
rmtree(alembic_path, True)
extract_dirs('uliweb.contrib.orm', 'templates/alembic', alembic_path,
verbose=global_options.verbose, replace=True)
make_simple_application(project_dir=global_options.project,
settings_file=global_options.settings,
local_settings_file=global_options.local_settings)
ini_file = os.path.join(alembic_path, 'alembic.ini')
text = template_file(ini_file,
{'connection':engine_manager[options.engine].options.connection_string,
'engine_name':options.engine,
'script_location':alembic_path})
with open(ini_file, 'w') as f:
f.write(text)
#drop old alembic_version table
db = get_connection(engine_name=options.engine)
metadata = MetaData(db)
if db.dialect.has_table(db.connect(), 'alembic_version'):
version = Table('alembic_version', metadata, autoload=True)
version.drop()
def handle(self, options, global_options, *args):
from uliweb import orm
if args:
message = """This command will delete all data of [%s] before loading,
are you sure to load data""" % ','.join(args)
else:
message = """This command will delete whole database before loading,
are you sure to load data"""
get_answer(message)
if not os.path.exists(options.dir):
os.makedirs(options.dir)
engine = get_engine(global_options.apps_dir)
con = orm.get_connection(engine)
for name, t in get_tables(global_options.apps_dir, args, engine=engine, settings_file=global_options.settings, local_settings_file=global_options.local_settings).items():
if global_options.verbose:
print 'Loading %s...' % name
try:
con.begin()
filename = os.path.join(options.dir, name+'.txt')
if options.text:
format = 'txt'
else:
format = None
load_table(t, filename, con, delimiter=options.delimiter,
format=format, encoding=options.encoding)
con.commit()
except:
log.exception("There are something wrong when loading table [%s]" % name)
con.rollback()
def handle(self, options, global_options, *args):
from uliweb.core.SimpleFrame import get_app_dir, Dispatcher
from uliweb import orm
app = Dispatcher(project_dir=global_options.project, start=False)
if not args:
apps_list = self.get_apps(global_options)
else:
apps_list = args
con = orm.get_connection()
for p in apps_list:
if not is_pyfile_exist(get_app_dir(p), 'dbinit'):
continue
m = '%s.dbinit' % p
try:
if global_options.verbose:
print "Processing %s..." % m
con.begin()
mod = __import__(m, {}, {}, [''])
con.commit()
except ImportError:
con.rollback()
log.exception("There are something wrong when importing module [%s]" % m)
def handle(self, options, global_options, *args):
from uliweb.manage import make_simple_application
from uliweb import orm
from getpass import getpass
app = make_simple_application(apps_dir=global_options.apps_dir,
settings_file=global_options.settings, local_settings_file=global_options.local_settings)
db = orm.get_connection()
username = ''
while not username:
username = raw_input("Please enter the super user's name: ")
email = ''
while not email:
email = raw_input("Please enter the email of [%s]: " % username)
password = ''
while not password:
password = getpass("Please enter the password for [%s(%s)]: " % (username, email))
repassword = ''
while not repassword:
repassword = getpass("Please enter the password again: ")
if password != repassword:
print "The password is not matched, can't create super user!"
return
orm.set_dispatch_send(False)
User = orm.get_model('user', options.engine)
user = User(username=username, email=email)
user.set_password(password)
user.is_superuser = True
user.save()
def patch_mysqldb():
from uliweb.orm import get_connection
db = get_connection()
if db.dialect.driver == 'mysqldb':
from MySQLdb.cursors import BaseCursor
setattr(BaseCursor, '_warning_check', _warning_check)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
from uliweb.manage import make_simple_application
from uliweb import orm, settings
# engine = engine_from_config(
# config.get_section(config.config_ini_section),
# prefix='sqlalchemy.',
# poolclass=pool.NullPool)
name = config.get_main_option("engine_name")
make_simple_application(project_dir='.')
target_metadata = orm.get_metadata(name)
connection = orm.get_connection(engine_name=name).connect()
# connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_server_default=True,
include_object=uliweb_include_object,
# compare_server_default=uliweb_compare_server_default,
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def handle(self, options, global_options, *args):
from uliweb.core.SimpleFrame import Dispatcher
from uliweb import orm
from getpass import getpass
app = Dispatcher(project_dir=global_options.project, start=False)
orm.set_auto_create(True)
db = orm.get_connection(app.settings.ORM.CONNECTION)
username = ''
while not username:
username = raw_input("Please enter the super user's name: ")
email = ''
while not email:
email = raw_input("Please enter the email of [%s]: " % username)
password = ''
while not password:
password = getpass("Please enter the password for [%s(%s)]: " % (username, email))
repassword = ''
while not repassword:
repassword = getpass("Please enter the password again: ")
if password != repassword:
print "The password is not matched, can't create super user!"
return
orm.set_dispatch_send(False)
User = orm.get_model('user')
user = User(username=username, email=email)
user.set_password(password)
user.is_superuser = True
user.save()
def handle(self, options, global_options, *args):
from uliweb.utils.common import extract_dirs
from uliweb.core.template import template_file
from uliweb.manage import make_simple_application
from uliweb import settings
from sqlalchemy import create_engine, MetaData, Table
from shutil import rmtree
from uliweb.orm import get_connection, engine_manager
alembic_path = os.path.join(global_options.project, 'alembic', options.engine).replace('\\', '/')
#delete alembic path
if os.path.exists(alembic_path):
rmtree(alembic_path, True)
extract_dirs('uliweb.contrib.orm', 'templates/alembic', alembic_path,
verbose=global_options.verbose, replace=True)
make_simple_application(project_dir=global_options.project,
settings_file=global_options.settings,
local_settings_file=global_options.local_settings)
ini_file = os.path.join(alembic_path, 'alembic.ini')
text = template_file(ini_file,
{'connection':engine_manager[options.engine].options.connection_string,
'engine_name':options.engine,
'script_location':alembic_path})
with open(ini_file, 'w') as f:
f.write(text)
#drop old alembic_version table
db = get_connection(options.engine)
metadata = MetaData(db)
if db.dialect.has_table(db.connect(), 'alembic_version'):
version = Table('alembic_version', metadata, autoload=True)
version.drop()
def patch_mysqldb():
from uliweb.orm import get_connection
db = get_connection()
if db.dialect.driver == 'mysqldb':
from MySQLdb.cursors import BaseCursor
setattr(BaseCursor, '_warning_check', _warning_check)
def load_table_file(table,
filename,
max_workers=4,
message='',
bulk=0,
engine=None,
delete=True):
from uliweb.orm import get_connection
queue = Queue()
result_queue = Queue()
engine = engine or get_connection()
if delete:
table.drop(engine, checkfirst=True)
table.create(engine)
fields = []
with open(filename) as f:
line = f.readline()
fields = line.split()
workers = []
workers.append(
PutWorker(filename=filename,
queue=queue,
max_workers=max_workers,
name='PutWorker'))
for i in range(max_workers):
workers.append(
InsertWorker(table=table,
fields=fields,
queue=queue,
result_queue=result_queue,
name='InsertWorker',
bulk=bulk))
manager = Manager(workers)
b = time()
log.info('Starting! {}'.format(message))
#drop table
# T = reflect_table(table)
# engine = get_connection()
# Model = functions.get_model(table)
# Model.drop(checkfirst=True)
# T.drop(engine, checkfirst=True)
# T.create(engine)
# Model.table.create()
manager.start()
manager.join()
t = 0
for i in range(max_workers):
if not result_queue.empty():
t += result_queue.get()
else:
break
log.info('Finished! Total {} records, time used: {}'.format(t, time() - b))
def process_response(self, request, response):
try:
return response
finally:
CommitAll(close=True)
if self.settings.ORM.CONNECTION_TYPE == 'short':
db = get_connection()
db.dispose()
def process_response(self, request, response):
try:
return response
finally:
CommitAll(close=True)
if self.settings.ORM.CONNECTION_TYPE == 'short':
db = get_connection()
db.dispose()
def get_tables(apps_dir, apps=None, engine=None, import_models=False, settings_file='settings.ini', local_settings_file='local_settings.ini'):
from uliweb.core.SimpleFrame import get_apps, get_app_dir
from uliweb import orm
from sqlalchemy import create_engine
from StringIO import StringIO
if not engine:
engine = get_engine(apps_dir)
_engine = engine[:engine.find('://')+3]
buf = StringIO()
con = create_engine(_engine, strategy='mock', executor=lambda s, p='': buf.write(str(s) + p))
db = orm.get_connection(con)
if import_models:
apps = get_apps(apps_dir, settings_file=settings_file, local_settings_file=local_settings_file)
if apps:
apps_list = apps
else:
apps_list = apps[:]
models = []
for p in apps_list:
if p not in apps:
log.error('Error: Appname %s is not a valid app' % p)
continue
if not is_pyfile_exist(get_app_dir(p), 'models'):
continue
m = '%s.models' % p
try:
mod = __import__(m, {}, {}, [''])
models.append(mod)
except ImportError:
log.exception("There are something wrong when importing module [%s]" % m)
else:
old_models = orm.__models__.keys()
try:
for tablename, m in orm.__models__.items():
orm.get_model(tablename)
except:
print "Problems to models like:", list(set(old_models) ^ set(orm.__models__.keys()))
raise
if apps:
tables = {}
for tablename, m in db.metadata.tables.iteritems():
if hasattr(m, '__appname__') and m.__appname__ in apps:
tables[tablename] = db.metadata.tables[tablename]
else:
tables = db.metadata.tables
return tables
def get_engine(options, global_options):
from uliweb.manage import make_simple_application
settings = {'ORM/DEBUG_LOG':False, 'ORM/AUTO_CREATE':False, 'ORM/AUTO_DOTRANSACTION':False}
app = make_simple_application(apps_dir=global_options.apps_dir,
settings_file=global_options.settings,
local_settings_file=global_options.local_settings,
default_settings=settings)
#because set_auto_set_model will be invoked in orm initicalization, so
#below setting will be executed after Dispatcher started
set_auto_set_model(True)
engine_name = options.engine
engine = get_connection(engine_name=engine_name)
return engine
def get_engine(options, global_options):
from uliweb.manage import make_simple_application
settings = {'ORM/DEBUG_LOG':False, 'ORM/AUTO_CREATE':False, 'ORM/AUTO_DOTRANSACTION':False}
app = make_simple_application(apps_dir=global_options.apps_dir,
settings_file=global_options.settings,
local_settings_file=global_options.local_settings,
default_settings=settings)
#because set_auto_set_model will be invoked in orm initicalization, so
#below setting will be executed after Dispatcher started
set_auto_set_model(True)
engine_name = options.engine
engine = get_connection(engine_name=engine_name)
return engine
def process_response(self, request, response):
from uliweb import response as res
try:
return response
finally:
CommitAll(close=True)
if self.settings.ORM.CONNECTION_TYPE == 'short':
db = get_connection()
db.dispose()
#add post_commit process
if hasattr(response, 'post_commit') and response.post_commit:
response.post_commit()
if hasattr(res, 'post_commit') and res.post_commit:
res.post_commit()
def process_response(self, request, response):
from uliweb import response as res
try:
return response
finally:
CommitAll(close=True)
if self.settings.ORM.CONNECTION_TYPE == 'short':
db = get_connection()
db.dispose()
#add post_commit process
if hasattr(response, 'post_commit') and response.post_commit:
response.post_commit()
if hasattr(res, 'post_commit') and res.post_commit:
res.post_commit()
def load_table_file(table, filename, max_workers=4, message='', bulk=0, engine=None, delete=True):
from uliweb.orm import get_connection
queue = Queue()
result_queue = Queue()
engine = engine or get_connection()
if delete:
table.drop(engine, checkfirst=True)
table.create(engine)
fields = []
with open(filename) as f:
line = f.readline()
fields = line.split()
workers = []
workers.append(PutWorker(filename=filename, queue=queue,
max_workers=max_workers, name='PutWorker'))
for i in range(max_workers):
workers.append(InsertWorker(table=table, fields=fields, queue=queue,
result_queue=result_queue,
name='InsertWorker', bulk=bulk))
manager = Manager(workers)
b = time()
log.info('Starting! {}'.format(message))
#drop table
# T = reflect_table(table)
# engine = get_connection()
# Model = functions.get_model(table)
# Model.drop(checkfirst=True)
# T.drop(engine, checkfirst=True)
# T.create(engine)
# Model.table.create()
manager.start()
manager.join()
t = 0
for i in range(max_workers):
if not result_queue.empty():
t += result_queue.get()
else:
break
log.info('Finished! Total {} records, time used: {}'.format(t, time()-b))
def handle(self, options, global_options, *args):
from uliweb import orm
if args:
message = """This command will delete all data of [%s] before loading,
are you sure to load data""" % ','.join(args)
else:
print "Failed! You should pass one or more tables name."
sys.exit(1)
get_answer(message)
if not os.path.exists(options.dir):
os.makedirs(options.dir)
engine = get_engine(global_options.apps_dir)
con = orm.get_connection(engine)
for name in args:
m = orm.get_model(name)
if not m:
print "Error! Can't find the table %s...Skipped!" % name
continue
t = m.table
if global_options.verbose:
print 'Loading %s...' % name
try:
con.begin()
filename = os.path.join(options.dir, name+'.txt')
if options.text:
format = 'txt'
else:
format = None
load_table(t, filename, con, delimiter=options.delimiter,
format=format, encoding=options.encoding)
con.commit()
except:
log.exception("There are something wrong when loading table [%s]" % name)
con.rollback()
def handle(self, options, global_options, *args):
from uliweb import orm
if len(args) != 2:
print self.print_help(self.prog_name, 'loadtablefile')
sys.exit(1)
if args:
message = """Do you want to delete all data of [%s] before loading, if you choose N, the data will not be deleted""" % args[0]
else:
print "Failed! You should pass one or more tables name."
sys.exit(1)
ans = get_answer(message, answers='Yn', quit='q')
engine = get_engine(global_options.apps_dir)
con = orm.get_connection(engine)
name = args[0]
m = orm.get_model(name)
if not m:
print "Error! Can't find the table %s...Skipped!" % name
t = m.table
if global_options.verbose:
print 'Loading %s...' % name
try:
con.begin()
if options.text:
format = 'txt'
else:
format = None
load_table(t, args[1], con, delimiter=options.delimiter,
format=format, encoding=options.encoding, delete=ans=='Y')
con.commit()
except:
log.exception("There are something wrong when loading table [%s]" % name)
con.rollback()
def handle(self, options, global_options, *args):
from uliweb.manage import make_simple_application
from uliweb import orm
from getpass import getpass
app = make_simple_application(
apps_dir=global_options.apps_dir,
settings_file=global_options.settings,
local_settings_file=global_options.local_settings)
db = orm.get_connection()
username = ''
while not username:
username = raw_input("Please enter the super user's name: ")
email = ''
while not email:
email = raw_input("Please enter the email of [%s]: " % username)
password = ''
while not password:
password = getpass("Please enter the password for [%s(%s)]: " %
(username, email))
repassword = ''
while not repassword:
repassword = getpass("Please enter the password again: ")
if password != repassword:
print "The password is not matched, can't create super user!"
return
orm.set_dispatch_send(False)
User = orm.get_model('user', options.engine)
user = User(username=username, email=email)
user.set_password(password)
user.is_superuser = True
user.save()
def process_exception(self, request, exception):
RollbackAll(close=True)
if self.settings.ORM.CONNECTION_TYPE == 'short':
db = get_connection()
db.dispose()
def process_request(self, request):
from sqlalchemy.pool import NullPool
self.db = get_connection(settings.ORM.CONNECTION, poolclass=NullPool,
**settings.ORM.CONNECTION_ARGS)
def process_exception(self, request, exception):
RollbackAll(close=True)
if self.settings.ORM.CONNECTION_TYPE == 'short':
db = get_connection()
db.dispose()
