def install_exposes(self): #EXPOSES format #endpoint = topic #bind_name will be the same with function #expose_name = topic, func #expose_name = topic, func, {args} d = settings.get('EXPOSES', {}) for name, args in d.items(): if not args: continue is_wrong = False if isinstance(args, (tuple, list)): if len(args) == 2: expose(args[0], name=name)(args[1]) elif len(args) == 3: if not isinstance(args[2], dict): is_wrong = True else: expose(args[0], name=name, **args[2])(args[1]) else: is_wrong = True elif isinstance(args, (str, unicode)): expose(args)(name) else: is_wrong = True if is_wrong: log.error('EXPOSES definition should be "endpoint=url" or "name=url, endpoint" or "name=url, endpoint, {"args":value1,...}"') raise UliwebError('EXPOSES definition [%s=%r] is not right' % (name, args))
def install_binds(self): #process DISPATCH hooks #BINDS format #func = topic #bind_name will be the same with function #bind_name = topic, func #bind_name = topic, func, {args} d = settings.get('BINDS', {}) for bind_name, args in d.items(): if not args: continue is_wrong = False if isinstance(args, (tuple, list)): if len(args) == 2: dispatch.bind(args[0])(args[1]) elif len(args) == 3: if not isinstance(args[2], dict): is_wrong = True else: dispatch.bind(args[0], **args[2])(args[1]) else: is_wrong = True elif isinstance(args, (str, unicode)): dispatch.bind(args)(bind_name) else: is_wrong = True if is_wrong: log.error('BINDS definition should be "function=topic" or "bind_name=topic, function" or "bind_name=topic, function, {"args":value1,...}"') raise UliwebError('BINDS definition [%s=%r] is not right' % (bind_name, args))
def install_binds(self): #process DISPATCH hooks #BINDS format #func = topic #bind_name will be the same with function #bind_name = topic, func #bind_name = topic, func, {args} d = settings.get('BINDS', {}) for bind_name, args in d.iteritems(): if not args: continue is_wrong = False if isinstance(args, (tuple, list)): if len(args) == 2: dispatch.bind(args[0])(args[1]) elif len(args) == 3: if not isinstance(args[2], dict): is_wrong = True else: dispatch.bind(args[0], **args[2])(args[1]) else: is_wrong = True elif isinstance(args, (str, unicode)): dispatch.bind(args)(bind_name) else: is_wrong = True if is_wrong: log.error('BINDS definition should be "function=topic" or "bind_name=topic, function" or "bind_name=topic, function, {"args":value1,...}"') raise UliwebError('BINDS definition [%s=%r] is not right' % (bind_name, args))
def install_exposes(self): #EXPOSES format #endpoint = topic #bind_name will be the same with function #expose_name = topic, func #expose_name = topic, func, {args} d = settings.get('EXPOSES', {}) for name, args in d.iteritems(): if not args: continue is_wrong = False if isinstance(args, (tuple, list)): if len(args) == 2: expose(args[0], name=name)(args[1]) elif len(args) == 3: if not isinstance(args[2], dict): is_wrong = True else: expose(args[0], name=name, **args[2])(args[1]) else: is_wrong = True elif isinstance(args, (str, unicode)): expose(args)(name) else: is_wrong = True if is_wrong: log.error('EXPOSES definition should be "endpoint=url" or "name=url, endpoint" or "name=url, endpoint, {"args":value1,...}"') raise UliwebError('EXPOSES definition [%s=%r] is not right' % (name, args))
def load_table(table, filename, con, delimiter=',', format=None, encoding='utf-8', delete=True): import csv from uliweb.utils.date import to_date, to_datetime table = reflect_table(con, table.name) if delete: do_(table.delete()) if not os.path.exists(filename): log.info("The table [%s] data is not existed." % table.name) return f = fin = open(filename, 'rb') try: first_line = f.readline() fields = first_line[1:].strip().split() n = 1 if format: fin = csv.reader(f, delimiter=delimiter) for line in fin: try: n += 1 if not format: line = eval(line.strip()) record = dict(zip(fields, line)) params = {} for c in table.c: if c.name in record: if not format: params[c.name] = record[c.name] else: if record[c.name] == 'NULL': params[c.name] = None else: if isinstance(c.type, String): params[c.name] = unicode( record[c.name], encoding) elif isinstance(c.type, Date): params[c.name] = to_date( to_datetime(record[c.name])) elif isinstance(c.type, DateTime): params[c.name] = to_datetime( record[c.name]) else: params[c.name] = record[c.name] ins = table.insert().values(**params) do_(ins) except: log.error('Error: Line %d' % n) raise finally: f.close()
def use(vars, env, plugin, *args, **kwargs): from uliweb.core.SimpleFrame import get_app_dir from uliweb import application as app, settings from uliweb.utils.common import is_pyfile_exist if plugin in UseNode.__saved_template_plugins_modules__: mod = UseNode.__saved_template_plugins_modules__[plugin] else: #add settings support, only support simple situation #so for complex cases you should still write module #format just like: # #[TEMPLATE_USE] #name = { # 'toplinks':[ # 'myapp/jquery.myapp.{version}.min.js', # ], # 'depends':[xxxx], # 'config':{'version':'UI_CONFIG/test'}, # 'default':{'version':'1.2.0'}, #} # mod = None c = settings.get_var('TEMPLATE_USE/' + plugin) if c: config = c.pop('config', {}) default = c.pop('default', {}) #evaluate config value config = dict([(k, settings.get_var(v, default.get(k, ''))) for k, v in config.items()]) #merge passed arguments config.update(kwargs) for t in ['toplinks', 'bottomlinks']: if t in c: c[t] = [x.format(**config) for x in c[t]] mod = c else: for p in app.apps: if not is_pyfile_exist( os.path.join(get_app_dir(p), 'template_plugins'), plugin): continue module = '.'.join([p, 'template_plugins', plugin]) try: mod = __import__(module, {}, {}, ['']) except ImportError, e: log.exception(e) mod = None if mod: UseNode.__saved_template_plugins_modules__[plugin] = mod else: log.error( "Can't find the [%s] template plugin, please check if you've installed special app already" % plugin) if settings.get_var('TEMPLATE/RAISE_USE_EXCEPTION'): raise UseModuleNotFound( "Can't find the %s template plugin, check if you've installed special app already" % plugin)
def get_tables(apps_dir, apps=None, engine=None, import_models=False, tables=None, settings_file='settings.ini', local_settings_file='local_settings.ini'): from uliweb.core.SimpleFrame import get_apps, get_app_dir from uliweb import orm from StringIO import StringIO engine = orm.engine_manager[engine] e = engine.options['connection_string'] engine_name = e[:e.find('://')+3] buf = StringIO() if import_models: apps = get_apps(apps_dir, settings_file=settings_file, local_settings_file=local_settings_file) if apps: apps_list = apps else: apps_list = apps[:] models = [] for p in apps_list: if p not in apps: log.error('Error: Appname %s is not a valid app' % p) continue if not is_pyfile_exist(get_app_dir(p), 'models'): continue m = '%s.models' % p try: mod = __import__(m, {}, {}, ['']) models.append(mod) except ImportError: log.exception("There are something wrong when importing module [%s]" % m) else: old_models = orm.__models__.keys() try: for tablename, m in orm.__models__.items(): orm.get_model(tablename) except: print "Problems to models like:", list(set(old_models) ^ set(orm.__models__.keys())) raise if apps: t = {} for tablename, m in engine.metadata.tables.items(): if hasattr(m, '__appname__') and m.__appname__ in apps: t[tablename] = engine.metadata.tables[tablename] elif tables: t = {} for tablename, m in engine.metadata.tables.items(): if tablename in tables: t[tablename] = engine.metadata.tables[tablename] else: t = engine.metadata.tables return t
def __call__(self, f): from uliweb.utils.common import safe_import if isinstance(f, (str, unicode)): try: _, f = safe_import(f) except: log.error('Import error: rule=%s' % f) raise self.parse(f) return f
def get_tables(apps_dir, apps=None, engine=None, import_models=False, settings_file='settings.ini', local_settings_file='local_settings.ini'): from uliweb.core.SimpleFrame import get_apps, get_app_dir from uliweb import orm from sqlalchemy import create_engine from StringIO import StringIO if not engine: engine = get_engine(apps_dir) _engine = engine[:engine.find('://')+3] buf = StringIO() con = create_engine(_engine, strategy='mock', executor=lambda s, p='': buf.write(str(s) + p)) db = orm.get_connection(con) if import_models: apps = get_apps(apps_dir, settings_file=settings_file, local_settings_file=local_settings_file) if apps: apps_list = apps else: apps_list = apps[:] models = [] for p in apps_list: if p not in apps: log.error('Error: Appname %s is not a valid app' % p) continue if not is_pyfile_exist(get_app_dir(p), 'models'): continue m = '%s.models' % p try: mod = __import__(m, {}, {}, ['']) models.append(mod) except ImportError: log.exception("There are something wrong when importing module [%s]" % m) else: old_models = orm.__models__.keys() try: for tablename, m in orm.__models__.items(): orm.get_model(tablename) except: print "Problems to models like:", list(set(old_models) ^ set(orm.__models__.keys())) raise if apps: tables = {} for tablename, m in db.metadata.tables.iteritems(): if hasattr(m, '__appname__') and m.__appname__ in apps: tables[tablename] = db.metadata.tables[tablename] else: tables = db.metadata.tables return tables
def init_urls(self): #initialize urls for v in rules.merge_rules(): appname, endpoint, url, kw = v static = kw.pop('static', None) if static: static_views.append(endpoint) try: rules.add_rule(url_map, url, endpoint, **kw) except: log.error("Wrong url url=%s, endpoint=%s" % (url, endpoint)) raise
def load_table(table, filename, con, delimiter=',', format=None, encoding='utf-8', delete=True): import csv from uliweb.utils.date import to_date, to_datetime table = reflect_table(con, table.name) if delete: do_(table.delete()) if not os.path.exists(filename): log.info("The table [%s] data is not existed." % table.name) return f = fin = open(filename, 'rb') try: first_line = f.readline() fields = first_line[1:].strip().split() n = 1 if format: fin = csv.reader(f, delimiter=delimiter) for line in fin: try: n += 1 if not format: line = eval(line.strip()) record = dict(zip(fields, line)) params = {} for c in table.c: if c.name in record: if not format: params[c.name] = record[c.name] else: if record[c.name] == 'NULL': params[c.name] = None else: if isinstance(c.type, String): params[c.name] = unicode(record[c.name], encoding) elif isinstance(c.type, Date): params[c.name] = to_date(to_datetime(record[c.name])) elif isinstance(c.type, DateTime): params[c.name] = to_datetime(record[c.name]) else: params[c.name] = record[c.name] ins = table.insert().values(**params) do_(ins) except: log.error('Error: Line %d' % n) raise finally: f.close()
def use(vars, env, plugin, *args, **kwargs): from uliweb.core.SimpleFrame import get_app_dir from uliweb import application as app, settings from uliweb.utils.common import is_pyfile_exist if plugin in UseNode.__saved_template_plugins_modules__: mod = UseNode.__saved_template_plugins_modules__[plugin] else: #add settings support, only support simple situation #so for complex cases you should still write module #format just like: # #[TEMPLATE_USE] #name = { # 'toplinks':[ # 'myapp/jquery.myapp.{version}.min.js', # ], # 'depends':[xxxx], # 'config':{'version':'UI_CONFIG/test'}, # 'default':{'version':'1.2.0'}, #} # mod = None c = settings.get_var('TEMPLATE_USE/'+plugin) if c: config = c.pop('config', {}) default = c.pop('default', {}) #evaluate config value config = dict([(k, settings.get_var(v, default.get(k, ''))) for k, v in config.items()]) #merge passed arguments config.update(kwargs) for t in ['toplinks', 'bottomlinks']: if t in c: c[t] = [x.format(**config) for x in c[t]] mod = c else: for p in app.apps: if not is_pyfile_exist(os.path.join(get_app_dir(p), 'template_plugins'), plugin): continue module = '.'.join([p, 'template_plugins', plugin]) try: mod = __import__(module, {}, {}, ['']) except ImportError, e: log.exception(e) mod = None if mod: UseNode.__saved_template_plugins_modules__[plugin] = mod else: log.error("Can't find the [%s] template plugin, please check if you've installed special app already" % plugin) if settings.get_var('TEMPLATE/RAISE_USE_EXCEPTION'): raise UseModuleNotFound("Can't find the %s template plugin, check if you've installed special app already" % plugin)
def enum_views(views_path, appname, subfolder=None, pattern=None): if not os.path.exists(views_path): log.error("Can't found the app %s path, please check if the path is right" % appname) return for f in os.listdir(views_path): fname, ext = os.path.splitext(f) if os.path.isfile(os.path.join(views_path, f)) and ext in ['.py', '.pyc', '.pyo'] and fname!='__init__': if pattern: import fnmatch if not fnmatch.fnmatch(f, pattern): continue if subfolder: views.add('.'.join([appname, subfolder, fname])) else: views.add('.'.join([appname, fname]))
def dispatch_hooks(self): #process DISPATCH hooks d = conf.settings.get('BINDS', {}) for func, args in d.iteritems(): if not args: continue is_wrong = False if isinstance(args, (tuple, list)): if len(args) != 2: is_wrong = True if not isinstance(args[1], dict): is_wrong = True if not is_wrong: dispatch.bind(args[0], **args[1])(func) elif isinstance(args, (str, unicode)): dispatch.bind(args)(func) else: is_wrong = True if is_wrong: log.error('BINDS definition should be "function=topic" or "function=topic, {"args":value1,...}"') raise Exception, 'BINDS definition [%s=%r] is not right' % (func, args) d = conf.settings.get('EXPOSES', {}) for name, args in d.iteritems(): if not args: continue is_wrong = False if isinstance(args, (tuple, list)): if len(args) == 2: url, method = args kwargs = {'name':name} elif len(args) == 3: url, method, kwargs = args if not isinstance(kwargs, dict): is_wrong = True else: kwargs['name'] = name else: is_wrong = True if not is_wrong: expose(url, **kwargs)(method) else: is_wrong = True if is_wrong: log.error('EXPOSES definition should be "name=url, endpoint" or "name=url, endpoint, {"args":value1,...}"') raise Exception, 'EXPOSES definition [%s=%r] is not right' % (name, args)
def get_app_dir(app): """ Get an app's directory """ path = __app_dirs__.get(app) if path is not None: return path else: p = app.split('.') try: path = pkg.resource_filename(p[0], '') except ImportError, e: log.error("Can't import app %s" % p[0]) log.exception(e) path = '' if len(p) > 1: path = os.path.join(path, *p[1:]) __app_dirs__[app] = path return path
def init_urls(self): #initialize urls for v in rules.merge_rules(): appname, endpoint, url, kw = v static = kw.pop('static', None) if static: domain_name = 'static' else: domain_name = 'default' domain = self.domains.get(domain_name, {}) url_prefix = domain.get('url_prefix', '') _url = url_prefix + url if static: static_views.append(endpoint) try: rules.add_rule(url_map, _url, endpoint, **kw) except: log.error("Wrong url url=%s, endpoint=%s" % (_url, endpoint)) raise
def use(vars, env, plugin, *args, **kwargs): from uliweb.core.SimpleFrame import get_app_dir from uliweb import application as app, settings if plugin in UseNode.__saved_template_plugins_modules__: mod = UseNode.__saved_template_plugins_modules__[plugin] else: from uliweb.utils.common import is_pyfile_exist mod = None for p in app.apps: if not is_pyfile_exist(os.path.join(get_app_dir(p), 'template_plugins'), plugin): continue module = '.'.join([p, 'template_plugins', plugin]) try: mod = __import__(module, {}, {}, ['']) except ImportError, e: log.exception(e) mod = None if mod: UseNode.__saved_template_plugins_modules__[plugin] = mod else: log.error("Can't find the [%s] template plugin, please check if you've installed special app already" % plugin) if settings.get_var('TEMPLATE/RAISE_USE_EXCEPTION'): raise UseModuleNotFound("Can't find the %s template plugin, check if you've installed special app already" % plugin)
def after_init_apps(sender): """ Check redis version """ from uliweb import settings from uliweb.utils.common import log check = settings.get_var('REDIS/check_version') if check: client = get_redis() try: info = client.info() except Exception as e: log.exception(e) log.error('Redis is not started!') return redis_version = info['redis_version'] version = tuple(map(int, redis_version.split('.'))) op = re_compare_op.search(check) if op: _op = op.group() _v = check[op.end() + 1:].strip() else: _op = '=' _v = check nv = tuple(map(int, _v.split('.'))) if _op == '=': flag = version[:len(nv)] == nv elif _op == '>=': flag = version >= nv elif _op == '>': flag = version > nv elif _op == '<=': flag = version <= nv elif _op == '<': flag = version < nv else: log.error("Can't support operator %s when check redis version" % _op) if not flag: log.error("Redis version %s is not matched what you want %s" % (redis_version, _v))
def after_init_apps(sender): """ Check redis version """ from uliweb import settings from uliweb.utils.common import log check = settings.get_var('REDIS/check_version') if check: client = get_redis() try: info = client.info() except Exception as e: log.exception(e) log.error('Redis is not started!') return redis_version = info['redis_version'] version = tuple(map(int, redis_version.split('.'))) op = re_compare_op.search(check) if op: _op = op.group() _v = check[op.end()+1:].strip() else: _op = '=' _v = check nv = tuple(map(int, _v.split('.'))) if _op == '=': flag = version[:len(nv)] == nv elif _op == '>=': flag = version >= nv elif _op == '>': flag = version > nv elif _op == '<=': flag = version <= nv elif _op == '<': flag = version < nv else: log.error("Can't support operator %s when check redis version" % _op) if not flag: log.error("Redis version %s is not matched what you want %s" % (redis_version, _v))
def find(plugin, *args, **kwargs): from uliweb.core.SimpleFrame import get_app_dir from uliweb import application as app, settings from uliweb.utils.common import is_pyfile_exist key = (plugin, repr(args) + repr(sorted(kwargs.items()))) if key in __use_cached__: return __use_cached__[key] if plugin in __saved_template_plugins_modules__: mod = __saved_template_plugins_modules__[plugin] else: #add settings support, only support simple situation #so for complex cases you should still write module #format just like: # #[TEMPLATE_USE] #name = { # 'toplinks':[ # 'myapp/jquery.myapp.{version}.min.js', # ], # 'depends':[xxxx], # 'config':{'version':'UI_CONFIG/test'}, # 'default':{'version':'1.2.0'}, #} # mod = None c = settings.get_var('TEMPLATE_USE/'+plugin) if c: config = c.pop('config', {}) default = c.pop('default', {}) #evaluate config value config = dict([(k, settings.get_var(v, default.get(k, ''))) for k, v in config.items()]) #merge passed arguments config.update(kwargs) for t in ['toplinks', 'bottomlinks']: if t in c: c[t] = [x.format(**config) for x in c[t]] mod = c else: for p in reversed(app.apps): if not is_pyfile_exist(os.path.join(get_app_dir(p), 'template_plugins'), plugin): continue module = '.'.join([p, 'template_plugins', plugin]) try: mod = __import__(module, fromlist=['*']) break except ImportError as e: log.exception(e) mod = None if mod: __saved_template_plugins_modules__[plugin] = mod else: log.error("Can't find the [%s] template plugin, please check if you've installed special app already" % plugin) raise UseModuleNotFound("Can't find the %s template plugin, check if you've installed special app already" % plugin) #mod maybe an dict if isinstance(mod, dict): v = mod else: v = None call = getattr(mod, 'call', None) call.__name__ = call.__module__ if call: para = inspect.getargspec(call)[0] #test if the funtion is defined as old style if ['app', 'var', 'env'] == para[:3]: warnings.simplefilter('default') warnings.warn("Tmplate plugs call function(%s) should be defined" " as call(*args, **kwargs) not need (app, var, env) any more" % call.__module__, DeprecationWarning) v = call(app, {}, {}, *args, **kwargs) else: v = call(*args, **kwargs) toplinks = [] bottomlinks = [] if v: if 'depends' in v: for _t in v['depends']: if isinstance(_t, str): t, b = find(_t) else: d, kw = _t t, b = find(d, **kw) toplinks.extend(t) bottomlinks.extend(b) if 'toplinks' in v: links = v['toplinks'] if not isinstance(links, (tuple, list)): links = [links] toplinks.extend(links) if 'bottomlinks' in v: links = v['bottomlinks'] if not isinstance(links, (tuple, list)): links = [links] bottomlinks.extend(links) if 'depends_after' in v: for _t in v['depends_after']: if isinstance(_t, str): t, b = use(env, _t) else: d, kw = _t t, b = use(env, d, **kw) toplinks.extend(t) bottomlinks.extend(b) __use_cached__[key] = toplinks, bottomlinks return toplinks, bottomlinks
def get_tables(apps_dir, apps=None, engine=None, import_models=False, tables=None, settings_file='settings.ini', local_settings_file='local_settings.ini'): from uliweb.core.SimpleFrame import get_apps, get_app_dir from uliweb import orm from StringIO import StringIO engine = orm.engine_manager[engine] e = engine.options['connection_string'] engine_name = e[:e.find('://') + 3] buf = StringIO() if import_models: apps = get_apps(apps_dir, settings_file=settings_file, local_settings_file=local_settings_file) if apps: apps_list = apps else: apps_list = apps[:] models = [] for p in apps_list: if p not in apps: log.error('Error: Appname %s is not a valid app' % p) continue if not is_pyfile_exist(get_app_dir(p), 'models'): continue m = '%s.models' % p try: mod = __import__(m, {}, {}, ['']) models.append(mod) except ImportError: log.exception( "There are something wrong when importing module [%s]" % m) else: old_models = orm.__models__.keys() try: for tablename, m in orm.__models__.items(): orm.get_model(tablename) except: print "Problems to models like:", list( set(old_models) ^ set(orm.__models__.keys())) raise if apps: t = {} for tablename, m in engine.metadata.tables.items(): if hasattr(m, '__appname__') and m.__appname__ in apps: t[tablename] = engine.metadata.tables[tablename] elif tables: t = {} for tablename, m in engine.metadata.tables.items(): if tablename in tables: t[tablename] = engine.metadata.tables[tablename] else: t = engine.metadata.tables return t
def load_table(table, filename, con, delimiter=',', format=None, encoding='utf-8', delete=True, bulk=100, engine_name=None): import csv from uliweb.utils.date import to_date, to_datetime if not os.path.exists(filename): return "Skipped (data not found)" table = reflect_table(con, table.name) if delete: do_(table.delete(), engine_name) b = time() bulk = max(1, bulk) f = fin = open(filename, 'rb') try: first_line = f.readline() if first_line.startswith('#'): first_line = first_line[1:] n = 0 count = 0 if format: fields = first_line.strip().split(delimiter) fin = csv.reader(f, delimiter=delimiter) else: fields = first_line.strip().split() buf = [] for line in fin: try: n += 1 count += 1 if not format: line = eval(line.strip()) record = dict(zip(fields, line)) params = {} for c in table.c: if c.name in record: if not format: params[c.name] = record[c.name] else: if record[c.name] == 'NULL': params[c.name] = None else: if isinstance(c.type, String): params[c.name] = unicode( record[c.name], encoding) elif isinstance(c.type, Date): params[c.name] = to_date( to_datetime(record[c.name])) elif isinstance(c.type, DateTime): params[c.name] = to_datetime( record[c.name]) else: params[c.name] = record[c.name] buf.append(params) if count >= bulk: do_(table.insert(), engine_name, args=buf) count = 0 buf = [] except: log.error('Error: Line %d' % n) raise if buf: do_(table.insert(), engine_name, args=buf) return 'OK (%d/%lfs)' % (n, time() - b) finally: f.close()
def find(plugin, *args, **kwargs): from uliweb.core.SimpleFrame import get_app_dir from uliweb import application as app, settings from uliweb.utils.common import is_pyfile_exist, import_attr key = (plugin, repr(args) + repr(sorted(kwargs.items()))) if key in __use_cached__: return __use_cached__[key] if plugin in __saved_template_plugins_modules__: mod = __saved_template_plugins_modules__[plugin] else: #add settings support, only support simple situation #so for complex cases you should still write module #format just like: # #[TEMPLATE_USE] #name = { # 'toplinks':[ # 'myapp/jquery.myapp.{version}.min.js', # ], # 'depends':[xxxx], # 'config':{'version':'UI_CONFIG/test'}, # 'default':{'version':'1.2.0'}, #} # # add toplinks and bottomlinks could be config by a function path, # just like: # #[TEMPLATE_USE] #name = { # 'toplinks':'#{appname}.load_js', # 'config':{'version':'UI_CONFIG/test'}, #} mod = None c = settings.get_var('TEMPLATE_USE/' + plugin) if c: config = c.pop('config', {}) default = c.pop('default', {}) #evaluate config value config = dict([(k, settings.get_var(v, default.get(k, ''))) for k, v in config.items()]) #merge passed arguments config.update(kwargs) for t in ['toplinks', 'bottomlinks']: if t in c: links = c[t] if isinstance(links, (tuple, list)): c[t] = [x.format(**config) for x in c[t]] elif isinstance(links, string_types): m = import_attr(links) c[t] = [x for x in m(**config)] mod = c else: for p in reversed(app.apps): if not is_pyfile_exist( os.path.join(get_app_dir(p), 'template_plugins'), plugin): continue module = '.'.join([p, 'template_plugins', plugin]) try: mod = __import__(module, fromlist=['*']) break except ImportError as e: log.info('Module path is {}'.format( os.path.join(get_app_dir(p), 'template_plugins', plugin))) log.exception(e) mod = None if mod: __saved_template_plugins_modules__[plugin] = mod else: log.error( "Can't find the [%s] template plugin, please check if you've installed special app already" % plugin) raise UseModuleNotFound( "Can't find the %s template plugin, check if you've installed special app already" % plugin) #mod maybe an dict if isinstance(mod, dict): v = mod else: v = None call = getattr(mod, 'call', None) call.__name__ = call.__module__ if call: para = inspect.getargspec(call)[0] #test if the funtion is defined as old style if ['app', 'var', 'env'] == para[:3]: warnings.simplefilter('default') warnings.warn( "Tmplate plugs call function(%s) should be defined" " as call(*args, **kwargs) not need (app, var, env) any more" % call.__module__, DeprecationWarning) v = call(app, {}, {}, *args, **kwargs) else: v = call(*args, **kwargs) toplinks = [] bottomlinks = [] if v: if 'depends' in v: for _t in v['depends']: if isinstance(_t, str): t, b = find(_t) else: d, kw = _t t, b = find(d, **kw) toplinks.extend(t) bottomlinks.extend(b) if 'toplinks' in v: links = v['toplinks'] if not isinstance(links, (tuple, list)): links = [links] toplinks.extend(links) if 'bottomlinks' in v: links = v['bottomlinks'] if not isinstance(links, (tuple, list)): links = [links] bottomlinks.extend(links) if 'depends_after' in v: for _t in v['depends_after']: if isinstance(_t, str): t, b = use(env, _t) else: d, kw = _t t, b = use(env, d, **kw) toplinks.extend(t) bottomlinks.extend(b) __use_cached__[key] = toplinks, bottomlinks return toplinks, bottomlinks
def load_table(table, filename, con, delimiter=',', format=None, encoding='utf-8', delete=True, bulk=100, engine_name=None): import csv from uliweb.utils.date import to_date, to_datetime if not os.path.exists(filename): return "Skipped (data not found)" table = reflect_table(con, table.name) if delete: table.drop(con) table.create(con) # do_(table.drop(), engine_name) # do_(table.create(), engine_name) b = time() bulk = max(1, bulk) f = fin = open(filename, 'rb') try: first_line = f.readline() if first_line.startswith('#'): first_line = first_line[1:] n = 0 count = 0 if format: fields = first_line.strip().split(delimiter) fin = csv.reader(f, delimiter=delimiter) else: fields = first_line.strip().split() buf = [] for line in fin: try: n += 1 count += 1 if not format: line = eval(line.strip()) record = dict(zip(fields, line)) params = {} for c in table.c: if c.name in record: if not format: params[c.name] = record[c.name] else: if record[c.name] == 'NULL': params[c.name] = None else: if isinstance(c.type, String): params[c.name] = unicode(record[c.name], encoding) elif isinstance(c.type, Date): params[c.name] = to_date(to_datetime(record[c.name])) elif isinstance(c.type, DateTime): params[c.name] = to_datetime(record[c.name]) else: params[c.name] = record[c.name] buf.append(params) if count >= bulk: do_(table.insert(), engine_name, args=buf) count = 0 buf = [] except: log.error('Error: Line %d of %s' % (n, filename)) raise if buf: do_(table.insert(), engine_name, args=buf) return 'OK (%d/%lfs)' % (n, time()-b) finally: f.close()