def init_static_combine(): from uliweb import settings from hashlib import md5 import os PLUGINS = settings.get_var("TEMPLATE_GULP") d = {} if settings.get_var('STATIC_COMBINE_CONFIG/enable', False): # for k, v in settings.get('STATIC_COMBINE', {}).items(): # key = '_cmb_' + md5(''.join(v)).hexdigest() + os.path.splitext(v[0])[1] # d[key] = v from uliweb.contrib.template.tags import find for k, v in PLUGINS.items(): js_list = [] css_list = [] for x in v: s = find(x) m = s[0] + s[1] for i in m: if not i.startswith('<!--'): e = os.path.splitext(i)[1] if e == ".css": css_list.append(i) elif e == ".js": js_list.append(i) if js_list: d[k+".js"] = js_list if css_list: d[k + ".css"] = css_list return d
def get_sequence(key, default=1, step=1, retry_times=None, retry_waittime=None): from uliweb.orm import SaveError from uliweb import settings assert step > 0 and default > 0 Sequence = functions.get_model('sequence') i = 0 waittime = retry_waittime or settings.get_var('SEQUENCE/retry_waittime', 0.05) retry_times = retry_times or settings.get_var('SEQUENCE/retry_times', 3) while 1: try: row = Sequence.get(Sequence.c.key == key) if row: row.value = row.value + step row.save(version=True) else: row = Sequence(key=key, value=(default + step - 1)) row.save() break except SaveError: i += 1 if i == retry_times: raise else: sleep(waittime) return row.value
def __init__(self, application, settings): from datetime import timedelta self.options = dict(settings.get('SESSION_STORAGE', {})) self.options['data_dir'] = application_path(self.options['data_dir']) if 'url' not in self.options: _url = (settings.get_var('ORM/CONNECTION', '') or settings.get_var('ORM/CONNECTIONS', {}).get( 'default', {}).get('CONNECTION', '')) if _url: self.options['url'] = _url #process Session options self.remember_me_timeout = settings.SESSION.remember_me_timeout self.session_storage_type = settings.SESSION.type self.timeout = settings.SESSION.timeout Session.force = settings.SESSION.force #process Cookie options SessionCookie.default_domain = settings.SESSION_COOKIE.domain SessionCookie.default_path = settings.SESSION_COOKIE.path SessionCookie.default_secure = settings.SESSION_COOKIE.secure SessionCookie.default_cookie_id = settings.SESSION_COOKIE.cookie_id if isinstance(settings.SESSION_COOKIE.timeout, int): timeout = timedelta(seconds=settings.SESSION_COOKIE.timeout) else: timeout = settings.SESSION_COOKIE.timeout SessionCookie.default_expiry_time = timeout
def __init__(self, application, settings): from datetime import timedelta self.options = dict(settings.get('SESSION_STORAGE', {})) self.options['data_dir'] = application_path(self.options['data_dir']) if 'url' not in self.options: _url = (settings.get_var('ORM/CONNECTION', '') or settings.get_var('ORM/CONNECTIONS', {}).get('default', {}).get('CONNECTION', '')) if _url: self.options['url'] = _url #process Session options self.remember_me_timeout = settings.SESSION.remember_me_timeout self.session_storage_type = settings.SESSION.type self.timeout = settings.SESSION.timeout Session.force = settings.SESSION.force #process Cookie options SessionCookie.default_domain = settings.SESSION_COOKIE.domain SessionCookie.default_path = settings.SESSION_COOKIE.path SessionCookie.default_secure = settings.SESSION_COOKIE.secure SessionCookie.default_cookie_id = settings.SESSION_COOKIE.cookie_id if isinstance(settings.SESSION_COOKIE.timeout, int): timeout = timedelta(seconds=settings.SESSION_COOKIE.timeout) else: timeout = settings.SESSION_COOKIE.timeout SessionCookie.default_expiry_time = timeout
def get_sequence(key, default=1, step=1, retry_times=None, retry_waittime=None): from uliweb.orm import SaveError from uliweb import settings assert step > 0 and default > 0 Sequence = functions.get_model('sequence') i = 0 waittime = retry_waittime or settings.get_var('SEQUENCE/retry_waittime', 0.05) retry_times = retry_times or settings.get_var('SEQUENCE/retry_times', 3) while 1: try: row = Sequence.get(Sequence.c.key==key) if row: row.value = row.value + step row.save(version=True) else: row = Sequence(key=key, value=(default+step-1)) row.save() break except SaveError: i += 1 if i == retry_times: raise else: sleep(waittime) return row.value
def list(self): from uliweb import request from uliweb.utils.generic import ListView from uliweb.utils.common import get_choice import math pageno = int(request.values.get('page', 1)) - 1 rows_per_page=int(request.values.get('rows', settings.get_var('MESSAGES/PAGE_NUMS', 10))) read_flag = request.GET.get('read', '') type_flag = request.GET.get('type', '') condition = None condition = (self.model.c.user == request.user.id) & condition condition = (self.model.c.send_flag == 'r') & condition if read_flag: condition = (self.model.c.read_flag == bool(read_flag=='1')) & condition if type_flag: condition = (self.model.c.type == type_flag) & condition def create_date(value, obj): from uliweb.utils.timesince import timesince return timesince(value) def user_image(value, obj): return functions.get_user_image(obj.sender, size=20) def message(value, obj): return value fields_convert_map = {'create_date':create_date, 'user_image':user_image, 'message':message} view = ListView(self.model, condition=condition, order_by=[self.model.c.create_date.desc()], rows_per_page=rows_per_page, pageno=pageno, fields_convert_map=fields_convert_map) view.query() result = {} result['read_flag'] = read_flag result['type_flag'] = type_flag result['message_type_name'] = get_choice(settings.get_var('MESSAGES/MESSAGE_TYPE'), type_flag, '全部类型') pages = int(math.ceil(1.0*view.total/rows_per_page)) # result['page'] = pageno+1 # result['total'] = view.total # result['pages'] = pages result['pagination'] = functions.create_pagination(functions.request_url(), view.total, pageno+1, rows_per_page) result['objects'] = list(view.objects()) ids = [] for row in result['objects']: ids.append(row._obj_.id) self.model.filter(self.model.c.id.in_(ids)).update(read_flag=True) _del_key(request.user.id) return result
def get_cache(**kwargs): from uliweb import settings from weto.cache import Cache from uliweb.utils.common import import_attr, application_path serial_cls_path = settings.get_var('CACHE/serial_cls') if serial_cls_path: serial_cls = import_attr(serial_cls_path) if settings.GLOBAL.PICKLE_PROTOCAL_LEVEL is not None: serial_cls.protocal_level = settings.GLOBAL.PICKLE_PROTOCAL_LEVEL else: serial_cls = None options = dict(settings.get_var('CACHE_STORAGE', {})) options['data_dir'] = application_path(options['data_dir']) args = { 'storage_type': settings.get_var('CACHE/type'), 'options': options, 'expiry_time': settings.get_var('CACHE/expiretime'), 'serial_cls': serial_cls } args.update(kwargs) cache = Cache(**args) return cache
def get_session(key=None): options = dict(settings.get('SESSION_STORAGE', {})) options['data_dir'] = application_path(options['data_dir']) if 'url' not in options: _url = (settings.get_var('ORM/CONNECTION', '') or settings.get_var( 'ORM/CONNECTIONS', {}).get('default', {}).get('CONNECTION', '')) if _url: options['url'] = _url #process Session options session_storage_type = settings.SESSION.type Session.force = settings.SESSION.force serial_cls_path = settings.SESSION.serial_cls if serial_cls_path: serial_cls = import_attr(serial_cls_path) else: serial_cls = None session = Session(key, storage_type=session_storage_type, options=options, expiry_time=settings.SESSION.timeout, serial_cls=serial_cls) return session
def process_exception(self, request, e): from uliweb import settings import traceback from uliweb.mail import Mail from uliweb.utils.common import Serial from uliweb.core.SimpleFrame import HTTPError, HTTPException, NotFound if isinstance(e, (HTTPError, NotFound, HTTPException)): return type, value, tb = sys.exc_info() txt = "".join(traceback.format_exception(type, value, tb)) if settings.GLOBAL.EXCEPTION_PROCESS_TYPE == "mail": Mail().send_mail( settings.get_var("PARA/EMAIL_SENDER"), settings.get_var("PARA/DEV_TEAM"), u"程序运行出错:" + request.path, txt ) elif settings.GLOBAL.EXCEPTION_PROCESS_TYPE == "print": print txt elif settings.GLOBAL.EXCEPTION_PROCESS_TYPE == "redis": mail = { "from": settings.get_var("PARA/EMAIL_SENDER"), "to": settings.get_var("PARA/DEV_TEAM"), "title": u"程序运行出错" + request.url, "message": txt, } _t = Serial.dump(mail) self.redis.lpush("send_mails", _t)
def post_save(obj, data): from uliweb import functions from uliweb.utils.common import Serial from uliweb.mail import Mail Topic.filter(Topic.c.id==int(topic_id)).update(num_replies=Topic.c.num_replies+1, last_post_user=request.user.id, last_reply_on=date.now()) Forum.filter(Forum.c.id==int(forum_id)).update(num_posts=Forum.c.num_posts+1, last_post_user=request.user.id, last_reply_on=date.now()) self._clear_files(obj.slug, data['content']) #増加发送邮件的处理 emails = [] for u_id in Post.filter((Post.c.topic==int(topic_id)) & (Post.c.reply_email==True) & (Post.c.floor<obj.floor)).values(Post.c.posted_by): user = User.get(u_id[0]) if user and user.email and (user.email not in emails) and (user.email!=request.user.email): emails.append(user.email) if not emails: return _type = settings.get_var('PARA/FORUM_REPLY_PROCESS', 'print') url = '%s/forum/%s/%s' % (settings.get_var('PARA/DOMAIN'), forum_id, topic_id) d = {'url':str(url)} mail = {'from_':settings.get_var('PARA/EMAIL_SENDER'), 'to_':emails, 'subject':settings.get_var('FORUM_EMAIL/FORUM_EMAIL_TITLE'), 'message':settings.get_var('FORUM_EMAIL/FORUM_EMAIL_TEXT') % d, 'html':True} if _type == 'mail': Mail().send_mail(**mail) elif _type == 'print': print mail elif _type == 'redis': redis = functions.get_redis() _t = Serial.dump(mail) redis.lpush('send_mails', _t)
def _find_option(self, global_options, option): from uliweb import settings from uliweb.core.SimpleFrame import collect_settings from uliweb.utils.pyini import Ini print "------ Combined value of [%s] ------" % option print settings.get_var(option) print "------ Detail value of [%s] ------" % option sec_flag = "/" not in option if not sec_flag: section, key = option.split("/") for f in collect_settings( global_options.project, settings_file=global_options.settings, local_settings_file=global_options.local_settings, ): x = Ini(f, raw=True) if sec_flag: if option in x: print x[option] else: if section in x: if key in x[section]: v = x[section][key] print "%s %s%s" % (str(v), key, v.value())
def process_response(self, request, response): from uliweb import settings, functions, json_dumps import base64 #if not debug status it'll quit if not settings.get_var('GLOBAL/DEBUG'): return response S = functions.get_model('uliwebrecorderstatus') s = S.all().one() if not s or s.status == 'E': return response if settings.get_var('ULIWEBRECORDER/response_text'): try: text = response.data except Exception as e: text = str(e) else: text = '' #test if post_data need to convert base64 if not request.content_type: post_data_is_text = True else: post_data_is_text = self.test_text(request.content_type) if not post_data_is_text: post_data = base64.encodestring(request.data) else: post_data = json_dumps(request.POST.to_dict()) #test if response.data need to convert base64 response_data_is_text = self.test_text(response.content_type) if not response_data_is_text: response_data = base64.encodestring(text) else: response_data = text R = functions.get_model('uliwebrecorder') if request.user: user_id = request.user.id else: user_id = None max_content_length = settings.get_var('ULIWEBRECORDER/max_content_length') if len(response_data) > max_content_length: msg = "Content length is great than %d so it will be omitted." % max_content_length log.info(msg) response_data = msg response_data_is_text = True recorder = R(method=request.method, url=request_url(request), post_data_is_text=post_data_is_text, post_data=post_data, user=user_id, response_data=response_data, response_data_is_text=response_data_is_text, status_code=response.status_code, ) recorder.save() return response
def sended_list(self): from uliweb import request from uliweb.utils.generic import ListView from uliweb.utils.common import get_choice import math pageno = int(request.values.get("page", 1)) - 1 rows_per_page = int(request.values.get("rows", settings.get_var("MESSAGES/PAGE_NUMS", 10))) read_flag = request.GET.get("read", "") type_flag = request.GET.get("type", "") condition = None condition = (self.model.c.sender == request.user.id) & condition condition = (self.model.c.send_flag == "s") & condition if read_flag: condition = (self.model.c.read_flag == bool(read_flag == "1")) & condition if type_flag: condition = (self.model.c.type == type_flag) & condition def create_date(value, obj): from uliweb.utils.timesince import timesince return timesince(value) def user_image(value, obj): return functions.get_user_image(obj.user, size=20) def message(value, obj): return value fields_convert_map = {"create_date": create_date, "user_image": user_image, "message": message} view = ListView( self.model, condition=condition, order_by=[self.model.c.create_date.desc()], rows_per_page=rows_per_page, pageno=pageno, fields_convert_map=fields_convert_map, ) view.query() result = {} result["read_flag"] = read_flag result["type_flag"] = type_flag result["message_type_name"] = get_choice(settings.get_var("MESSAGES/MESSAGE_TYPE"), type_flag, "全部类型") pages = int(math.ceil(1.0 * view.total / rows_per_page)) # result['page'] = pageno+1 # result['total'] = view.total # result['pages'] = pages result["pagination"] = functions.create_pagination(request.url, view.total, pageno + 1, rows_per_page) result["objects"] = view.objects() return result
def use(vars, env, plugin, *args, **kwargs): from uliweb.core.SimpleFrame import get_app_dir from uliweb import application as app, settings from uliweb.utils.common import is_pyfile_exist if plugin in UseNode.__saved_template_plugins_modules__: mod = UseNode.__saved_template_plugins_modules__[plugin] else: #add settings support, only support simple situation #so for complex cases you should still write module #format just like: # #[TEMPLATE_USE] #name = { # 'toplinks':[ # 'myapp/jquery.myapp.{version}.min.js', # ], # 'depends':[xxxx], # 'config':{'version':'UI_CONFIG/test'}, # 'default':{'version':'1.2.0'}, #} # mod = None c = settings.get_var('TEMPLATE_USE/' + plugin) if c: config = c.pop('config', {}) default = c.pop('default', {}) #evaluate config value config = dict([(k, settings.get_var(v, default.get(k, ''))) for k, v in config.items()]) #merge passed arguments config.update(kwargs) for t in ['toplinks', 'bottomlinks']: if t in c: c[t] = [x.format(**config) for x in c[t]] mod = c else: for p in app.apps: if not is_pyfile_exist( os.path.join(get_app_dir(p), 'template_plugins'), plugin): continue module = '.'.join([p, 'template_plugins', plugin]) try: mod = __import__(module, {}, {}, ['']) except ImportError, e: log.exception(e) mod = None if mod: UseNode.__saved_template_plugins_modules__[plugin] = mod else: log.error( "Can't find the [%s] template plugin, please check if you've installed special app already" % plugin) if settings.get_var('TEMPLATE/RAISE_USE_EXCEPTION'): raise UseModuleNotFound( "Can't find the %s template plugin, check if you've installed special app already" % plugin)
def install_template_loader(self, dirs): Loader = import_attr(settings.get_var('TEMPLATE_PROCESSOR/loader')) args = settings.get_var('TEMPLATE') if self.debug: args['check_modified_time'] = True args['log'] = log args['debug'] = settings.get_var('GLOBAL/DEBUG_TEMPLATE', False) return Loader(dirs, **args)
def post_save(obj, data): from uliweb import functions from uliweb.utils.common import Serial from uliweb.mail import Mail Post.filter(Post.c.id == int(parent_id)).update( num_replies=Post.c.num_replies + 1, last_post_user=request.user.id, last_reply_on=date.now()) self._clear_files(obj.slug, data['content']) Topic.filter(Topic.c.id == int(topic_id)).update( num_replies=Topic.c.num_replies + 1, last_post_user=request.user.id, last_reply_on=date.now(), last_post=obj.id) Forum.filter(Forum.c.id == int(forum_id)).update( num_posts=Forum.c.num_posts + 1, last_post_user=request.user.id, last_reply_on=date.now(), last_post=obj.id) #増加发送邮件的处理 emails = [] for u_id in Post.filter((Post.c.topic == int(topic_id)) & (Post.c.reply_email == True) & (Post.c.id == parent_id)).values( Post.c.posted_by): user = User.get(u_id[0]) if user and user.email and (user.email not in emails) and ( user.email != request.user.email): emails.append(user.email) if not emails: return _type = settings.get_var('PARA/FORUM_REPLY_PROCESS', 'print') url = '%s/forum/%s/%s' % (settings.get_var('PARA/DOMAIN'), forum_id, topic_id) d = {'url': str(url)} mail = { 'from_': settings.get_var('PARA/EMAIL_SENDER'), 'to_': emails, 'subject': settings.get_var('FORUM_EMAIL/FORUM_EMAIL_TITLE'), 'message': settings.get_var('FORUM_EMAIL/FORUM_EMAIL_TEXT') % d, 'html': True } if _type == 'mail': Mail().send_mail(**mail) elif _type == 'print': print mail elif _type == 'redis': redis = functions.get_redis() _t = Serial.dump(mail) redis.lpush('send_mails', _t)
def get_id(engine, tablename, id=0, table_prefix=False): from uliweb import settings table = functions.get_table(tablename) d = {'engine':engine, 'tableid':table.id, 'id':str(id), 'tablename':tablename} if table_prefix: format = settings.get_var('OBJCACHE/table_format', 'OC:%(engine)s:%(tableid)d:') else: format = settings.get_var('OBJCACHE/key_format', 'OC:%(engine)s:%(tableid)d:%(id)s') return format % d
def post_save(obj, data): from uliweb import functions from uliweb.utils.common import Serial from uliweb.mail import Mail Post.filter(Post.c.id == int(parent_id)).update( num_replies=Post.c.num_replies + 1, last_post_user=request.user.id, last_reply_on=date.now() ) self._clear_files(obj.slug, data["content"]) Topic.filter(Topic.c.id == int(topic_id)).update( num_replies=Topic.c.num_replies + 1, last_post_user=request.user.id, last_reply_on=date.now(), last_post=obj.id, ) Forum.filter(Forum.c.id == int(forum_id)).update( num_posts=Forum.c.num_posts + 1, last_post_user=request.user.id, last_reply_on=date.now(), last_post=obj.id, ) # 増加发送邮件的处理 emails = [] for u_id in Post.filter( (Post.c.topic == int(topic_id)) & (Post.c.reply_email == True) & (Post.c.id == parent_id) ).values(Post.c.posted_by): user = User.get(u_id[0]) if user and user.email and (user.email not in emails) and (user.email != request.user.email): emails.append(user.email) if not emails: return _type = settings.get_var("PARA/FORUM_REPLY_PROCESS", "print") url = "%s/forum/%s/%s" % (settings.get_var("PARA/DOMAIN"), forum_id, topic_id) d = {"url": str(url)} mail = { "from_": settings.get_var("PARA/EMAIL_SENDER"), "to_": emails, "subject": settings.get_var("FORUM_EMAIL/FORUM_EMAIL_TITLE"), "message": settings.get_var("FORUM_EMAIL/FORUM_EMAIL_TEXT") % d, "html": True, } if _type == "mail": Mail().send_mail(**mail) elif _type == "print": print mail elif _type == "redis": redis = functions.get_redis() _t = Serial.dump(mail) redis.lpush("send_mails", _t)
def __init__(self, host=None, port=None, user=None, password=None, backend=None): from uliweb import settings from uliweb.utils.common import import_attr self.host = host or (settings and settings.get_var('MAIL/HOST')) self.port = port or (settings and settings.get_var('MAIL/PORT', 25)) self.user = user or (settings and settings.get_var('MAIL/USER')) self.password = password or (settings and settings.get_var('MAIL/PASSWORD')) self.backend = backend or (settings and settings.get_var('MAIL/BACKEND', 'uliweb.mail.backends.smtp')) cls = import_attr(self.backend + '.MailConnection') self.con = cls(self)
def get_staticize(backend=None, **kwargs): from uliweb import settings type = settings.get_var('STATICIZE/backend', 'file') _cls = _types.get(type) if not _cls: raise UliwebError("Can't found staticize type %s, please check the spell." % type) kw = settings.get_var('STATICIZE_BACKENDS/%s' % type, {}) kw.update(kwargs) handler = _cls(**kw) return handler
def use(vars, env, plugin, *args, **kwargs): from uliweb.core.SimpleFrame import get_app_dir from uliweb import application as app, settings from uliweb.utils.common import is_pyfile_exist if plugin in UseNode.__saved_template_plugins_modules__: mod = UseNode.__saved_template_plugins_modules__[plugin] else: #add settings support, only support simple situation #so for complex cases you should still write module #format just like: # #[TEMPLATE_USE] #name = { # 'toplinks':[ # 'myapp/jquery.myapp.{version}.min.js', # ], # 'depends':[xxxx], # 'config':{'version':'UI_CONFIG/test'}, # 'default':{'version':'1.2.0'}, #} # mod = None c = settings.get_var('TEMPLATE_USE/'+plugin) if c: config = c.pop('config', {}) default = c.pop('default', {}) #evaluate config value config = dict([(k, settings.get_var(v, default.get(k, ''))) for k, v in config.items()]) #merge passed arguments config.update(kwargs) for t in ['toplinks', 'bottomlinks']: if t in c: c[t] = [x.format(**config) for x in c[t]] mod = c else: for p in app.apps: if not is_pyfile_exist(os.path.join(get_app_dir(p), 'template_plugins'), plugin): continue module = '.'.join([p, 'template_plugins', plugin]) try: mod = __import__(module, {}, {}, ['']) except ImportError, e: log.exception(e) mod = None if mod: UseNode.__saved_template_plugins_modules__[plugin] = mod else: log.error("Can't find the [%s] template plugin, please check if you've installed special app already" % plugin) if settings.get_var('TEMPLATE/RAISE_USE_EXCEPTION'): raise UseModuleNotFound("Can't find the %s template plugin, check if you've installed special app already" % plugin)
def get_cache(**kwargs): from uliweb import settings from weto.cache import Cache from uliweb.utils.common import import_attr serial_cls_name = settings.get_var('CACHE/serial_cls', 'weto.cache.Serial') args = {'storage_type':settings.get_var('CACHE/type'), 'options':settings.get_var('CACHE_STORAGE'), 'expiry_time':settings.get_var('CACHE/expiretime'), 'serial_cls':import_attr(serial_cls_name)} args.update(kwargs) cache = Cache(**args) return cache
def run_command(row): import json from uliweb import settings keys_queue = settings.get_var('ASYNC_TASKS/tasks_keys_queue') kw = row.command_info.copy() kw['task_id'] = row.task_id queue = kw['queue'] try: redis = functions.get_redis() if not isinstance(queue, (tuple, list)): queue = [queue] msg = json.dumps(kw) #在添加消息到公共队列的同时,将任务ID添加到keys_queue中,用来记录当前将要,或正在执行的任务 for q in queue: pipe = redis.pipeline() name = settings.ASYNC_TASKS_QUEUES.get(q or 'default') or q pipe.lpush(name, msg).sadd(keys_queue, row.task_id) pipe.execute() log.info('Put task_id {} to queue=[{}] and keys_queue=[{}]'.format( row.task_id, name, keys_queue)) except Exception as e: _type, value, tb = sys.exc_info() txt = ''.join(traceback.format_exception(_type, value, tb)) handler = row.get_handler(log=log) handler.save(status=ERROR, execution_info=txt, finished_time=date.now()) log.error('Run async task {} failed when push with redis'.format( row.task_id)) log.exception(e)
def call(args, options, global_options): from uliweb import settings signal.signal(signal.SIGINT, handler) signal.signal(signal.SIGTERM, handler) check_point = settings.get_var('ASYNC_TASKS/check_point') log.info('=============================') log.info(' Async Task Daemon Starting') log.info('=============================') log.info('Daemon process %d' % os.getpid()) log.info('Check point %ds' % check_point) Task = functions.get_model('async_tasks') query = Task.filter(~Task.c.status.in_([SUCCESS, CANCEL, FAILED])).\ order_by(Task.c.created_time) redis = functions.get_redis() while not is_exit: Reset() for row in query: try: #log.debug('Ready to process async task {} - {}'.format(row.task_id, row.command_name)) process_task(row) except Exception as e: log.exception(e) beat = redis.brpop('async_tasks_beat', check_point) #to prevent database submit slow than redis time.sleep(0.5)
def delete_user_session(): from uliweb import settings, request user_fieldname = settings.get_var('AUTH/GET_AUTH_USER_FIELDNAME', 'id') share_session = settings.get_var('AUTH/AUTH_SHARE_USER_SESSION', False) user = request.user if user and share_session: if isinstance(user, dict): user_id = user[user_fieldname] else: user_id = getattr(user, user_fieldname) cache = functions.get_cache() key = get_user_session_key(user_id) log.debug('Auth: delete user session, userid={}, session_id={}'.format( key, request.session.key)) cache.delete(key)
def get_state_name(self): if self.obj: return self.obj.get_display_value('state') else: from uliweb import settings choices = settings.get_var('PARA/WF_STATUS') return choices[self.state]
def get_id(engine, tablename, id): from uliweb import settings d = {'engine': engine, 'tablename': str(tablename), 'id': str(id)} format = settings.get_var('OBJCACHE/key_format', 'OC:%(engine)s:%(tablename)s:%(id)s') return format % d
def post_delete(model, instance): from uliweb import response, settings if not check_enable(): return tablename = model.tablename if tablename not in settings.get_var('OBJCACHE_TABLES'): return def f(): redis = get_redis() if not redis: return info = settings.get_var('OBJCACHE_TABLES/%s' % tablename) key = 'id' if info and isinstance(info, dict): key = info.get('key', key) if '.' in key or key not in model.properties: _key = import_attr(key)(instance) else: _key = getattr(instance, key) _id = get_id(model.get_engine_name(), tablename, _key) try: redis.delete(_id) log.debug("objcache:post_delete:id=" + _id) except Exception, e: log.exception(e)
def post_save(model, instance, created, data, old_data): from uliweb import response, settings if not check_enable(): return tablename = model.tablename if tablename not in settings.get_var('OBJCACHE_TABLES'): return #if response is False, then it means you may in batch program #so it can't use post_commit machenism def f(): fields = get_fields(tablename) flag = created #if update then check if the record has changed if not flag: if not fields: flag = True else: fields = model.properties.keys() flag = bool(filter(lambda x: x in data, fields)) if flag: set_object(model, instance) log.debug("objcache:post_save:id=%d" % instance.id) if response: response.post_commit = f else: f()
def get_object(model, id, engine_name=None, connection=None): """ Get cached object from redis if id is None then return None: """ from uliweb import settings if not id: return if not check_enable(): return redis = get_redis() if not redis: return tablename = model.tablename info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {}) if info is None: return _id = get_id(engine_name or model.get_engine_name(), tablename, id) try: if redis.exists(_id): v = redis.hgetall(_id) o = model.load(v) log.debug("objcache:get:id="+_id) return o except Exception, e: log.exception(e)
def startup_installed(sender): from uliweb import settings import uliweb.form.uliform as form from uliweb.utils.common import import_attr for k, v in settings.get_var('FORM_FIELDS_MAP', {}).items(): form.fields_mapping[k] = import_attr(v)
def get_object(model, cid, engine_name=None, connection=None): """ Get cached object from redis if id is None then return None: """ from uliweb import settings if not id: return if not check_enable(): return redis = get_redis() if not redis: return tablename = model._alias or model.tablename info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {}) if info is None: return _id = get_id(engine_name or model.get_engine_name(), tablename, cid) try: log.debug("Try to find objcache:get:table=%s:id=[%s]" % (tablename, _id)) if redis.exists(_id): v = redis.hgetall(_id) o = model.load(v, from_dump=True) log.debug("Found!") return o else: log.debug("Not Found!") except Exception, e: log.exception(e)
def get_object(model, id, engine_name=None, connection=None): """ Get cached object from redis if id is None then return None: """ from uliweb import settings if not id: return if not check_enable(): return redis = get_redis() if not redis: return tablename = model.tablename info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {}) if info is None: return _id = get_id(engine_name or model.get_engine_name(), tablename, id) try: if redis.exists(_id): v = redis.hgetall(_id) o = model.load(v) log.debug("objcache:get:id=" + _id) return o except Exception, e: log.exception(e)
def get_object(model, cid, engine_name=None, connection=None): """ Get cached object from redis if id is None then return None: """ from uliweb import settings if not id: return if not check_enable(): return redis = get_redis() if not redis: return tablename = model._alias or model.tablename info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {}) if info is None: return _id = get_id(engine_name or model.get_engine_name(), tablename, cid) try: log.debug("Try to find objcache:get:table=%s:id=[%s]" % (tablename, _id)) if redis.exists(_id): v = redis.hgetall(_id) o = model.load(v, from_='dump') log.debug("Found!") return o else: log.debug("Not Found!") except Exception as e: log.exception(e)
def get_menu(cls, parent): from uliweb import functions, settings from uliweb.utils.common import Serial def _f(_p): menus = [] for row in cls.filter(cls.c.parent == _p, cls.c.type == 'M0000', cls.c.deploy_type == 'F', cls.c.status == '00001').order_by( cls.c.order): item = row.to_dict() item['link'] = row.value item['permissions'] = [x.name for x in row.permissions] menus.append(item) if row.has_children: item['subs'] = _f(item.id) else: item['subs'] = [] return menus menus = [] use_redis = settings.get_var('RESOURCE/USE_REDIS') key = 'MENU:{}'.format(parent) if use_redis: redis = functions.get_redis() v = redis.get(key) if v: menus = Serial.load(v) if not menus: p = cls.get(cls.c.name == parent) menus = _f(p.id) if menus and use_redis: redis.set(key, Serial.dump(menus)) return menus
def run_command(row): import json from uliweb import settings keys_queue = settings.get_var('ASYNC_TASKS/tasks_keys_queue') kw = row.command_info.copy() kw['task_id'] = row.task_id queue =kw['queue'] try: redis = functions.get_redis() if not isinstance(queue, (tuple, list)): queue = [queue] msg = json.dumps(kw) #在添加消息到公共队列的同时,将任务ID添加到keys_queue中,用来记录当前将要,或正在执行的任务 for q in queue: pipe = redis.pipeline() name = settings.ASYNC_TASKS_QUEUES.get(q or 'default') or q pipe.lpush(name, msg).sadd(keys_queue, row.task_id) pipe.execute() log.info('Put task_id {} to queue=[{}] and keys_queue=[{}]'.format(row.task_id, name, keys_queue)) except Exception as e: _type, value, tb = sys.exc_info() txt = ''.join(traceback.format_exception(_type, value, tb)) handler = row.get_handler(log=log) handler.save(status=ERROR, execution_info=txt, finished_time=date.now()) log.error('Run async task {} failed when push with redis'.format(row.task_id)) log.exception(e)
def test_text(self, content_type): from uliweb.utils.common import match from uliweb import settings m = content_type.split(';', 1)[0] r = match(m, settings.get_var('ULIWEBRECORDER/text_content_types')) return r
def get_var(key, default=None): """ Default get_var function """ from uliweb import settings return settings.get_var(key, default)
def _wiki_view(self, pagename): from uliweb import request, response response.template = 'WikiView/wiki_view.html' if not pagename: if 'pages' in request.GET: response.template = 'WikiView/wiki_pages.html' objects = self.model.filter(self.model.c.enabled==True).filter(self.model.c.deleted==False).fields('name').order_by(self.model.c.name) return {'objects':objects} return redirect(url_for(self.__class__.wiki, pagename='Index')) wiki = self.model.get((self.model.c.name == pagename) & (self.model.c.enabled==True)) if not wiki: #check read permission self._check_permission('read') if pagename == 'Index': content = settings.WIKI.IndexContent wiki = self.model(name='Index', content=content, enabled=True, creator=request.user, modified_user=request.user) wiki.save() wiki.new_revision() if wiki: #check read permission self._check_permission('read', wiki, page=wiki) #处理阅读次数 key = '__wikivisited__:%s:%d' % (request.remote_addr, wiki.id) cache = functions.get_cache() v = cache.get(key, None) if not v: self.model.filter(self.model.c.id==int(wiki.id)).update(hits=self.model.c.hits+1) cache.set(key, 1, settings.get_var('WIKI/WIKI_VISITED_TIMEOUT')) rev = int(request.GET.get('rev', 0)) rev_time = None if rev: cached_pagename = wiki.name + '&ver=%d' % rev r = self.changeset.get((self.changeset.c.wiki==wiki.id) & (self.changeset.c.revision==rev)) rev_time = r.modified_time and r.modified_time.strftime('%Y-%m-%d %H:%M:%S') content, kwargs = self._get_cached_page_html(cached_pagename, r.old_content) else: cached_pagename = wiki.name content, kwargs = self._get_cached_page_html(cached_pagename, wiki.content) last_rev = wiki.changeset.order_by(self.changeset.c.id.desc()).one().revision return {'wiki':wiki, 'content':content, 'revision':rev, 'rev_time':rev_time, 'last_rev':last_rev, 'page_args':kwargs, 'create_new':self._check_permission('write', raise_exception=False), 'permissions':self._find_tree_permissions(['read', 'write', 'delete'], wiki, request.user) } else: #check read permission self._check_permission('write') return redirect(url_for(self.__class__.wiki, pagename=pagename, action='edit'))
def post_delete(model, instance): from uliweb import response, settings if not check_enable(): return tablename = model.tablename if tablename not in settings.get_var('OBJCACHE_TABLES'): return def f(): redis = get_redis() if not redis: return info = settings.get_var('OBJCACHE_TABLES/%s' % tablename) key = 'id' if info and isinstance(info, dict): key = info.get('key', key) if '.' in key or key not in model.properties: _key = import_attr(key)(instance) else: _key = getattr(instance, key) _id = get_id(model.get_engine_name(), tablename, _key) try: redis.delete(_id) log.debug("objcache:post_delete:id="+_id) except Exception, e: log.exception(e)
def post_save(model, instance, created, data, old_data): from uliweb import response, settings if not check_enable(): return tablename = model.tablename if tablename not in settings.get_var('OBJCACHE_TABLES'): return #if response is False, then it means you may in batch program #so it can't use post_commit machenism def f(): fields = get_fields(tablename) flag = created #if update then check if the record has changed if not flag: if not fields: flag = True else: fields = model.properties.keys() flag = bool(filter(lambda x:x in data, fields)) if flag: set_object(model, instance) log.debug("objcache:post_save:id=%d" % instance.id) if response: response.post_commit = f else: f()
def __init__(self, default_filename_converter_cls=UUIDFilenameConverter, config=None): self.config = config or self.default_config for k, v in self.options.items(): item, default = v #if there is no '/' in option, then combine config with option #else assume the option is just like 'SECTION/OPTION', then skip it if '/' not in item: item = self.config + '/' + item value = settings.get_var(item, default) setattr(self, k, value) if self.x_sendfile and not self.x_header_name: if self.x_sendfile == 'nginx': self.x_header_name = 'X-Accel-Redirect' elif self.x_sendfile == 'apache': self.x_header_name = 'X-Sendfile' else: raise Exception( "X_HEADER can't be None, or X_SENDFILE is not supprted") if isinstance(self._filename_converter, (str, unicode)): self._filename_converter_cls = import_attr( self._filename_converter) else: self._filename_converter_cls = self._filename_converter or default_filename_converter_cls
def set_object(model, instance, fields=None, engine_name=None): """ Only support simple condition, for example: Model.c.id == n if not id provided, then use instance.id """ from uliweb import settings if not check_enable(): return redis = get_redis() if not redis: return tablename = model._alias or model.tablename exclude = [] if not fields: fields, exclude = get_fields(tablename) v = instance.dump(fields, exclude=exclude) info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {}) if info is None: return expire = settings.get_var('OBJCACHE/timeout', 0) key = 'id' if info and isinstance(info, dict): expire = info.get('expire', expire) key = info.get('key', key) if '.' in key or key not in model.properties: _key = import_attr(key)(instance) else: _key = getattr(instance, key) _id = get_id(engine_name or model.get_engine_name(), tablename, _key) try: pipe = redis.pipeline() p = pipe.delete(_id).hmset(_id, v) expire_msg = '' if expire: p = p.expire(_id, expire) expire_msg = ':expire=%d' % expire r = p.execute() log.debug("Saving to cache objcache:set:table=%s:id=[%s]:%s" % (tablename, _id, expire_msg)) except Exception, e: log.exception(e)
def get_key(tablename): from uliweb import settings tables = settings.get_var('OBJCACHE/config') if not tablename in tables: return return tables[tablename]
def get_fields(tablename): from uliweb import settings tables = settings.get_var('OBJCACHE_TABLES') if not tablename in tables: return return tables[tablename]
def init(self): from uliweb import settings self.redis = functions.get_redis() self.queue = self.kwargs['queue'] self.work_queue = self.queue + ':' + self.name self.keys_queue = settings.get_var('ASYNC_TASKS/tasks_keys_queue') #todo log self.log.info( '{} {} created with queue={}, work queue={} timeout={} ' 'max_requests={} soft_memory_limit={} hard_memory_limit={}'.format( self.name, self.pid, self.queue, self.work_queue, self.timeout, self.max_requests, self.soft_memory_limit, self.hard_memory_limit)) self.check_point = settings.get_var('ASYNC_TASKS_WORKERS/check_point')
def _upload_file(self, image=False, show_filename=True): import os import Image if image: from forms import ImageUploadForm as Form else: from forms import FileUploadForm as Form from uliweb.utils.image import thumbnail_image, fix_filename from uliweb import json_dumps File = get_model('forumattachment') forum_id = request.GET.get('forum_id') slug = request.GET.get('slug') form = Form() suffix = date.now().strftime('_%Y_%m_%d') if request.method == 'GET': form.bind({'is_thumbnail': True}) return {'form': form} else: flag = form.validate(request.values, request.files) if flag: f = form.data['filedata'] _f = os.path.basename(f['filename']) #文件格式为forum/<forum_id>/<filename_yyyy_mm_dd> filename = fix_filename('forum/%s/%s' % (forum_id, _f), suffix) if image: filename = functions.save_file(filename, f['file']) if form.data['is_thumbnail']: #process thumbnail rfilename, thumbnail = thumbnail_image( functions.get_filename(filename, filesystem=True), filename, settings.get_var('PARA/FORUM_THUMBNAIL_SIZE')) _file = functions.get_href(thumbnail) else: _file = functions.get_href(filename) name = functions.get_href(filename) else: filename = functions.save_file(filename, f['file']) name = form.data['title'] if not name: name = _f _file = functions.get_href(filename, alt=name) ff = File(slug=slug, file_name=filename, name=name) ff.save() name = json_dumps(name, unicode=True) if show_filename: fargs = '||%s' % name[1:-1] else: fargs = '' return '''<script type="text/javascript"> var url='%s%s'; setTimeout(function(){callback(url);},100); </script> ''' % (_file, fargs) else: return {'form': form}
def success_data(obj, data): import math return { 'page': int( math.ceil(1.0 * obj.floor / settings.get_var('PARA/FORUM_PAGE_NUMS'))) }
def __init__(self, host=None, port=None, user=None, password=None, backend=None): from uliweb import settings from uliweb.utils.common import import_attr self.host = host or (settings and settings.get_var('MAIL/HOST')) self.port = port or (settings and settings.get_var('MAIL/PORT', 25)) self.user = user or (settings and settings.get_var('MAIL/USER')) self.password = password or (settings and settings.get_var('MAIL/PASSWORD')) self.backend = backend or (settings and settings.get_var( 'MAIL/BACKEND', 'uliweb.mail.backends.smtp')) cls = import_attr(self.backend + '.MailConnection') self.con = cls(self)