def init_static_combine():
from uliweb import settings
from hashlib import md5
import os
PLUGINS = settings.get_var("TEMPLATE_GULP")
d = {}
if settings.get_var('STATIC_COMBINE_CONFIG/enable', False):
# for k, v in settings.get('STATIC_COMBINE', {}).items():
# key = '_cmb_' + md5(''.join(v)).hexdigest() + os.path.splitext(v[0])[1]
# d[key] = v
from uliweb.contrib.template.tags import find
for k, v in PLUGINS.items():
js_list = []
css_list = []
for x in v:
s = find(x)
m = s[0] + s[1]
for i in m:
if not i.startswith('<!--'):
e = os.path.splitext(i)[1]
if e == ".css":
css_list.append(i)
elif e == ".js":
js_list.append(i)
if js_list:
d[k+".js"] = js_list
if css_list:
d[k + ".css"] = css_list
return d
def get_sequence(key,
default=1,
step=1,
retry_times=None,
retry_waittime=None):
from uliweb.orm import SaveError
from uliweb import settings
assert step > 0 and default > 0
Sequence = functions.get_model('sequence')
i = 0
waittime = retry_waittime or settings.get_var('SEQUENCE/retry_waittime',
0.05)
retry_times = retry_times or settings.get_var('SEQUENCE/retry_times', 3)
while 1:
try:
row = Sequence.get(Sequence.c.key == key)
if row:
row.value = row.value + step
row.save(version=True)
else:
row = Sequence(key=key, value=(default + step - 1))
row.save()
break
except SaveError:
i += 1
if i == retry_times:
raise
else:
sleep(waittime)
return row.value
def __init__(self, application, settings):
from datetime import timedelta
self.options = dict(settings.get('SESSION_STORAGE', {}))
self.options['data_dir'] = application_path(self.options['data_dir'])
if 'url' not in self.options:
_url = (settings.get_var('ORM/CONNECTION', '')
or settings.get_var('ORM/CONNECTIONS', {}).get(
'default', {}).get('CONNECTION', ''))
if _url:
self.options['url'] = _url
#process Session options
self.remember_me_timeout = settings.SESSION.remember_me_timeout
self.session_storage_type = settings.SESSION.type
self.timeout = settings.SESSION.timeout
Session.force = settings.SESSION.force
#process Cookie options
SessionCookie.default_domain = settings.SESSION_COOKIE.domain
SessionCookie.default_path = settings.SESSION_COOKIE.path
SessionCookie.default_secure = settings.SESSION_COOKIE.secure
SessionCookie.default_cookie_id = settings.SESSION_COOKIE.cookie_id
if isinstance(settings.SESSION_COOKIE.timeout, int):
timeout = timedelta(seconds=settings.SESSION_COOKIE.timeout)
else:
timeout = settings.SESSION_COOKIE.timeout
SessionCookie.default_expiry_time = timeout
def __init__(self, application, settings):
from datetime import timedelta
self.options = dict(settings.get('SESSION_STORAGE', {}))
self.options['data_dir'] = application_path(self.options['data_dir'])
if 'url' not in self.options:
_url = (settings.get_var('ORM/CONNECTION', '') or
settings.get_var('ORM/CONNECTIONS', {}).get('default', {}).get('CONNECTION', ''))
if _url:
self.options['url'] = _url
#process Session options
self.remember_me_timeout = settings.SESSION.remember_me_timeout
self.session_storage_type = settings.SESSION.type
self.timeout = settings.SESSION.timeout
Session.force = settings.SESSION.force
#process Cookie options
SessionCookie.default_domain = settings.SESSION_COOKIE.domain
SessionCookie.default_path = settings.SESSION_COOKIE.path
SessionCookie.default_secure = settings.SESSION_COOKIE.secure
SessionCookie.default_cookie_id = settings.SESSION_COOKIE.cookie_id
if isinstance(settings.SESSION_COOKIE.timeout, int):
timeout = timedelta(seconds=settings.SESSION_COOKIE.timeout)
else:
timeout = settings.SESSION_COOKIE.timeout
SessionCookie.default_expiry_time = timeout
def get_sequence(key, default=1, step=1, retry_times=None, retry_waittime=None):
from uliweb.orm import SaveError
from uliweb import settings
assert step > 0 and default > 0
Sequence = functions.get_model('sequence')
i = 0
waittime = retry_waittime or settings.get_var('SEQUENCE/retry_waittime', 0.05)
retry_times = retry_times or settings.get_var('SEQUENCE/retry_times', 3)
while 1:
try:
row = Sequence.get(Sequence.c.key==key)
if row:
row.value = row.value + step
row.save(version=True)
else:
row = Sequence(key=key, value=(default+step-1))
row.save()
break
except SaveError:
i += 1
if i == retry_times:
raise
else:
sleep(waittime)
return row.value
def list(self):
from uliweb import request
from uliweb.utils.generic import ListView
from uliweb.utils.common import get_choice
import math
pageno = int(request.values.get('page', 1)) - 1
rows_per_page=int(request.values.get('rows', settings.get_var('MESSAGES/PAGE_NUMS', 10)))
read_flag = request.GET.get('read', '')
type_flag = request.GET.get('type', '')
condition = None
condition = (self.model.c.user == request.user.id) & condition
condition = (self.model.c.send_flag == 'r') & condition
if read_flag:
condition = (self.model.c.read_flag == bool(read_flag=='1')) & condition
if type_flag:
condition = (self.model.c.type == type_flag) & condition
def create_date(value, obj):
from uliweb.utils.timesince import timesince
return timesince(value)
def user_image(value, obj):
return functions.get_user_image(obj.sender, size=20)
def message(value, obj):
return value
fields_convert_map = {'create_date':create_date,
'user_image':user_image,
'message':message}
view = ListView(self.model, condition=condition,
order_by=[self.model.c.create_date.desc()],
rows_per_page=rows_per_page, pageno=pageno,
fields_convert_map=fields_convert_map)
view.query()
result = {}
result['read_flag'] = read_flag
result['type_flag'] = type_flag
result['message_type_name'] = get_choice(settings.get_var('MESSAGES/MESSAGE_TYPE'), type_flag, '全部类型')
pages = int(math.ceil(1.0*view.total/rows_per_page))
# result['page'] = pageno+1
# result['total'] = view.total
# result['pages'] = pages
result['pagination'] = functions.create_pagination(functions.request_url(), view.total, pageno+1, rows_per_page)
result['objects'] = list(view.objects())
ids = []
for row in result['objects']:
ids.append(row._obj_.id)
self.model.filter(self.model.c.id.in_(ids)).update(read_flag=True)
_del_key(request.user.id)
return result
def get_cache(**kwargs):
from uliweb import settings
from weto.cache import Cache
from uliweb.utils.common import import_attr, application_path
serial_cls_path = settings.get_var('CACHE/serial_cls')
if serial_cls_path:
serial_cls = import_attr(serial_cls_path)
if settings.GLOBAL.PICKLE_PROTOCAL_LEVEL is not None:
serial_cls.protocal_level = settings.GLOBAL.PICKLE_PROTOCAL_LEVEL
else:
serial_cls = None
options = dict(settings.get_var('CACHE_STORAGE', {}))
options['data_dir'] = application_path(options['data_dir'])
args = {
'storage_type': settings.get_var('CACHE/type'),
'options': options,
'expiry_time': settings.get_var('CACHE/expiretime'),
'serial_cls': serial_cls
}
args.update(kwargs)
cache = Cache(**args)
return cache
def get_session(key=None):
options = dict(settings.get('SESSION_STORAGE', {}))
options['data_dir'] = application_path(options['data_dir'])
if 'url' not in options:
_url = (settings.get_var('ORM/CONNECTION', '') or settings.get_var(
'ORM/CONNECTIONS', {}).get('default', {}).get('CONNECTION', ''))
if _url:
options['url'] = _url
#process Session options
session_storage_type = settings.SESSION.type
Session.force = settings.SESSION.force
serial_cls_path = settings.SESSION.serial_cls
if serial_cls_path:
serial_cls = import_attr(serial_cls_path)
else:
serial_cls = None
session = Session(key,
storage_type=session_storage_type,
options=options,
expiry_time=settings.SESSION.timeout,
serial_cls=serial_cls)
return session
def process_exception(self, request, e):
from uliweb import settings
import traceback
from uliweb.mail import Mail
from uliweb.utils.common import Serial
from uliweb.core.SimpleFrame import HTTPError, HTTPException, NotFound
if isinstance(e, (HTTPError, NotFound, HTTPException)):
return
type, value, tb = sys.exc_info()
txt = "".join(traceback.format_exception(type, value, tb))
if settings.GLOBAL.EXCEPTION_PROCESS_TYPE == "mail":
Mail().send_mail(
settings.get_var("PARA/EMAIL_SENDER"), settings.get_var("PARA/DEV_TEAM"), u"程序运行出错:" + request.path, txt
)
elif settings.GLOBAL.EXCEPTION_PROCESS_TYPE == "print":
print txt
elif settings.GLOBAL.EXCEPTION_PROCESS_TYPE == "redis":
mail = {
"from": settings.get_var("PARA/EMAIL_SENDER"),
"to": settings.get_var("PARA/DEV_TEAM"),
"title": u"程序运行出错" + request.url,
"message": txt,
}
_t = Serial.dump(mail)
self.redis.lpush("send_mails", _t)
def post_save(obj, data):
from uliweb import functions
from uliweb.utils.common import Serial
from uliweb.mail import Mail
Topic.filter(Topic.c.id==int(topic_id)).update(num_replies=Topic.c.num_replies+1, last_post_user=request.user.id, last_reply_on=date.now())
Forum.filter(Forum.c.id==int(forum_id)).update(num_posts=Forum.c.num_posts+1, last_post_user=request.user.id, last_reply_on=date.now())
self._clear_files(obj.slug, data['content'])
#増加发送邮件的处理
emails = []
for u_id in Post.filter((Post.c.topic==int(topic_id)) & (Post.c.reply_email==True) & (Post.c.floor<obj.floor)).values(Post.c.posted_by):
user = User.get(u_id[0])
if user and user.email and (user.email not in emails) and (user.email!=request.user.email):
emails.append(user.email)
if not emails:
return
_type = settings.get_var('PARA/FORUM_REPLY_PROCESS', 'print')
url = '%s/forum/%s/%s' % (settings.get_var('PARA/DOMAIN'), forum_id, topic_id)
d = {'url':str(url)}
mail = {'from_':settings.get_var('PARA/EMAIL_SENDER'), 'to_':emails,
'subject':settings.get_var('FORUM_EMAIL/FORUM_EMAIL_TITLE'),
'message':settings.get_var('FORUM_EMAIL/FORUM_EMAIL_TEXT') % d,
'html':True}
if _type == 'mail':
Mail().send_mail(**mail)
elif _type == 'print':
print mail
elif _type == 'redis':
redis = functions.get_redis()
_t = Serial.dump(mail)
redis.lpush('send_mails', _t)
def _find_option(self, global_options, option):
from uliweb import settings
from uliweb.core.SimpleFrame import collect_settings
from uliweb.utils.pyini import Ini
print "------ Combined value of [%s] ------" % option
print settings.get_var(option)
print "------ Detail value of [%s] ------" % option
sec_flag = "/" not in option
if not sec_flag:
section, key = option.split("/")
for f in collect_settings(
global_options.project,
settings_file=global_options.settings,
local_settings_file=global_options.local_settings,
):
x = Ini(f, raw=True)
if sec_flag:
if option in x:
print x[option]
else:
if section in x:
if key in x[section]:
v = x[section][key]
print "%s %s%s" % (str(v), key, v.value())
def process_response(self, request, response):
from uliweb import settings, functions, json_dumps
import base64
#if not debug status it'll quit
if not settings.get_var('GLOBAL/DEBUG'):
return response
S = functions.get_model('uliwebrecorderstatus')
s = S.all().one()
if not s or s.status == 'E':
return response
if settings.get_var('ULIWEBRECORDER/response_text'):
try:
text = response.data
except Exception as e:
text = str(e)
else:
text = ''
#test if post_data need to convert base64
if not request.content_type:
post_data_is_text = True
else:
post_data_is_text = self.test_text(request.content_type)
if not post_data_is_text:
post_data = base64.encodestring(request.data)
else:
post_data = json_dumps(request.POST.to_dict())
#test if response.data need to convert base64
response_data_is_text = self.test_text(response.content_type)
if not response_data_is_text:
response_data = base64.encodestring(text)
else:
response_data = text
R = functions.get_model('uliwebrecorder')
if request.user:
user_id = request.user.id
else:
user_id = None
max_content_length = settings.get_var('ULIWEBRECORDER/max_content_length')
if len(response_data) > max_content_length:
msg = "Content length is great than %d so it will be omitted." % max_content_length
log.info(msg)
response_data = msg
response_data_is_text = True
recorder = R(method=request.method,
url=request_url(request),
post_data_is_text=post_data_is_text,
post_data=post_data, user=user_id,
response_data=response_data,
response_data_is_text=response_data_is_text,
status_code=response.status_code,
)
recorder.save()
return response
def sended_list(self):
from uliweb import request
from uliweb.utils.generic import ListView
from uliweb.utils.common import get_choice
import math
pageno = int(request.values.get("page", 1)) - 1
rows_per_page = int(request.values.get("rows", settings.get_var("MESSAGES/PAGE_NUMS", 10)))
read_flag = request.GET.get("read", "")
type_flag = request.GET.get("type", "")
condition = None
condition = (self.model.c.sender == request.user.id) & condition
condition = (self.model.c.send_flag == "s") & condition
if read_flag:
condition = (self.model.c.read_flag == bool(read_flag == "1")) & condition
if type_flag:
condition = (self.model.c.type == type_flag) & condition
def create_date(value, obj):
from uliweb.utils.timesince import timesince
return timesince(value)
def user_image(value, obj):
return functions.get_user_image(obj.user, size=20)
def message(value, obj):
return value
fields_convert_map = {"create_date": create_date, "user_image": user_image, "message": message}
view = ListView(
self.model,
condition=condition,
order_by=[self.model.c.create_date.desc()],
rows_per_page=rows_per_page,
pageno=pageno,
fields_convert_map=fields_convert_map,
)
view.query()
result = {}
result["read_flag"] = read_flag
result["type_flag"] = type_flag
result["message_type_name"] = get_choice(settings.get_var("MESSAGES/MESSAGE_TYPE"), type_flag, "全部类型")
pages = int(math.ceil(1.0 * view.total / rows_per_page))
# result['page'] = pageno+1
# result['total'] = view.total
# result['pages'] = pages
result["pagination"] = functions.create_pagination(request.url, view.total, pageno + 1, rows_per_page)
result["objects"] = view.objects()
return result
def use(vars, env, plugin, *args, **kwargs):
from uliweb.core.SimpleFrame import get_app_dir
from uliweb import application as app, settings
from uliweb.utils.common import is_pyfile_exist
if plugin in UseNode.__saved_template_plugins_modules__:
mod = UseNode.__saved_template_plugins_modules__[plugin]
else:
#add settings support, only support simple situation
#so for complex cases you should still write module
#format just like:
#
#[TEMPLATE_USE]
#name = {
# 'toplinks':[
# 'myapp/jquery.myapp.{version}.min.js',
# ],
# 'depends':[xxxx],
# 'config':{'version':'UI_CONFIG/test'},
# 'default':{'version':'1.2.0'},
#}
#
mod = None
c = settings.get_var('TEMPLATE_USE/' + plugin)
if c:
config = c.pop('config', {})
default = c.pop('default', {})
#evaluate config value
config = dict([(k, settings.get_var(v, default.get(k, '')))
for k, v in config.items()])
#merge passed arguments
config.update(kwargs)
for t in ['toplinks', 'bottomlinks']:
if t in c:
c[t] = [x.format(**config) for x in c[t]]
mod = c
else:
for p in app.apps:
if not is_pyfile_exist(
os.path.join(get_app_dir(p), 'template_plugins'),
plugin):
continue
module = '.'.join([p, 'template_plugins', plugin])
try:
mod = __import__(module, {}, {}, [''])
except ImportError, e:
log.exception(e)
mod = None
if mod:
UseNode.__saved_template_plugins_modules__[plugin] = mod
else:
log.error(
"Can't find the [%s] template plugin, please check if you've installed special app already"
% plugin)
if settings.get_var('TEMPLATE/RAISE_USE_EXCEPTION'):
raise UseModuleNotFound(
"Can't find the %s template plugin, check if you've installed special app already"
% plugin)
def install_template_loader(self, dirs):
Loader = import_attr(settings.get_var('TEMPLATE_PROCESSOR/loader'))
args = settings.get_var('TEMPLATE')
if self.debug:
args['check_modified_time'] = True
args['log'] = log
args['debug'] = settings.get_var('GLOBAL/DEBUG_TEMPLATE', False)
return Loader(dirs, **args)
def post_save(obj, data):
from uliweb import functions
from uliweb.utils.common import Serial
from uliweb.mail import Mail
Post.filter(Post.c.id == int(parent_id)).update(
num_replies=Post.c.num_replies + 1,
last_post_user=request.user.id,
last_reply_on=date.now())
self._clear_files(obj.slug, data['content'])
Topic.filter(Topic.c.id == int(topic_id)).update(
num_replies=Topic.c.num_replies + 1,
last_post_user=request.user.id,
last_reply_on=date.now(),
last_post=obj.id)
Forum.filter(Forum.c.id == int(forum_id)).update(
num_posts=Forum.c.num_posts + 1,
last_post_user=request.user.id,
last_reply_on=date.now(),
last_post=obj.id)
#増加发送邮件的处理
emails = []
for u_id in Post.filter((Post.c.topic == int(topic_id))
& (Post.c.reply_email == True)
& (Post.c.id == parent_id)).values(
Post.c.posted_by):
user = User.get(u_id[0])
if user and user.email and (user.email not in emails) and (
user.email != request.user.email):
emails.append(user.email)
if not emails:
return
_type = settings.get_var('PARA/FORUM_REPLY_PROCESS', 'print')
url = '%s/forum/%s/%s' % (settings.get_var('PARA/DOMAIN'),
forum_id, topic_id)
d = {'url': str(url)}
mail = {
'from_': settings.get_var('PARA/EMAIL_SENDER'),
'to_': emails,
'subject': settings.get_var('FORUM_EMAIL/FORUM_EMAIL_TITLE'),
'message':
settings.get_var('FORUM_EMAIL/FORUM_EMAIL_TEXT') % d,
'html': True
}
if _type == 'mail':
Mail().send_mail(**mail)
elif _type == 'print':
print mail
elif _type == 'redis':
redis = functions.get_redis()
_t = Serial.dump(mail)
redis.lpush('send_mails', _t)
def install_template_loader(self, dirs):
Loader = import_attr(settings.get_var('TEMPLATE_PROCESSOR/loader'))
args = settings.get_var('TEMPLATE')
if self.debug:
args['check_modified_time'] = True
args['log'] = log
args['debug'] = settings.get_var('GLOBAL/DEBUG_TEMPLATE', False)
return Loader(dirs, **args)
def get_id(engine, tablename, id=0, table_prefix=False):
from uliweb import settings
table = functions.get_table(tablename)
d = {'engine':engine, 'tableid':table.id, 'id':str(id), 'tablename':tablename}
if table_prefix:
format = settings.get_var('OBJCACHE/table_format', 'OC:%(engine)s:%(tableid)d:')
else:
format = settings.get_var('OBJCACHE/key_format', 'OC:%(engine)s:%(tableid)d:%(id)s')
return format % d
def get_id(engine, tablename, id=0, table_prefix=False):
from uliweb import settings
table = functions.get_table(tablename)
d = {'engine':engine, 'tableid':table.id, 'id':str(id), 'tablename':tablename}
if table_prefix:
format = settings.get_var('OBJCACHE/table_format', 'OC:%(engine)s:%(tableid)d:')
else:
format = settings.get_var('OBJCACHE/key_format', 'OC:%(engine)s:%(tableid)d:%(id)s')
return format % d
def post_save(obj, data):
from uliweb import functions
from uliweb.utils.common import Serial
from uliweb.mail import Mail
Post.filter(Post.c.id == int(parent_id)).update(
num_replies=Post.c.num_replies + 1, last_post_user=request.user.id, last_reply_on=date.now()
)
self._clear_files(obj.slug, data["content"])
Topic.filter(Topic.c.id == int(topic_id)).update(
num_replies=Topic.c.num_replies + 1,
last_post_user=request.user.id,
last_reply_on=date.now(),
last_post=obj.id,
)
Forum.filter(Forum.c.id == int(forum_id)).update(
num_posts=Forum.c.num_posts + 1,
last_post_user=request.user.id,
last_reply_on=date.now(),
last_post=obj.id,
)
# 増加发送邮件的处理
emails = []
for u_id in Post.filter(
(Post.c.topic == int(topic_id)) & (Post.c.reply_email == True) & (Post.c.id == parent_id)
).values(Post.c.posted_by):
user = User.get(u_id[0])
if user and user.email and (user.email not in emails) and (user.email != request.user.email):
emails.append(user.email)
if not emails:
return
_type = settings.get_var("PARA/FORUM_REPLY_PROCESS", "print")
url = "%s/forum/%s/%s" % (settings.get_var("PARA/DOMAIN"), forum_id, topic_id)
d = {"url": str(url)}
mail = {
"from_": settings.get_var("PARA/EMAIL_SENDER"),
"to_": emails,
"subject": settings.get_var("FORUM_EMAIL/FORUM_EMAIL_TITLE"),
"message": settings.get_var("FORUM_EMAIL/FORUM_EMAIL_TEXT") % d,
"html": True,
}
if _type == "mail":
Mail().send_mail(**mail)
elif _type == "print":
print mail
elif _type == "redis":
redis = functions.get_redis()
_t = Serial.dump(mail)
redis.lpush("send_mails", _t)
def __init__(self, host=None, port=None, user=None, password=None, backend=None):
from uliweb import settings
from uliweb.utils.common import import_attr
self.host = host or (settings and settings.get_var('MAIL/HOST'))
self.port = port or (settings and settings.get_var('MAIL/PORT', 25))
self.user = user or (settings and settings.get_var('MAIL/USER'))
self.password = password or (settings and settings.get_var('MAIL/PASSWORD'))
self.backend = backend or (settings and settings.get_var('MAIL/BACKEND', 'uliweb.mail.backends.smtp'))
cls = import_attr(self.backend + '.MailConnection')
self.con = cls(self)
def get_staticize(backend=None, **kwargs):
from uliweb import settings
type = settings.get_var('STATICIZE/backend', 'file')
_cls = _types.get(type)
if not _cls:
raise UliwebError("Can't found staticize type %s, please check the spell." % type)
kw = settings.get_var('STATICIZE_BACKENDS/%s' % type, {})
kw.update(kwargs)
handler = _cls(**kw)
return handler
def use(vars, env, plugin, *args, **kwargs):
from uliweb.core.SimpleFrame import get_app_dir
from uliweb import application as app, settings
from uliweb.utils.common import is_pyfile_exist
if plugin in UseNode.__saved_template_plugins_modules__:
mod = UseNode.__saved_template_plugins_modules__[plugin]
else:
#add settings support, only support simple situation
#so for complex cases you should still write module
#format just like:
#
#[TEMPLATE_USE]
#name = {
# 'toplinks':[
# 'myapp/jquery.myapp.{version}.min.js',
# ],
# 'depends':[xxxx],
# 'config':{'version':'UI_CONFIG/test'},
# 'default':{'version':'1.2.0'},
#}
#
mod = None
c = settings.get_var('TEMPLATE_USE/'+plugin)
if c:
config = c.pop('config', {})
default = c.pop('default', {})
#evaluate config value
config = dict([(k, settings.get_var(v, default.get(k, ''))) for k, v in config.items()])
#merge passed arguments
config.update(kwargs)
for t in ['toplinks', 'bottomlinks']:
if t in c:
c[t] = [x.format(**config) for x in c[t]]
mod = c
else:
for p in app.apps:
if not is_pyfile_exist(os.path.join(get_app_dir(p), 'template_plugins'), plugin):
continue
module = '.'.join([p, 'template_plugins', plugin])
try:
mod = __import__(module, {}, {}, [''])
except ImportError, e:
log.exception(e)
mod = None
if mod:
UseNode.__saved_template_plugins_modules__[plugin] = mod
else:
log.error("Can't find the [%s] template plugin, please check if you've installed special app already" % plugin)
if settings.get_var('TEMPLATE/RAISE_USE_EXCEPTION'):
raise UseModuleNotFound("Can't find the %s template plugin, check if you've installed special app already" % plugin)
def get_cache(**kwargs):
from uliweb import settings
from weto.cache import Cache
from uliweb.utils.common import import_attr
serial_cls_name = settings.get_var('CACHE/serial_cls', 'weto.cache.Serial')
args = {'storage_type':settings.get_var('CACHE/type'),
'options':settings.get_var('CACHE_STORAGE'),
'expiry_time':settings.get_var('CACHE/expiretime'),
'serial_cls':import_attr(serial_cls_name)}
args.update(kwargs)
cache = Cache(**args)
return cache
def run_command(row):
import json
from uliweb import settings
keys_queue = settings.get_var('ASYNC_TASKS/tasks_keys_queue')
kw = row.command_info.copy()
kw['task_id'] = row.task_id
queue = kw['queue']
try:
redis = functions.get_redis()
if not isinstance(queue, (tuple, list)):
queue = [queue]
msg = json.dumps(kw)
#在添加消息到公共队列的同时,将任务ID添加到keys_queue中,用来记录当前将要,或正在执行的任务
for q in queue:
pipe = redis.pipeline()
name = settings.ASYNC_TASKS_QUEUES.get(q or 'default') or q
pipe.lpush(name, msg).sadd(keys_queue, row.task_id)
pipe.execute()
log.info('Put task_id {} to queue=[{}] and keys_queue=[{}]'.format(
row.task_id, name, keys_queue))
except Exception as e:
_type, value, tb = sys.exc_info()
txt = ''.join(traceback.format_exception(_type, value, tb))
handler = row.get_handler(log=log)
handler.save(status=ERROR,
execution_info=txt,
finished_time=date.now())
log.error('Run async task {} failed when push with redis'.format(
row.task_id))
log.exception(e)
def call(args, options, global_options):
from uliweb import settings
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
check_point = settings.get_var('ASYNC_TASKS/check_point')
log.info('=============================')
log.info(' Async Task Daemon Starting')
log.info('=============================')
log.info('Daemon process %d' % os.getpid())
log.info('Check point %ds' % check_point)
Task = functions.get_model('async_tasks')
query = Task.filter(~Task.c.status.in_([SUCCESS, CANCEL, FAILED])).\
order_by(Task.c.created_time)
redis = functions.get_redis()
while not is_exit:
Reset()
for row in query:
try:
#log.debug('Ready to process async task {} - {}'.format(row.task_id, row.command_name))
process_task(row)
except Exception as e:
log.exception(e)
beat = redis.brpop('async_tasks_beat', check_point)
#to prevent database submit slow than redis
time.sleep(0.5)
def call(args, options, global_options):
from uliweb import settings
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
check_point = settings.get_var('ASYNC_TASKS/check_point')
log.info('=============================')
log.info(' Async Task Daemon Starting')
log.info('=============================')
log.info('Daemon process %d' % os.getpid())
log.info('Check point %ds' % check_point)
Task = functions.get_model('async_tasks')
query = Task.filter(~Task.c.status.in_([SUCCESS, CANCEL, FAILED])).\
order_by(Task.c.created_time)
redis = functions.get_redis()
while not is_exit:
Reset()
for row in query:
try:
#log.debug('Ready to process async task {} - {}'.format(row.task_id, row.command_name))
process_task(row)
except Exception as e:
log.exception(e)
beat = redis.brpop('async_tasks_beat', check_point)
#to prevent database submit slow than redis
time.sleep(0.5)
def delete_user_session():
from uliweb import settings, request
user_fieldname = settings.get_var('AUTH/GET_AUTH_USER_FIELDNAME', 'id')
share_session = settings.get_var('AUTH/AUTH_SHARE_USER_SESSION', False)
user = request.user
if user and share_session:
if isinstance(user, dict):
user_id = user[user_fieldname]
else:
user_id = getattr(user, user_fieldname)
cache = functions.get_cache()
key = get_user_session_key(user_id)
log.debug('Auth: delete user session, userid={}, session_id={}'.format(
key, request.session.key))
cache.delete(key)
def get_state_name(self):
if self.obj:
return self.obj.get_display_value('state')
else:
from uliweb import settings
choices = settings.get_var('PARA/WF_STATUS')
return choices[self.state]
def get_id(engine, tablename, id):
from uliweb import settings
d = {'engine': engine, 'tablename': str(tablename), 'id': str(id)}
format = settings.get_var('OBJCACHE/key_format',
'OC:%(engine)s:%(tablename)s:%(id)s')
return format % d
def post_delete(model, instance):
from uliweb import response, settings
if not check_enable():
return
tablename = model.tablename
if tablename not in settings.get_var('OBJCACHE_TABLES'):
return
def f():
redis = get_redis()
if not redis: return
info = settings.get_var('OBJCACHE_TABLES/%s' % tablename)
key = 'id'
if info and isinstance(info, dict):
key = info.get('key', key)
if '.' in key or key not in model.properties:
_key = import_attr(key)(instance)
else:
_key = getattr(instance, key)
_id = get_id(model.get_engine_name(), tablename, _key)
try:
redis.delete(_id)
log.debug("objcache:post_delete:id=" + _id)
except Exception, e:
log.exception(e)
def post_save(model, instance, created, data, old_data):
from uliweb import response, settings
if not check_enable():
return
tablename = model.tablename
if tablename not in settings.get_var('OBJCACHE_TABLES'):
return
#if response is False, then it means you may in batch program
#so it can't use post_commit machenism
def f():
fields = get_fields(tablename)
flag = created
#if update then check if the record has changed
if not flag:
if not fields:
flag = True
else:
fields = model.properties.keys()
flag = bool(filter(lambda x: x in data, fields))
if flag:
set_object(model, instance)
log.debug("objcache:post_save:id=%d" % instance.id)
if response:
response.post_commit = f
else:
f()
def get_object(model, id, engine_name=None, connection=None):
"""
Get cached object from redis
if id is None then return None:
"""
from uliweb import settings
if not id:
return
if not check_enable():
return
redis = get_redis()
if not redis: return
tablename = model.tablename
info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {})
if info is None:
return
_id = get_id(engine_name or model.get_engine_name(), tablename, id)
try:
if redis.exists(_id):
v = redis.hgetall(_id)
o = model.load(v)
log.debug("objcache:get:id="+_id)
return o
except Exception, e:
log.exception(e)
def startup_installed(sender):
from uliweb import settings
import uliweb.form.uliform as form
from uliweb.utils.common import import_attr
for k, v in settings.get_var('FORM_FIELDS_MAP', {}).items():
form.fields_mapping[k] = import_attr(v)
def get_object(model, cid, engine_name=None, connection=None):
"""
Get cached object from redis
if id is None then return None:
"""
from uliweb import settings
if not id:
return
if not check_enable():
return
redis = get_redis()
if not redis: return
tablename = model._alias or model.tablename
info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {})
if info is None:
return
_id = get_id(engine_name or model.get_engine_name(), tablename, cid)
try:
log.debug("Try to find objcache:get:table=%s:id=[%s]" % (tablename, _id))
if redis.exists(_id):
v = redis.hgetall(_id)
o = model.load(v, from_dump=True)
log.debug("Found!")
return o
else:
log.debug("Not Found!")
except Exception, e:
log.exception(e)
def get_state_name(self):
if self.obj:
return self.obj.get_display_value('state')
else:
from uliweb import settings
choices = settings.get_var('PARA/WF_STATUS')
return choices[self.state]
def get_object(model, id, engine_name=None, connection=None):
"""
Get cached object from redis
if id is None then return None:
"""
from uliweb import settings
if not id:
return
if not check_enable():
return
redis = get_redis()
if not redis: return
tablename = model.tablename
info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {})
if info is None:
return
_id = get_id(engine_name or model.get_engine_name(), tablename, id)
try:
if redis.exists(_id):
v = redis.hgetall(_id)
o = model.load(v)
log.debug("objcache:get:id=" + _id)
return o
except Exception, e:
log.exception(e)
def get_object(model, cid, engine_name=None, connection=None):
"""
Get cached object from redis
if id is None then return None:
"""
from uliweb import settings
if not id:
return
if not check_enable():
return
redis = get_redis()
if not redis: return
tablename = model._alias or model.tablename
info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {})
if info is None:
return
_id = get_id(engine_name or model.get_engine_name(), tablename, cid)
try:
log.debug("Try to find objcache:get:table=%s:id=[%s]" % (tablename, _id))
if redis.exists(_id):
v = redis.hgetall(_id)
o = model.load(v, from_='dump')
log.debug("Found!")
return o
else:
log.debug("Not Found!")
except Exception as e:
log.exception(e)
def get_menu(cls, parent):
from uliweb import functions, settings
from uliweb.utils.common import Serial
def _f(_p):
menus = []
for row in cls.filter(cls.c.parent == _p, cls.c.type == 'M0000',
cls.c.deploy_type == 'F',
cls.c.status == '00001').order_by(
cls.c.order):
item = row.to_dict()
item['link'] = row.value
item['permissions'] = [x.name for x in row.permissions]
menus.append(item)
if row.has_children:
item['subs'] = _f(item.id)
else:
item['subs'] = []
return menus
menus = []
use_redis = settings.get_var('RESOURCE/USE_REDIS')
key = 'MENU:{}'.format(parent)
if use_redis:
redis = functions.get_redis()
v = redis.get(key)
if v:
menus = Serial.load(v)
if not menus:
p = cls.get(cls.c.name == parent)
menus = _f(p.id)
if menus and use_redis:
redis.set(key, Serial.dump(menus))
return menus
def run_command(row):
import json
from uliweb import settings
keys_queue = settings.get_var('ASYNC_TASKS/tasks_keys_queue')
kw = row.command_info.copy()
kw['task_id'] = row.task_id
queue =kw['queue']
try:
redis = functions.get_redis()
if not isinstance(queue, (tuple, list)):
queue = [queue]
msg = json.dumps(kw)
#在添加消息到公共队列的同时,将任务ID添加到keys_queue中,用来记录当前将要,或正在执行的任务
for q in queue:
pipe = redis.pipeline()
name = settings.ASYNC_TASKS_QUEUES.get(q or 'default') or q
pipe.lpush(name, msg).sadd(keys_queue, row.task_id)
pipe.execute()
log.info('Put task_id {} to queue=[{}] and keys_queue=[{}]'.format(row.task_id, name, keys_queue))
except Exception as e:
_type, value, tb = sys.exc_info()
txt = ''.join(traceback.format_exception(_type, value, tb))
handler = row.get_handler(log=log)
handler.save(status=ERROR, execution_info=txt, finished_time=date.now())
log.error('Run async task {} failed when push with redis'.format(row.task_id))
log.exception(e)
def test_text(self, content_type):
from uliweb.utils.common import match
from uliweb import settings
m = content_type.split(';', 1)[0]
r = match(m, settings.get_var('ULIWEBRECORDER/text_content_types'))
return r
def get_var(key, default=None):
"""
Default get_var function
"""
from uliweb import settings
return settings.get_var(key, default)
def delete_user_session():
from uliweb import settings, request
user_fieldname = settings.get_var('AUTH/GET_AUTH_USER_FIELDNAME', 'id')
share_session = settings.get_var('AUTH/AUTH_SHARE_USER_SESSION', False)
user = request.user
if user and share_session:
if isinstance(user, dict):
user_id = user[user_fieldname]
else:
user_id = getattr(user, user_fieldname)
cache = functions.get_cache()
key = get_user_session_key(user_id)
log.debug('Auth: delete user session, userid={}, session_id={}'.format(
key, request.session.key))
cache.delete(key)
def test_text(self, content_type):
from uliweb.utils.common import match
from uliweb import settings
m = content_type.split(';', 1)[0]
r = match(m, settings.get_var('ULIWEBRECORDER/text_content_types'))
return r
def _wiki_view(self, pagename):
from uliweb import request, response
response.template = 'WikiView/wiki_view.html'
if not pagename:
if 'pages' in request.GET:
response.template = 'WikiView/wiki_pages.html'
objects = self.model.filter(self.model.c.enabled==True).filter(self.model.c.deleted==False).fields('name').order_by(self.model.c.name)
return {'objects':objects}
return redirect(url_for(self.__class__.wiki, pagename='Index'))
wiki = self.model.get((self.model.c.name == pagename) & (self.model.c.enabled==True))
if not wiki:
#check read permission
self._check_permission('read')
if pagename == 'Index':
content = settings.WIKI.IndexContent
wiki = self.model(name='Index', content=content, enabled=True, creator=request.user, modified_user=request.user)
wiki.save()
wiki.new_revision()
if wiki:
#check read permission
self._check_permission('read', wiki, page=wiki)
#处理阅读次数
key = '__wikivisited__:%s:%d' % (request.remote_addr, wiki.id)
cache = functions.get_cache()
v = cache.get(key, None)
if not v:
self.model.filter(self.model.c.id==int(wiki.id)).update(hits=self.model.c.hits+1)
cache.set(key, 1, settings.get_var('WIKI/WIKI_VISITED_TIMEOUT'))
rev = int(request.GET.get('rev', 0))
rev_time = None
if rev:
cached_pagename = wiki.name + '&ver=%d' % rev
r = self.changeset.get((self.changeset.c.wiki==wiki.id) & (self.changeset.c.revision==rev))
rev_time = r.modified_time and r.modified_time.strftime('%Y-%m-%d %H:%M:%S')
content, kwargs = self._get_cached_page_html(cached_pagename, r.old_content)
else:
cached_pagename = wiki.name
content, kwargs = self._get_cached_page_html(cached_pagename, wiki.content)
last_rev = wiki.changeset.order_by(self.changeset.c.id.desc()).one().revision
return {'wiki':wiki, 'content':content, 'revision':rev,
'rev_time':rev_time, 'last_rev':last_rev,
'page_args':kwargs,
'create_new':self._check_permission('write', raise_exception=False),
'permissions':self._find_tree_permissions(['read', 'write', 'delete'], wiki, request.user)
}
else:
#check read permission
self._check_permission('write')
return redirect(url_for(self.__class__.wiki, pagename=pagename, action='edit'))
def startup_installed(sender):
from uliweb import settings
import uliweb.form.uliform as form
from uliweb.utils.common import import_attr
for k, v in settings.get_var('FORM_FIELDS_MAP', {}).items():
form.fields_mapping[k] = import_attr(v)
def get_var(key, default=None):
"""
Default get_var function
"""
from uliweb import settings
return settings.get_var(key, default)
def post_delete(model, instance):
from uliweb import response, settings
if not check_enable():
return
tablename = model.tablename
if tablename not in settings.get_var('OBJCACHE_TABLES'):
return
def f():
redis = get_redis()
if not redis: return
info = settings.get_var('OBJCACHE_TABLES/%s' % tablename)
key = 'id'
if info and isinstance(info, dict):
key = info.get('key', key)
if '.' in key or key not in model.properties:
_key = import_attr(key)(instance)
else:
_key = getattr(instance, key)
_id = get_id(model.get_engine_name(), tablename, _key)
try:
redis.delete(_id)
log.debug("objcache:post_delete:id="+_id)
except Exception, e:
log.exception(e)
def post_save(model, instance, created, data, old_data):
from uliweb import response, settings
if not check_enable():
return
tablename = model.tablename
if tablename not in settings.get_var('OBJCACHE_TABLES'):
return
#if response is False, then it means you may in batch program
#so it can't use post_commit machenism
def f():
fields = get_fields(tablename)
flag = created
#if update then check if the record has changed
if not flag:
if not fields:
flag = True
else:
fields = model.properties.keys()
flag = bool(filter(lambda x:x in data, fields))
if flag:
set_object(model, instance)
log.debug("objcache:post_save:id=%d" % instance.id)
if response:
response.post_commit = f
else:
f()
def __init__(self,
default_filename_converter_cls=UUIDFilenameConverter,
config=None):
self.config = config or self.default_config
for k, v in self.options.items():
item, default = v
#if there is no '/' in option, then combine config with option
#else assume the option is just like 'SECTION/OPTION', then skip it
if '/' not in item:
item = self.config + '/' + item
value = settings.get_var(item, default)
setattr(self, k, value)
if self.x_sendfile and not self.x_header_name:
if self.x_sendfile == 'nginx':
self.x_header_name = 'X-Accel-Redirect'
elif self.x_sendfile == 'apache':
self.x_header_name = 'X-Sendfile'
else:
raise Exception(
"X_HEADER can't be None, or X_SENDFILE is not supprted")
if isinstance(self._filename_converter, (str, unicode)):
self._filename_converter_cls = import_attr(
self._filename_converter)
else:
self._filename_converter_cls = self._filename_converter or default_filename_converter_cls
def set_object(model, instance, fields=None, engine_name=None):
"""
Only support simple condition, for example: Model.c.id == n
if not id provided, then use instance.id
"""
from uliweb import settings
if not check_enable():
return
redis = get_redis()
if not redis: return
tablename = model._alias or model.tablename
exclude = []
if not fields:
fields, exclude = get_fields(tablename)
v = instance.dump(fields, exclude=exclude)
info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {})
if info is None:
return
expire = settings.get_var('OBJCACHE/timeout', 0)
key = 'id'
if info and isinstance(info, dict):
expire = info.get('expire', expire)
key = info.get('key', key)
if '.' in key or key not in model.properties:
_key = import_attr(key)(instance)
else:
_key = getattr(instance, key)
_id = get_id(engine_name or model.get_engine_name(), tablename, _key)
try:
pipe = redis.pipeline()
p = pipe.delete(_id).hmset(_id, v)
expire_msg = ''
if expire:
p = p.expire(_id, expire)
expire_msg = ':expire=%d' % expire
r = p.execute()
log.debug("Saving to cache objcache:set:table=%s:id=[%s]:%s" %
(tablename, _id, expire_msg))
except Exception, e:
log.exception(e)
def get_key(tablename):
from uliweb import settings
tables = settings.get_var('OBJCACHE/config')
if not tablename in tables:
return
return tables[tablename]
def set_object(model, instance, fields=None, engine_name=None):
"""
Only support simple condition, for example: Model.c.id == n
if not id provided, then use instance.id
"""
from uliweb import settings
if not check_enable():
return
redis = get_redis()
if not redis: return
tablename = model._alias or model.tablename
exclude = []
if not fields:
fields, exclude = get_fields(tablename)
v = instance.dump(fields, exclude=exclude)
info = settings.get_var('OBJCACHE_TABLES/%s' % tablename, {})
if info is None:
return
expire = settings.get_var('OBJCACHE/timeout', 0)
key = 'id'
if info and isinstance(info, dict):
expire = info.get('expire', expire)
key = info.get('key', key)
if '.' in key or key not in model.properties:
_key = import_attr(key)(instance)
else:
_key = getattr(instance, key)
_id = get_id(engine_name or model.get_engine_name(), tablename, _key)
try:
pipe = redis.pipeline()
p = pipe.delete(_id).hmset(_id, v)
expire_msg = ''
if expire:
p = p.expire(_id, expire)
expire_msg = ':expire=%d' % expire
r = p.execute()
log.debug("Saving to cache objcache:set:table=%s:id=[%s]:%s" % (tablename, _id, expire_msg))
except Exception, e:
log.exception(e)
def get_fields(tablename):
from uliweb import settings
tables = settings.get_var('OBJCACHE_TABLES')
if not tablename in tables:
return
return tables[tablename]
def init(self):
from uliweb import settings
self.redis = functions.get_redis()
self.queue = self.kwargs['queue']
self.work_queue = self.queue + ':' + self.name
self.keys_queue = settings.get_var('ASYNC_TASKS/tasks_keys_queue')
#todo log
self.log.info(
'{} {} created with queue={}, work queue={} timeout={} '
'max_requests={} soft_memory_limit={} hard_memory_limit={}'.format(
self.name, self.pid, self.queue, self.work_queue, self.timeout,
self.max_requests, self.soft_memory_limit,
self.hard_memory_limit))
self.check_point = settings.get_var('ASYNC_TASKS_WORKERS/check_point')
def _upload_file(self, image=False, show_filename=True):
import os
import Image
if image:
from forms import ImageUploadForm as Form
else:
from forms import FileUploadForm as Form
from uliweb.utils.image import thumbnail_image, fix_filename
from uliweb import json_dumps
File = get_model('forumattachment')
forum_id = request.GET.get('forum_id')
slug = request.GET.get('slug')
form = Form()
suffix = date.now().strftime('_%Y_%m_%d')
if request.method == 'GET':
form.bind({'is_thumbnail': True})
return {'form': form}
else:
flag = form.validate(request.values, request.files)
if flag:
f = form.data['filedata']
_f = os.path.basename(f['filename'])
#文件格式为forum/<forum_id>/<filename_yyyy_mm_dd>
filename = fix_filename('forum/%s/%s' % (forum_id, _f), suffix)
if image:
filename = functions.save_file(filename, f['file'])
if form.data['is_thumbnail']:
#process thumbnail
rfilename, thumbnail = thumbnail_image(
functions.get_filename(filename, filesystem=True),
filename,
settings.get_var('PARA/FORUM_THUMBNAIL_SIZE'))
_file = functions.get_href(thumbnail)
else:
_file = functions.get_href(filename)
name = functions.get_href(filename)
else:
filename = functions.save_file(filename, f['file'])
name = form.data['title']
if not name:
name = _f
_file = functions.get_href(filename, alt=name)
ff = File(slug=slug, file_name=filename, name=name)
ff.save()
name = json_dumps(name, unicode=True)
if show_filename:
fargs = '||%s' % name[1:-1]
else:
fargs = ''
return '''<script type="text/javascript">
var url='%s%s';
setTimeout(function(){callback(url);},100);
</script>
''' % (_file, fargs)
else:
return {'form': form}
def success_data(obj, data):
import math
return {
'page':
int(
math.ceil(1.0 * obj.floor /
settings.get_var('PARA/FORUM_PAGE_NUMS')))
}
def __init__(self,
host=None,
port=None,
user=None,
password=None,
backend=None):
from uliweb import settings
from uliweb.utils.common import import_attr
self.host = host or (settings and settings.get_var('MAIL/HOST'))
self.port = port or (settings and settings.get_var('MAIL/PORT', 25))
self.user = user or (settings and settings.get_var('MAIL/USER'))
self.password = password or (settings
and settings.get_var('MAIL/PASSWORD'))
self.backend = backend or (settings and settings.get_var(
'MAIL/BACKEND', 'uliweb.mail.backends.smtp'))
cls = import_attr(self.backend + '.MailConnection')
self.con = cls(self)
