def list_recent_edit(parent_id=None, offset=0, limit=None):
if limit is None:
limit = xconfig.PAGE_SIZE
db = xtables.get_file_table()
t = Timer()
t.start()
creator = xauth.get_current_name()
if creator:
where = "is_deleted = 0 AND (creator = $creator) AND type != 'group'"
else:
# 未登录
where = "is_deleted = 0 AND is_public = 1 AND type != 'group'"
cache_key = "[%s]note.recent$%s$%s" % (creator, offset, limit)
files = cacheutil.get(cache_key)
if files is None:
files = list(
db.select(
what=
"name, id, parent_id, ctime, mtime, type, creator, priority",
where=where,
vars=dict(creator=creator),
order="priority DESC, mtime DESC",
offset=offset,
limit=limit))
for item in files:
item.parent_name = query_note_name(item.parent_id)
cacheutil.set(cache_key, files, expire=600)
t.stop()
xutils.trace("NoteDao.ListRecentEdit", "", t.cost_millis())
return files
def add_user(name, password):
if name == "" or name == None:
return
if password == "" or password == None:
return
if not is_valid_username(name):
return dict(code="INVALID_NAME", message="非法的用户名")
name = name.lower()
found = find_by_name(name)
if found is not None:
found.mtime = xutils.format_time()
found.salt = textutil.random_string(6)
found.token = gen_new_token()
found.password = password
update_user(name, found)
else:
user = Storage(name=name,
password=password,
token=gen_new_token(),
ctime=xutils.format_time(),
salt=textutil.random_string(6),
mtime=xutils.format_time())
dbutil.put("user:%s" % name, user)
xutils.trace("UserAdd", name)
refresh_users()
def rdb_search_message(user_name, key, offset, limit):
db = xtables.get_message_table()
vars = dict(user=user_name)
kw = "user = $user"
start_time = time.time()
for item in key.split(" "):
if item == "":
continue
kw += " AND content LIKE " + fuzzy_item(item)
# when find numbers, the sql printed is not correct
# eg. LIKE '%1%' will be LIKE '%'
chatlist = list(
db.select(where=kw,
vars=vars,
order="ctime DESC",
limit=limit,
offset=offset))
end_time = time.time()
cost_time = int((end_time - start_time) * 1000)
xutils.trace("MessageSearch", key, cost_time)
if xconfig.search_history is not None:
xconfig.search_history.add(key, cost_time)
amount = db.count(where=kw, vars=vars)
return chatlist, amount
def _get_users():
"""获取用户，内部接口"""
global _users
# 有并发风险
if _users is not None:
return _users
_users = {}
# 默认的账号
_users["admin"] = Storage(name = "admin",
password = "******",
salt = "",
mtime = "",
token = gen_new_token())
user_list = dbutil.prefix_list("user")
for user in user_list:
if user.name is None:
xutils.trace("UserList", "invalid user %s" % user)
continue
if isinstance(user.config, dict):
user.config = Storage(**user.config)
else:
user.config = Storage()
name = user.name.lower()
_users[name] = user
return _users
def read_file(self, path):
# 强制缓存
if not xconfig.DEBUG:
web.header("Cache-Control", "max-age=3600")
environ = web.ctx.environ
etag = '"%s"' % os.path.getmtime(path)
client_etag = environ.get('HTTP_IF_NONE_MATCH')
web.header("Etag", etag)
self.handle_content_type(path)
# self.handle_content_encoding(ext)
if etag == client_etag:
web.ctx.status = "304 Not Modified"
return b'' # 其实webpy已经通过yield空bytes来避免None
else:
http_range = environ.get("HTTP_RANGE")
blocksize = 64 * 1024
# print_env()
if http_range is not None:
xutils.trace("==> HTTP_RANGE {}", http_range)
return self.read_range(path, http_range, blocksize)
else:
return self.read_all(path, blocksize)
def POST(self):
user_name = xauth.current_name()
part_file = True
chunksize = 5 * 1024 * 1024
chunk = xutils.get_argument("chunk", 0, type=int)
chunks = xutils.get_argument("chunks", 1, type=int)
file = xutils.get_argument("file", {})
prefix = xutils.get_argument("prefix", "")
dirname = xutils.get_argument("dirname", xconfig.DATA_DIR)
dirname = dirname.replace("$DATA", xconfig.DATA_DIR)
note_id = xutils.get_argument("note_id")
# 不能访问上级目录
if ".." in dirname:
return dict(code="fail", message="can not access parent directory")
filename = None
webpath = ""
origin_name = ""
if hasattr(file, "filename"):
origin_name = file.filename
xutils.trace("UploadFile", file.filename)
filename = os.path.basename(file.filename)
filename = xutils.get_real_path(filename)
if dirname == "auto":
filename = generate_filename(filename, prefix)
filepath, webpath = xutils.get_upload_file_path(user_name, filename, replace_exists=True)
dirname = os.path.dirname(filepath)
filename = os.path.basename(filepath)
else:
# TODO check permission.
pass
if part_file:
tmp_name = "%s_%d.part" % (filename, chunk)
seek = 0
else:
tmp_name = filename
seek = chunk * chunksize
xutils.makedirs(dirname)
tmp_path = os.path.join(dirname, tmp_name)
with open(tmp_path, "wb") as fp:
fp.seek(seek)
if seek != 0:
xutils.log("seek to {}", seek)
for file_chunk in file.file:
fp.write(file_chunk)
else:
return dict(code="fail", message="require file")
if part_file and chunk+1==chunks:
self.merge_files(dirname, filename, chunks)
try_touch_note(note_id)
if note_id != None and note_id != "":
xutils.call("note.touch", note_id)
return dict(code="success", webpath=webpath, link=get_link(origin_name, webpath))
def update_user(name, user):
if name == "" or name == None:
return
name = name.lower()
mem_user = _users[name]
mem_user.update(user)
dbutil.put("user:%s" % name, mem_user)
xutils.trace("UserUpdate", mem_user)
def check_and_run(task, tm):
if self.match(task, tm):
put_task(request_url, task)
try:
xutils.trace("RunTask", task.url)
if task.tm_wday == "no-repeat":
# 一次性任务直接删除
dbutil.delete(task.id)
self.load_tasks()
except Exception as e:
xutils.log("run task [%s] failed, %s" % (task.url, e))
def check_and_run(task, tm):
if self.match(task, tm):
put_task(request_url, task)
try:
xutils.trace("RunTask", task.url)
if task.tm_wday == "no-repeat":
# 一次性任务直接删除
xtables.get_schedule_table().delete(where=dict(id=task.id))
self.load_tasks()
except Exception as e:
xutils.log("run task [%s] failed, %s" % (task.url, e))
def count_ungrouped(creator):
t = Timer()
t.start()
count_key = "*****@*****.**" % creator
count = cacheutil.get(count_key)
if count is None:
count = xtables.get_file_table().count(where="creator=$creator AND parent_id=0 AND is_deleted=0 AND type!='group'",
vars=dict(creator=creator))
xutils.cache_put(count_key, count, expire=600)
t.stop()
xutils.trace("NoteDao.CountUngrouped", "", t.cost_millis())
return count
def execute(self, ctx=None):
try:
matched = self.pattern.match(ctx.key)
if not matched:
return
start = time.time()
ctx.groups = matched.groups()
self.func(ctx)
stop = time.time()
xutils.trace("SearchHandler", self.key, int((stop - start) * 1000))
except:
xutils.print_exc()
def on_reload(ctx=None):
for key in ('THEME', 'FS_HIDE_FILES', 'OPTION_STYLE', 'PAGE_OPEN',
'RECENT_SEARCH_LIMIT', "PAGE_SIZE", "RECENT_SIZE"):
value = cacheutil.hget('sys.config', key)
xutils.trace("HGET", "key=%s, value=%s" % (key, value))
if value is not None:
setattr(xconfig, key, value)
path = os.path.join(xconfig.SCRIPTS_DIR, "user.css")
if not os.path.exists(path):
return
xconfig.set("USER_CSS", xutils.readfile(path))
def count_user_note(creator):
t = Timer()
t.start()
count_key = "[%s]note.count" % creator
count = cacheutil.get(count_key)
if count is None:
db = xtables.get_file_table()
where = "is_deleted = 0 AND creator = $creator AND type != 'group'"
count = db.count(where, vars=dict(creator=xauth.get_current_name()))
cacheutil.set(count_key, count, expire=600)
t.stop()
xutils.trace("NoteDao.CountRecentEdit", "", t.cost_millis())
return count
def execute(self, ctx=None):
if self.is_async:
put_task(self.func, ctx)
else:
try:
if self.profile:
start = time.time()
self.func(ctx)
if self.profile:
stop = time.time()
xutils.trace("EventHandler", self.key, int((stop-start)*1000))
except:
xutils.print_exc()
def on_reload(ctx=None):
keys = ("THEME", 'FS_HIDE_FILES', 'OPTION_STYLE', 'PAGE_OPEN',
'RECENT_SEARCH_LIMIT', "PAGE_SIZE", "RECENT_SIZE",
"RECORD_LOCATION", "TRASH_EXPIRE", "PAGE_WIDTH", "FS_VIEW_MODE")
for key in keys:
value = cacheutil.hget('sys.config', key)
xutils.trace("HGET", "key=%s, value=%s" % (key, value))
if value is not None:
setattr(xconfig, key, value)
path = os.path.join(xconfig.SCRIPTS_DIR, "user.css")
if not os.path.exists(path):
return
xconfig.USER_CSS = xutils.readfile(path)
def add_handler(self, handler):
"""
注册事件处理器
事件处理器的去重,通过判断是不是同一个函数，不通过函数名，如果修改初始化脚本需要执行【重新加载模块】功能
"""
event_type = handler.event_type
handlers = self._handlers.get(event_type, [])
if handler in handlers:
warn("handler %s is already registered" % handler)
return
# XXX 使用str(handler)在Python2.7环境下报错
xutils.trace("EventRegister", "%s" % handler)
handlers.append(handler)
self._handlers[event_type] = handlers
def list_group(current_name=None):
if current_name is None:
current_name = str(xauth.get_current_name())
t1 = time.time()
cache_key = "[%s]note.group.list" % current_name
value = cacheutil.get(cache_key)
if value is None:
sql = "SELECT * FROM file WHERE type = 'group' AND is_deleted = 0 AND creator = $creator ORDER BY name LIMIT 1000"
value = list(xtables.get_file_table().query(
sql, vars=dict(creator=current_name)))
cacheutil.set(cache_key, value, expire=600)
t2 = time.time()
xutils.trace("NoteDao.ListGroup", "", int((t2 - t1) * 1000))
return value
def update_user(name, user):
if name == "" or name == None:
return
name = name.lower()
mem_user = find_by_name(name)
if mem_user is None:
raise Exception("user not found")
mem_user.update(user)
dbutil.put("user:%s" % name, mem_user)
xutils.trace("UserUpdate", mem_user)
refresh_users()
def list_tag(user_name):
t = Timer()
t.start()
cache_key = "%s@tag_list" % user_name
tag_list = xutils.cache_get(cache_key)
tag_list = None
if tag_list is None:
db = xtables.get_file_tag_table()
sql = """SELECT LOWER(name) AS name, COUNT(*) AS amount FROM file_tag
WHERE (user=$user OR is_public=1)
GROUP BY LOWER(name) ORDER BY amount DESC, name ASC"""
tag_list = list(db.query(sql, vars=dict(user=user_name)))
xutils.cache_put(cache_key, tag_list, 60 * 10)
t.stop()
xutils.trace("NoteDao.ListTag", "", t.cost_millis())
return tag_list
def list_recent_viewed(creator=None, offset=0, limit=10):
t = Timer()
t.start()
where = "is_deleted = 0 AND (creator = $creator)"
db = xtables.get_file_table()
result = list(
db.select(where=where,
vars=dict(creator=creator),
order="atime DESC",
offset=offset,
limit=limit))
for item in result:
item.parent_name = query_note_name(item.parent_id)
t.stop()
xutils.trace("NoteDao.ListRecentViewed", "", t.cost_millis())
return result
def GET(self):
pagesize = xutils.get_argument("pagesize", xconfig.PAGE_SIZE, type=int)
page = xutils.get_argument("page", 1, type=int)
status = xutils.get_argument("status")
key = xutils.get_argument("key")
offset = (page-1) * pagesize
db = xtables.get_message_table()
user_name = xauth.get_current_name()
# 未完成任务的分页
undone_pagesize = 1000
kw = "1=1"
if status == "created":
kw = "status = 0"
pagesize = undone_pagesize
if status == "done":
kw = "status = 100"
if status == "suspended":
kw = "status = 50"
pagesize = undone_pagesize
kw += " AND user = $user"
vars = dict(user=xauth.get_current_name())
if key != "" and key != None:
start_time = time.time()
for item in key.split(" "):
if item == "":
continue
kw += " AND content LIKE " + fuzzy_item(item)
# when find numbers, the sql printed is not correct
# eg. LIKE '%1%' will be LIKE '%'
# print(kw)
chatlist = list(db.select(where=kw, vars=vars, order="ctime DESC", limit=pagesize, offset=offset))
end_time = time.time()
cost_time = int((end_time-start_time)*1000)
xutils.trace("MessageSearch", key, cost_time)
if xconfig.search_history is not None:
xconfig.search_history.add(key, cost_time)
else:
chatlist = list(db.select(where=kw, vars=vars, order="ctime DESC", limit=pagesize, offset=offset))
chatlist.reverse()
amount = db.count(where=kw, vars=vars)
page_max = math.ceil(amount / pagesize)
chatlist = list(map(process_message, chatlist))
return dict(code="success", message="",
pagesize = pagesize,
data=chatlist, amount=amount,
page_max=page_max, current_user=xauth.get_current_name())
def read_range(self, path, http_range, blocksize):
range_list = http_range.split("bytes=")
if len(range_list) == 2:
# 包含完整的范围
range_list = range_list[1]
try:
range_start, range_end = range_list.split('-')
range_start = int(range_start)
total_size = os.stat(path).st_size
if range_end != "":
range_end = int(range_end)
else:
range_end = total_size - 1
web.header("Content-Length", total_size)
content_range = "bytes %s-%s/%s" % (range_start, range_end,
total_size)
# 设置HTTP响应状态
web.ctx.status = "206 Partial Content"
# 发送HTTP首部
web.header("Accept-Ranges", "bytes")
web.header("Content-Range", content_range)
xutils.trace("Download",
"<== Content-Range:%s" % content_range)
# 发送数据
fp = open(path, "rb")
try:
fp.seek(range_start)
rest = range_end - range_start + 1
readsize = min(rest, blocksize)
while readsize > 0:
# print("%s send %s K" % (time.ctime(), readsize))
yield fp.read(readsize)
rest -= readsize
readsize = min(rest, blocksize)
finally:
# 基本上和with等价，这里打印出来
xutils.trace("Download", "close %s" % path)
fp.close()
except Exception as e:
# 其他未知异常
xutils.print_stacktrace()
# yield最好不要和return混用
yield self.read_all(path, blocksize)
else:
# 处理不了，返回所有的数据
yield self.read_all(path, blocksize)
def apply_search_rules(ctx, key):
files = []
for rule in _RULES:
pattern = rule.pattern
func = rule.func
# re.match内部已经实现了缓存
m = re.match(pattern, key)
if m:
try:
start_time0 = time.time()
results = func(ctx, *m.groups())
cost_time0 = time.time() - start_time0
xutils.trace("SearchHandler", func.modfunc, int(cost_time0*1000))
if results is not None:
files += results
except Exception as e:
xutils.print_exc()
return files
def list_recent_created(parent_id=None, offset=0, limit=10):
t = Timer()
t.start()
where = "is_deleted = 0 AND (creator = $creator)"
if parent_id != None:
where += " AND parent_id = %s" % parent_id
db = xtables.get_file_table()
result = list(
db.select(where=where,
vars=dict(creator=xauth.get_current_name()),
order="ctime DESC",
offset=offset,
limit=limit))
for item in result:
item.parent_name = query_note_name(item.parent_id)
t.stop()
xutils.trace("NoteDao.ListRecentCreated", "", t.cost_millis())
return result
def add_user(name, password):
if name == "" or name == None:
return dict(code="PARAM_ERROR", message="name为空")
if password == "" or password == None:
return dict(code="PARAM_ERROR", message="password为空")
if not is_valid_username(name):
return dict(code="INVALID_NAME", message="非法的用户名")
name = name.lower()
found = find_by_name(name)
if found is not None:
return dict(code="fail", message="用户已存在")
else:
user = Storage(name=name,
password=password,
token=gen_new_token(),
ctime=xutils.format_time(),
salt=textutil.random_string(6),
mtime=xutils.format_time())
dbutil.put("user:%s" % name, user)
xutils.trace("UserAdd", name)
refresh_users()
return dict(code="success", message="create success")
def refresh_users():
global _users
_users = None
xutils.trace("ReLoadUsers", "reload users")
return _get_users()
def refresh_users():
xutils.trace("ReLoadUsers", "reload users")
return _get_users(force_reload=True)
def do_search(self, key, offset, limit):
global _rules
category = xutils.get_argument("category", "")
words = textutil.split_words(key)
files = []
user_name = xauth.get_current_name()
start_time = time.time()
ctx = SearchContext()
ctx.key = key
ctx.input_text = key
ctx.words = words
ctx.category = category
ctx.search_message = (category == "message")
ctx.search_note_content = (category == "content")
ctx.search_dict = (category == "dict")
ctx.user_name = user_name
if ctx.search_message:
ctx.search_note = False
ctx.search_note_content = False
ctx.search_tool = False
if ctx.search_dict:
ctx.search_note = False
ctx.search_tool = False
if ctx.search_note_content:
ctx.search_tool = False
if ctx.category == "book":
ctx.search_note = False
ctx.search_tool = False
xutils.trace("SearchKey", key)
# 阻断性的搜索，比如特定语法的
xmanager.fire("search.before", ctx)
if ctx.stop:
return ctx.dicts + ctx.tools + ctx.notes
# 普通的搜索行为
xmanager.fire("search", ctx)
for rule in _rules:
pattern = rule.pattern
func = rule.func
# re.match内部已经实现了缓存
m = re.match(pattern, key)
if m:
try:
start_time0 = time.time()
results = func(ctx, *m.groups())
cost_time0 = time.time() - start_time0
xutils.trace("SearchHandler", func.modfunc,
int(cost_time0 * 1000))
if results is not None:
files += results
except Exception as e:
xutils.print_exc()
cost_time = int((time.time() - start_time) * 1000)
xutils.trace("SearchTime", key, cost_time)
xconfig.search_history.add(key, cost_time)
log_search_history(user_name, key)
if ctx.stop:
return ctx.dicts + ctx.tools + files
# 慢搜索
xmanager.fire("search.slow", ctx)
xmanager.fire("search.after", ctx)
return ctx.dicts + ctx.tools + files
