def list_recent_edit(parent_id=None, offset=0, limit=None): if limit is None: limit = xconfig.PAGE_SIZE db = xtables.get_file_table() t = Timer() t.start() creator = xauth.get_current_name() if creator: where = "is_deleted = 0 AND (creator = $creator) AND type != 'group'" else: # 未登录 where = "is_deleted = 0 AND is_public = 1 AND type != 'group'" cache_key = "[%s]note.recent$%s$%s" % (creator, offset, limit) files = cacheutil.get(cache_key) if files is None: files = list( db.select( what= "name, id, parent_id, ctime, mtime, type, creator, priority", where=where, vars=dict(creator=creator), order="priority DESC, mtime DESC", offset=offset, limit=limit)) for item in files: item.parent_name = query_note_name(item.parent_id) cacheutil.set(cache_key, files, expire=600) t.stop() xutils.trace("NoteDao.ListRecentEdit", "", t.cost_millis()) return files
def add_user(name, password): if name == "" or name == None: return if password == "" or password == None: return if not is_valid_username(name): return dict(code="INVALID_NAME", message="非法的用户名") name = name.lower() found = find_by_name(name) if found is not None: found.mtime = xutils.format_time() found.salt = textutil.random_string(6) found.token = gen_new_token() found.password = password update_user(name, found) else: user = Storage(name=name, password=password, token=gen_new_token(), ctime=xutils.format_time(), salt=textutil.random_string(6), mtime=xutils.format_time()) dbutil.put("user:%s" % name, user) xutils.trace("UserAdd", name) refresh_users()
def rdb_search_message(user_name, key, offset, limit): db = xtables.get_message_table() vars = dict(user=user_name) kw = "user = $user" start_time = time.time() for item in key.split(" "): if item == "": continue kw += " AND content LIKE " + fuzzy_item(item) # when find numbers, the sql printed is not correct # eg. LIKE '%1%' will be LIKE '%' chatlist = list( db.select(where=kw, vars=vars, order="ctime DESC", limit=limit, offset=offset)) end_time = time.time() cost_time = int((end_time - start_time) * 1000) xutils.trace("MessageSearch", key, cost_time) if xconfig.search_history is not None: xconfig.search_history.add(key, cost_time) amount = db.count(where=kw, vars=vars) return chatlist, amount
def _get_users(): """获取用户,内部接口""" global _users # 有并发风险 if _users is not None: return _users _users = {} # 默认的账号 _users["admin"] = Storage(name = "admin", password = "******", salt = "", mtime = "", token = gen_new_token()) user_list = dbutil.prefix_list("user") for user in user_list: if user.name is None: xutils.trace("UserList", "invalid user %s" % user) continue if isinstance(user.config, dict): user.config = Storage(**user.config) else: user.config = Storage() name = user.name.lower() _users[name] = user return _users
def read_file(self, path): # 强制缓存 if not xconfig.DEBUG: web.header("Cache-Control", "max-age=3600") environ = web.ctx.environ etag = '"%s"' % os.path.getmtime(path) client_etag = environ.get('HTTP_IF_NONE_MATCH') web.header("Etag", etag) self.handle_content_type(path) # self.handle_content_encoding(ext) if etag == client_etag: web.ctx.status = "304 Not Modified" return b'' # 其实webpy已经通过yield空bytes来避免None else: http_range = environ.get("HTTP_RANGE") blocksize = 64 * 1024 # print_env() if http_range is not None: xutils.trace("==> HTTP_RANGE {}", http_range) return self.read_range(path, http_range, blocksize) else: return self.read_all(path, blocksize)
def POST(self): user_name = xauth.current_name() part_file = True chunksize = 5 * 1024 * 1024 chunk = xutils.get_argument("chunk", 0, type=int) chunks = xutils.get_argument("chunks", 1, type=int) file = xutils.get_argument("file", {}) prefix = xutils.get_argument("prefix", "") dirname = xutils.get_argument("dirname", xconfig.DATA_DIR) dirname = dirname.replace("$DATA", xconfig.DATA_DIR) note_id = xutils.get_argument("note_id") # 不能访问上级目录 if ".." in dirname: return dict(code="fail", message="can not access parent directory") filename = None webpath = "" origin_name = "" if hasattr(file, "filename"): origin_name = file.filename xutils.trace("UploadFile", file.filename) filename = os.path.basename(file.filename) filename = xutils.get_real_path(filename) if dirname == "auto": filename = generate_filename(filename, prefix) filepath, webpath = xutils.get_upload_file_path(user_name, filename, replace_exists=True) dirname = os.path.dirname(filepath) filename = os.path.basename(filepath) else: # TODO check permission. pass if part_file: tmp_name = "%s_%d.part" % (filename, chunk) seek = 0 else: tmp_name = filename seek = chunk * chunksize xutils.makedirs(dirname) tmp_path = os.path.join(dirname, tmp_name) with open(tmp_path, "wb") as fp: fp.seek(seek) if seek != 0: xutils.log("seek to {}", seek) for file_chunk in file.file: fp.write(file_chunk) else: return dict(code="fail", message="require file") if part_file and chunk+1==chunks: self.merge_files(dirname, filename, chunks) try_touch_note(note_id) if note_id != None and note_id != "": xutils.call("note.touch", note_id) return dict(code="success", webpath=webpath, link=get_link(origin_name, webpath))
def update_user(name, user): if name == "" or name == None: return name = name.lower() mem_user = _users[name] mem_user.update(user) dbutil.put("user:%s" % name, mem_user) xutils.trace("UserUpdate", mem_user)
def check_and_run(task, tm): if self.match(task, tm): put_task(request_url, task) try: xutils.trace("RunTask", task.url) if task.tm_wday == "no-repeat": # 一次性任务直接删除 dbutil.delete(task.id) self.load_tasks() except Exception as e: xutils.log("run task [%s] failed, %s" % (task.url, e))
def check_and_run(task, tm): if self.match(task, tm): put_task(request_url, task) try: xutils.trace("RunTask", task.url) if task.tm_wday == "no-repeat": # 一次性任务直接删除 xtables.get_schedule_table().delete(where=dict(id=task.id)) self.load_tasks() except Exception as e: xutils.log("run task [%s] failed, %s" % (task.url, e))
def count_ungrouped(creator): t = Timer() t.start() count_key = "*****@*****.**" % creator count = cacheutil.get(count_key) if count is None: count = xtables.get_file_table().count(where="creator=$creator AND parent_id=0 AND is_deleted=0 AND type!='group'", vars=dict(creator=creator)) xutils.cache_put(count_key, count, expire=600) t.stop() xutils.trace("NoteDao.CountUngrouped", "", t.cost_millis()) return count
def execute(self, ctx=None): try: matched = self.pattern.match(ctx.key) if not matched: return start = time.time() ctx.groups = matched.groups() self.func(ctx) stop = time.time() xutils.trace("SearchHandler", self.key, int((stop - start) * 1000)) except: xutils.print_exc()
def on_reload(ctx=None): for key in ('THEME', 'FS_HIDE_FILES', 'OPTION_STYLE', 'PAGE_OPEN', 'RECENT_SEARCH_LIMIT', "PAGE_SIZE", "RECENT_SIZE"): value = cacheutil.hget('sys.config', key) xutils.trace("HGET", "key=%s, value=%s" % (key, value)) if value is not None: setattr(xconfig, key, value) path = os.path.join(xconfig.SCRIPTS_DIR, "user.css") if not os.path.exists(path): return xconfig.set("USER_CSS", xutils.readfile(path))
def count_user_note(creator): t = Timer() t.start() count_key = "[%s]note.count" % creator count = cacheutil.get(count_key) if count is None: db = xtables.get_file_table() where = "is_deleted = 0 AND creator = $creator AND type != 'group'" count = db.count(where, vars=dict(creator=xauth.get_current_name())) cacheutil.set(count_key, count, expire=600) t.stop() xutils.trace("NoteDao.CountRecentEdit", "", t.cost_millis()) return count
def execute(self, ctx=None): if self.is_async: put_task(self.func, ctx) else: try: if self.profile: start = time.time() self.func(ctx) if self.profile: stop = time.time() xutils.trace("EventHandler", self.key, int((stop-start)*1000)) except: xutils.print_exc()
def on_reload(ctx=None): keys = ("THEME", 'FS_HIDE_FILES', 'OPTION_STYLE', 'PAGE_OPEN', 'RECENT_SEARCH_LIMIT', "PAGE_SIZE", "RECENT_SIZE", "RECORD_LOCATION", "TRASH_EXPIRE", "PAGE_WIDTH", "FS_VIEW_MODE") for key in keys: value = cacheutil.hget('sys.config', key) xutils.trace("HGET", "key=%s, value=%s" % (key, value)) if value is not None: setattr(xconfig, key, value) path = os.path.join(xconfig.SCRIPTS_DIR, "user.css") if not os.path.exists(path): return xconfig.USER_CSS = xutils.readfile(path)
def add_handler(self, handler): """ 注册事件处理器 事件处理器的去重,通过判断是不是同一个函数,不通过函数名,如果修改初始化脚本需要执行【重新加载模块】功能 """ event_type = handler.event_type handlers = self._handlers.get(event_type, []) if handler in handlers: warn("handler %s is already registered" % handler) return # XXX 使用str(handler)在Python2.7环境下报错 xutils.trace("EventRegister", "%s" % handler) handlers.append(handler) self._handlers[event_type] = handlers
def list_group(current_name=None): if current_name is None: current_name = str(xauth.get_current_name()) t1 = time.time() cache_key = "[%s]note.group.list" % current_name value = cacheutil.get(cache_key) if value is None: sql = "SELECT * FROM file WHERE type = 'group' AND is_deleted = 0 AND creator = $creator ORDER BY name LIMIT 1000" value = list(xtables.get_file_table().query( sql, vars=dict(creator=current_name))) cacheutil.set(cache_key, value, expire=600) t2 = time.time() xutils.trace("NoteDao.ListGroup", "", int((t2 - t1) * 1000)) return value
def update_user(name, user): if name == "" or name == None: return name = name.lower() mem_user = find_by_name(name) if mem_user is None: raise Exception("user not found") mem_user.update(user) dbutil.put("user:%s" % name, mem_user) xutils.trace("UserUpdate", mem_user) refresh_users()
def list_tag(user_name): t = Timer() t.start() cache_key = "%s@tag_list" % user_name tag_list = xutils.cache_get(cache_key) tag_list = None if tag_list is None: db = xtables.get_file_tag_table() sql = """SELECT LOWER(name) AS name, COUNT(*) AS amount FROM file_tag WHERE (user=$user OR is_public=1) GROUP BY LOWER(name) ORDER BY amount DESC, name ASC""" tag_list = list(db.query(sql, vars=dict(user=user_name))) xutils.cache_put(cache_key, tag_list, 60 * 10) t.stop() xutils.trace("NoteDao.ListTag", "", t.cost_millis()) return tag_list
def list_recent_viewed(creator=None, offset=0, limit=10): t = Timer() t.start() where = "is_deleted = 0 AND (creator = $creator)" db = xtables.get_file_table() result = list( db.select(where=where, vars=dict(creator=creator), order="atime DESC", offset=offset, limit=limit)) for item in result: item.parent_name = query_note_name(item.parent_id) t.stop() xutils.trace("NoteDao.ListRecentViewed", "", t.cost_millis()) return result
def GET(self): pagesize = xutils.get_argument("pagesize", xconfig.PAGE_SIZE, type=int) page = xutils.get_argument("page", 1, type=int) status = xutils.get_argument("status") key = xutils.get_argument("key") offset = (page-1) * pagesize db = xtables.get_message_table() user_name = xauth.get_current_name() # 未完成任务的分页 undone_pagesize = 1000 kw = "1=1" if status == "created": kw = "status = 0" pagesize = undone_pagesize if status == "done": kw = "status = 100" if status == "suspended": kw = "status = 50" pagesize = undone_pagesize kw += " AND user = $user" vars = dict(user=xauth.get_current_name()) if key != "" and key != None: start_time = time.time() for item in key.split(" "): if item == "": continue kw += " AND content LIKE " + fuzzy_item(item) # when find numbers, the sql printed is not correct # eg. LIKE '%1%' will be LIKE '%' # print(kw) chatlist = list(db.select(where=kw, vars=vars, order="ctime DESC", limit=pagesize, offset=offset)) end_time = time.time() cost_time = int((end_time-start_time)*1000) xutils.trace("MessageSearch", key, cost_time) if xconfig.search_history is not None: xconfig.search_history.add(key, cost_time) else: chatlist = list(db.select(where=kw, vars=vars, order="ctime DESC", limit=pagesize, offset=offset)) chatlist.reverse() amount = db.count(where=kw, vars=vars) page_max = math.ceil(amount / pagesize) chatlist = list(map(process_message, chatlist)) return dict(code="success", message="", pagesize = pagesize, data=chatlist, amount=amount, page_max=page_max, current_user=xauth.get_current_name())
def read_range(self, path, http_range, blocksize): range_list = http_range.split("bytes=") if len(range_list) == 2: # 包含完整的范围 range_list = range_list[1] try: range_start, range_end = range_list.split('-') range_start = int(range_start) total_size = os.stat(path).st_size if range_end != "": range_end = int(range_end) else: range_end = total_size - 1 web.header("Content-Length", total_size) content_range = "bytes %s-%s/%s" % (range_start, range_end, total_size) # 设置HTTP响应状态 web.ctx.status = "206 Partial Content" # 发送HTTP首部 web.header("Accept-Ranges", "bytes") web.header("Content-Range", content_range) xutils.trace("Download", "<== Content-Range:%s" % content_range) # 发送数据 fp = open(path, "rb") try: fp.seek(range_start) rest = range_end - range_start + 1 readsize = min(rest, blocksize) while readsize > 0: # print("%s send %s K" % (time.ctime(), readsize)) yield fp.read(readsize) rest -= readsize readsize = min(rest, blocksize) finally: # 基本上和with等价,这里打印出来 xutils.trace("Download", "close %s" % path) fp.close() except Exception as e: # 其他未知异常 xutils.print_stacktrace() # yield最好不要和return混用 yield self.read_all(path, blocksize) else: # 处理不了,返回所有的数据 yield self.read_all(path, blocksize)
def apply_search_rules(ctx, key): files = [] for rule in _RULES: pattern = rule.pattern func = rule.func # re.match内部已经实现了缓存 m = re.match(pattern, key) if m: try: start_time0 = time.time() results = func(ctx, *m.groups()) cost_time0 = time.time() - start_time0 xutils.trace("SearchHandler", func.modfunc, int(cost_time0*1000)) if results is not None: files += results except Exception as e: xutils.print_exc() return files
def list_recent_created(parent_id=None, offset=0, limit=10): t = Timer() t.start() where = "is_deleted = 0 AND (creator = $creator)" if parent_id != None: where += " AND parent_id = %s" % parent_id db = xtables.get_file_table() result = list( db.select(where=where, vars=dict(creator=xauth.get_current_name()), order="ctime DESC", offset=offset, limit=limit)) for item in result: item.parent_name = query_note_name(item.parent_id) t.stop() xutils.trace("NoteDao.ListRecentCreated", "", t.cost_millis()) return result
def add_user(name, password): if name == "" or name == None: return dict(code="PARAM_ERROR", message="name为空") if password == "" or password == None: return dict(code="PARAM_ERROR", message="password为空") if not is_valid_username(name): return dict(code="INVALID_NAME", message="非法的用户名") name = name.lower() found = find_by_name(name) if found is not None: return dict(code="fail", message="用户已存在") else: user = Storage(name=name, password=password, token=gen_new_token(), ctime=xutils.format_time(), salt=textutil.random_string(6), mtime=xutils.format_time()) dbutil.put("user:%s" % name, user) xutils.trace("UserAdd", name) refresh_users() return dict(code="success", message="create success")
def refresh_users(): global _users _users = None xutils.trace("ReLoadUsers", "reload users") return _get_users()
def refresh_users(): xutils.trace("ReLoadUsers", "reload users") return _get_users(force_reload=True)
def do_search(self, key, offset, limit): global _rules category = xutils.get_argument("category", "") words = textutil.split_words(key) files = [] user_name = xauth.get_current_name() start_time = time.time() ctx = SearchContext() ctx.key = key ctx.input_text = key ctx.words = words ctx.category = category ctx.search_message = (category == "message") ctx.search_note_content = (category == "content") ctx.search_dict = (category == "dict") ctx.user_name = user_name if ctx.search_message: ctx.search_note = False ctx.search_note_content = False ctx.search_tool = False if ctx.search_dict: ctx.search_note = False ctx.search_tool = False if ctx.search_note_content: ctx.search_tool = False if ctx.category == "book": ctx.search_note = False ctx.search_tool = False xutils.trace("SearchKey", key) # 阻断性的搜索,比如特定语法的 xmanager.fire("search.before", ctx) if ctx.stop: return ctx.dicts + ctx.tools + ctx.notes # 普通的搜索行为 xmanager.fire("search", ctx) for rule in _rules: pattern = rule.pattern func = rule.func # re.match内部已经实现了缓存 m = re.match(pattern, key) if m: try: start_time0 = time.time() results = func(ctx, *m.groups()) cost_time0 = time.time() - start_time0 xutils.trace("SearchHandler", func.modfunc, int(cost_time0 * 1000)) if results is not None: files += results except Exception as e: xutils.print_exc() cost_time = int((time.time() - start_time) * 1000) xutils.trace("SearchTime", key, cost_time) xconfig.search_history.add(key, cost_time) log_search_history(user_name, key) if ctx.stop: return ctx.dicts + ctx.tools + files # 慢搜索 xmanager.fire("search.slow", ctx) xmanager.fire("search.after", ctx) return ctx.dicts + ctx.tools + files