def async_update_visits(bucket, visits_key, visitors_key, is_visitor=False): visits_db_name = get_visits_db_name_for_bucket(bucket) hincr(visits_db_name, visits_key, 1) # pv 始终是记录的 if is_visitor: hincr(visits_db_name, visitors_key, 1) # uv 要做次判断, diff = get_bucket_updated_at_diff_to_now(bucket) if diff > 3 * 24 * 60 * 60: # 每隔 3 天的间隙,把访问数的统计路径进行一次 dump visits_csv_raw_content = export_all_posts_visits_as_csv(bucket) if not visits_csv_raw_content: return sync_file_by_server_side(bucket, relative_path="_data/visits.csv", content=visits_csv_raw_content)
def setting_settings_view(): bucket = get_logined_bucket(check=True) if not bucket: return abort(410) data_obj = get_json_content_by_path(bucket, 'settings.json') or {} if request.method == "POST": keys = request.values.keys() data_obj = get_data_obj_from_POST(keys) sync_file_by_server_side(bucket, "settings.json", content=json_dumps(data_obj, indent=4)) return render_api_template_as_response("page_user_site_settings.jade", data_obj=data_obj)
def do_download_from_internet_and_sync(bucket, path, url, timeout=10): # 下载行为, 这里要获得site,因为如果是异步的话,g.site是无效的 if not has_bucket(bucket): return try: response = requests.get(url, timeout=timeout, verify=False) if response.status_code > 300: return # ignore response_content = smart_str(response.content) if not response_content: return sync_file_by_server_side(bucket=bucket, relative_path=path, content=response_content) # 进行同步 return True except: pass
def settings_json_editor_callback(new_data_obj): for key, value in new_data_obj.items(): if value and isinstance(value, (str, unicode)) and len(value) > 20000: return 'error: the length of value is too long' if private_config_keys: private_configs_to_update = { key: new_data_obj.get(key) for key in private_config_keys } update_bucket_private_configs(self.bucket, **private_configs_to_update) else: content = json.dumps(new_data_obj, indent=4) sync_file_by_server_side(bucket=self.bucket, relative_path=config_store_path, content=content)
def update_json(self, path, **kwargs): # 更新一个 json 文件,用 simple_sync 的调用 # 注意,调用这个函数的时候,并不需要管理员权限,所以,要慎重处理!!!! if not path.lower().endswith('.json'): return 'must be a json file' json_data = get_json_content_by_path(self.bucket, path) if not isinstance(json_data, dict): return 'update_json failed, not a dict obj' to_update = False for key, value in kwargs.items(): old_value = json_data.get(key) if value != old_value: to_update = True break if to_update: json_data.update(kwargs) new_content = json.dumps(json_data, indent=4) sync_file_by_server_side(bucket=self.bucket, relative_path=path, content=new_content, is_dir=False, is_deleted=False) return ''
def do_get_smart_scss_url(scss_filepath, **kwargs): # 根据其内容内的变量名,进行替换处理; # 总是要读取源文件的,不然不知道是否要重新编译; 由于页面本身的缓存逻辑,性能影响有限 # filename.scss -> filename-xxxxxxxxxxxx.css ext = os.path.splitext(scss_filepath)[-1] if ext not in [".less", ".scss"]: return scss_filepath #ignore prefix_name = get_prefix_name_from_source_filepath(scss_filepath) filename = get_md5(get_md5(kwargs.keys()) + get_md5(kwargs.values())) + '.css' filepath = '/_cache/scss/%s-%s' % (prefix_name, filename) bucket = get_bucket_in_request_context() or get_bucket_from_request() if not bucket: return scss_filepath # ignore if has_record_by_path(bucket, path=filepath): # 缓存的文件已经存在了 return filepath raw_content = "" if scss_filepath.startswith("/fb_static/"): raw_content = get_static_raw_content(scss_filepath) else: bucket = get_bucket_in_request_context() if bucket: raw_content = get_raw_content_by_path(bucket=bucket, path=scss_filepath) if not raw_content: return scss_filepath #ignore css_content = replace_vars_in_scss(raw_content=raw_content, new_vars=kwargs, compile_scss=True) sync_file_by_server_side(bucket=bucket, relative_path=filepath, content=css_content) return filepath
def wechat_text_image_handler(wechat_user_id, bucket, xml_data): msg_id = xml_data.get("MsgId") create_time = xml_data.get("CreateTime") msg_type = xml_data.get("MsgType") # post_root, auto_add_date(bool), silent_reply(bool), image_folder, image_insert_type(image/markdown_syntax) wechat_configs = get_json_content_by_path(bucket, "__wechat.json", force_dict=True) today = get_now_from_bucket(bucket, "%Y-%m-%d") silent_reply = wechat_configs.get("silent_reply") or False post_root = (wechat_configs.get("post_folder") or "").strip("/") auto_add_date = wechat_configs.get("auto_add_date", False) add_nickname = wechat_configs.get("add_nickname", False) draft_by_default = wechat_configs.get("draft_by_default", False) one_user_one_post = wechat_configs.get("one_user_one_post", False) user_post_per_day = wechat_configs.get("user_post_per_day", False) if msg_type == "image": # 纯图片 pic_url = xml_data.get("PicUrl") if not pic_url: return "error: PicUrl is blank" filename = "%s.jpg" % (msg_id or create_time) image_insert_type = wechat_configs.get("image_insert_type") raw_image_folder = smart_unicode( wechat_configs.get("image_folder") or "").strip("/") image_folder = smart_unicode(raw_image_folder) or today if image_insert_type == "image": # 直接保存图片 path = "%s/%s" % (image_folder, filename) download_from_internet_and_sync(bucket=bucket, url=pic_url, path=path, timeout=60) # 下载图片并进行保存 if silent_reply: return "" else: return u"将保存到 %s" % path else: if raw_image_folder: # 还是按照 day 增加一层子目录,避免多了找不到的情况 path = "/%s/%s/%s" % (raw_image_folder, today, filename) else: path = "/_image/%s/%s" % (today, filename) text_to_append = "![](%s)" % path # 插入图片的语法 download_from_internet_and_sync(bucket=bucket, url=pic_url, path=path, timeout=60) else: text_to_append = xml_data.get("Content") or xml_data.get( "Recognition") or xml_data.get("EventKey") or "" text_to_append = text_to_append.strip() if "\n" not in text_to_append and re.match(u"name ", text_to_append): name = text_to_append[5:].strip() if name: set_name_by_wechat_user_id(wechat_user_id, name) return u"昵称已设定为 %s" % name if one_user_one_post: if user_post_per_day: today_string = get_now_from_bucket(bucket, "%Y-%m-%d") else: today_string = "" if post_root: if today_string: post_path = "%s/%s/%s.txt" % (post_root, wechat_user_id, today_string) else: post_path = "%s/%s.txt" % (post_root, wechat_user_id) else: if today_string: post_path = "%s/%s.txt" % (wechat_user_id, today_string) else: post_path = "%s.txt" % wechat_user_id else: if post_root: post_path = post_root + "/" + get_now_from_bucket( bucket, "%Y-%m-%d.txt") else: post_path = get_now_from_bucket(bucket, "%Y/%Y-%m-%d.txt") if text_to_append.strip() == "reset": if one_user_one_post: sync_file_by_server_side(bucket=bucket, relative_path=post_path, content=" ") return u"%s 已重置" % post_path else: return u"只有 `One User One Post` 的时候才能使用 reset 命令。" if text_to_append: if add_nickname: nickname = get_name_by_wechat_user_id(wechat_user_id) if nickname: text_to_append = "%s: %s" % (nickname, text_to_append) if auto_add_date and "\n" not in text_to_append: if text_to_append.startswith("---") or re.match( u"\w+[:\uff1a]", text_to_append): auto_add_date = False if auto_add_date: # 添加时间戳 bucket_now = get_now_from_bucket(bucket, "%Y-%m-%d %H:%M:%S") text_to_append = "%s %s" % (bucket_now, text_to_append) is_voice = True if xml_data.get("Recognition") else False text_to_append = compile_tag_to_wiki_link_syntax(text_to_append, is_voice=is_voice) # 保存文章, 5 分钟的间隔自动多一个空line append_to_markdown_doc_and_sync(bucket=bucket, path=post_path, content=text_to_append, lines_more_diff=5 * 60, draft_by_default=draft_by_default) if silent_reply: return "" else: return u"已保存至 %s" % post_path
def append_to_markdown_doc_and_sync(bucket, path, content, lines_to_append=1, reverse=False, do_not_repeat=True, lines_more_diff=None, draft_by_default=False): # 默认检测 append 的内容是否重复 if not bucket or not path or not content: return if not isinstance(bucket, string_types) or not isinstance( path, string_types) or not isinstance(content, string_types): return if not has_bucket(bucket): return if not is_a_markdown_file(path): return content = smart_unicode(content) old_doc = get_record_by_path(bucket, path=path) or {} if not isinstance(old_doc, dict): old_doc = {} if lines_more_diff: # 多长时间hi后,自动多一空行 if old_doc and old_doc.get('timestamp'): try: diff = time.time() - old_doc.get('timestamp') if diff > lines_more_diff: lines_to_append += 1 except: pass interval = '\r\n' * abs(to_int(lines_to_append, max_value=10)) # 间隔换行 if old_doc: if get_type_from_record(old_doc) == 'post': # 目前仅支持日志类型文件的append old_content = old_doc.get('raw_content') if old_content == " ": old_content = "" if do_not_repeat: if reverse: if old_content.strip().startswith(content.strip()): return "" else: old_content_s = old_content.strip() appended_content_s = content.strip() if old_content_s.endswith( '\n' + appended_content_s ) or old_content_s == appended_content_s: return '' # ignore, 重复的内容不处理 if reverse: # 插入头部位置 new_content = '%s%s' % (content, interval) content = '%s%s' % (new_content, old_content) else: new_content = '%s%s' % (interval, content) content = '%s%s' % (old_content, new_content) else: return else: # new doc content = content.strip() if draft_by_default: # 新建文档默认是 draft 的状态 if re.match(u"\w+[:\uff1a]", content): # 可能用户自己声明了 metadata content = "status: draft\n%s" % content else: now = get_now_from_bucket(bucket) content = "date: %s\nstatus: draft\n\n%s" % (now, content) sync_file_by_server_side(bucket=bucket, relative_path=path, content=content) return True # done
def get_response_for_resized_image(bucket, record, storage): from farbox_bucket.server.helpers.file_manager import sync_file_by_server_side if not record or not isinstance(record, dict): return if record.get("_get_im_failed"): return #if not is_doc_modified(record, date_field='mtime'): #todo ???? # return get_304_response() if request.args.get('origin') in ['true']: return # ignore relative_path = record.get('path') if not relative_path: return file_id = get_file_id_from_record(record) if not file_id: return ext = os.path.splitext(relative_path)[-1].strip('.').lower() if ext not in ['png', 'jpg', 'jpeg', 'bmp', 'png']: return if not can_resize_by_system(): return site_settings = get_bucket_site_configs(bucket) image_max_width = to_int(site_settings.get('image_max_width'), default_if_fail=None) size_from_request = request.values.get("size") if size_from_request == "m": image_max_width = 600 if not isinstance(image_max_width, int) or image_max_width < 100: image_max_width = None image_max_height = to_int(site_settings.get('image_max_height'), default_if_fail=None) if not isinstance(image_max_height, int) or image_max_height < 100: image_max_height = None if not image_max_height and not image_max_width: return # 处理 image_max_type 允许的值 image_max_type = (get_site_config('image_max_type') or 'webp-jpg').lower().strip() if not isinstance(image_max_type, string_types): image_max_type = 'webp-jpg' if not re.match("[a-z-]+$", image_max_type): image_max_type = "jpg" if "webp" in image_max_type: request_client_accept = request.headers.get("Accept") or "" if "image/webp" not in request_client_accept: # webp 浏览器不支持的情况下 image_max_type = "png" if ext == "png" else "jpg" if image_max_type == 'webp-jpg': mimetype = 'image/webp' else: if '/' not in image_max_type: mimetype = 'image/%s' % image_max_type else: mimetype = image_max_type if mimetype == 'image/jpg': mimetype = 'image/jpeg' if "webp" not in mimetype: # 非 webp,并且文件中能读取到 size 的,判断是否需要进行缩略图的处理 image_width = record.get("image_width") image_height = record.get("image_height") if image_width and image_height: if image_max_width and image_width < image_max_width: return if image_max_height and image_height < image_max_height: return if "webp" in image_max_type: ext = "webp" cache_image_filepath = "_cache/images/%s/%s-%s-%s.%s" % ( file_id, image_max_type, image_max_width, image_max_height, ext) cache_image_record = get_record_by_path(bucket=bucket, path=cache_image_filepath) if cache_image_record: # 已经存在了 return storage.as_web_response(bucket=bucket, record=cache_image_record, mimetype=mimetype, try_resized_image=False) else: # todo 这里应该增加一个 time-block,避免潜在的攻击存在 # 构建一个缩略图 raw_content = storage.get_raw_content(bucket=bucket, record_data=record) if not raw_content: return try: im = get_im(raw_content) except: im = None if not im: # 标识,下次就不会尝试了 update_record(bucket, record_id=record.get("_id"), _get_im_failed=True) return degrees = to_int(record.get("degrees"), default_if_fail=0) if image_max_type == 'webp-jpg': #resized_jpg_content = resize_image(im, width=image_max_width, height=image_max_height, image_type='jpg') #resized_jpg_im = get_im(resized_jpg_content) resized_im_content = resize_image(im, width=image_max_width, height=image_max_height, image_type='webp', degrees=degrees) if not resized_im_content: update_record(bucket, record_id=record.get("_id"), _get_im_failed=True) return #del im, resized_im_content # resized_jpg_im, else: resized_im_content = resize_image(im, width=image_max_width, height=image_max_height, image_type=image_max_type, degrees=degrees) cache_image_record = sync_file_by_server_side( bucket=bucket, relative_path=cache_image_filepath, content=resized_im_content, is_dir=False, is_deleted=False, return_record=True) if cache_image_record: return storage.as_web_response(bucket=bucket, record=cache_image_record, mimetype=mimetype, try_resized_image=False)