def async_update_visits(bucket, visits_key, visitors_key, is_visitor=False):
visits_db_name = get_visits_db_name_for_bucket(bucket)
hincr(visits_db_name, visits_key, 1) # pv 始终是记录的
if is_visitor:
hincr(visits_db_name, visitors_key, 1) # uv 要做次判断，
diff = get_bucket_updated_at_diff_to_now(bucket)
if diff > 3 * 24 * 60 * 60: # 每隔 3 天的间隙，把访问数的统计路径进行一次 dump
visits_csv_raw_content = export_all_posts_visits_as_csv(bucket)
if not visits_csv_raw_content:
return
sync_file_by_server_side(bucket,
relative_path="_data/visits.csv",
content=visits_csv_raw_content)
def setting_settings_view():
bucket = get_logined_bucket(check=True)
if not bucket:
return abort(410)
data_obj = get_json_content_by_path(bucket, 'settings.json') or {}
if request.method == "POST":
keys = request.values.keys()
data_obj = get_data_obj_from_POST(keys)
sync_file_by_server_side(bucket,
"settings.json",
content=json_dumps(data_obj, indent=4))
return render_api_template_as_response("page_user_site_settings.jade",
data_obj=data_obj)
def do_download_from_internet_and_sync(bucket, path, url, timeout=10):
# 下载行为， 这里要获得site，因为如果是异步的话，g.site是无效的
if not has_bucket(bucket):
return
try:
response = requests.get(url, timeout=timeout, verify=False)
if response.status_code > 300:
return # ignore
response_content = smart_str(response.content)
if not response_content:
return
sync_file_by_server_side(bucket=bucket, relative_path=path, content=response_content) # 进行同步
return True
except:
pass
def settings_json_editor_callback(new_data_obj):
for key, value in new_data_obj.items():
if value and isinstance(value,
(str, unicode)) and len(value) > 20000:
return 'error: the length of value is too long'
if private_config_keys:
private_configs_to_update = {
key: new_data_obj.get(key)
for key in private_config_keys
}
update_bucket_private_configs(self.bucket,
**private_configs_to_update)
else:
content = json.dumps(new_data_obj, indent=4)
sync_file_by_server_side(bucket=self.bucket,
relative_path=config_store_path,
content=content)
def update_json(self, path, **kwargs):
# 更新一个 json 文件，用 simple_sync 的调用
# 注意，调用这个函数的时候，并不需要管理员权限，所以，要慎重处理!!!!
if not path.lower().endswith('.json'):
return 'must be a json file'
json_data = get_json_content_by_path(self.bucket, path)
if not isinstance(json_data, dict):
return 'update_json failed, not a dict obj'
to_update = False
for key, value in kwargs.items():
old_value = json_data.get(key)
if value != old_value:
to_update = True
break
if to_update:
json_data.update(kwargs)
new_content = json.dumps(json_data, indent=4)
sync_file_by_server_side(bucket=self.bucket, relative_path=path, content=new_content, is_dir=False, is_deleted=False)
return ''
def do_get_smart_scss_url(scss_filepath, **kwargs):
# 根据其内容内的变量名，进行替换处理;
# 总是要读取源文件的，不然不知道是否要重新编译; 由于页面本身的缓存逻辑，性能影响有限
# filename.scss -> filename-xxxxxxxxxxxx.css
ext = os.path.splitext(scss_filepath)[-1]
if ext not in [".less", ".scss"]:
return scss_filepath #ignore
prefix_name = get_prefix_name_from_source_filepath(scss_filepath)
filename = get_md5(get_md5(kwargs.keys()) +
get_md5(kwargs.values())) + '.css'
filepath = '/_cache/scss/%s-%s' % (prefix_name, filename)
bucket = get_bucket_in_request_context() or get_bucket_from_request()
if not bucket:
return scss_filepath # ignore
if has_record_by_path(bucket, path=filepath):
# 缓存的文件已经存在了
return filepath
raw_content = ""
if scss_filepath.startswith("/fb_static/"):
raw_content = get_static_raw_content(scss_filepath)
else:
bucket = get_bucket_in_request_context()
if bucket:
raw_content = get_raw_content_by_path(bucket=bucket,
path=scss_filepath)
if not raw_content:
return scss_filepath #ignore
css_content = replace_vars_in_scss(raw_content=raw_content,
new_vars=kwargs,
compile_scss=True)
sync_file_by_server_side(bucket=bucket,
relative_path=filepath,
content=css_content)
return filepath
def wechat_text_image_handler(wechat_user_id, bucket, xml_data):
msg_id = xml_data.get("MsgId")
create_time = xml_data.get("CreateTime")
msg_type = xml_data.get("MsgType")
# post_root, auto_add_date(bool), silent_reply(bool), image_folder, image_insert_type(image/markdown_syntax)
wechat_configs = get_json_content_by_path(bucket,
"__wechat.json",
force_dict=True)
today = get_now_from_bucket(bucket, "%Y-%m-%d")
silent_reply = wechat_configs.get("silent_reply") or False
post_root = (wechat_configs.get("post_folder") or "").strip("/")
auto_add_date = wechat_configs.get("auto_add_date", False)
add_nickname = wechat_configs.get("add_nickname", False)
draft_by_default = wechat_configs.get("draft_by_default", False)
one_user_one_post = wechat_configs.get("one_user_one_post", False)
user_post_per_day = wechat_configs.get("user_post_per_day", False)
if msg_type == "image": # 纯图片
pic_url = xml_data.get("PicUrl")
if not pic_url:
return "error: PicUrl is blank"
filename = "%s.jpg" % (msg_id or create_time)
image_insert_type = wechat_configs.get("image_insert_type")
raw_image_folder = smart_unicode(
wechat_configs.get("image_folder") or "").strip("/")
image_folder = smart_unicode(raw_image_folder) or today
if image_insert_type == "image": # 直接保存图片
path = "%s/%s" % (image_folder, filename)
download_from_internet_and_sync(bucket=bucket,
url=pic_url,
path=path,
timeout=60) # 下载图片并进行保存
if silent_reply:
return ""
else:
return u"将保存到 %s" % path
else:
if raw_image_folder:
# 还是按照 day 增加一层子目录，避免多了找不到的情况
path = "/%s/%s/%s" % (raw_image_folder, today, filename)
else:
path = "/_image/%s/%s" % (today, filename)
text_to_append = "" % path # 插入图片的语法
download_from_internet_and_sync(bucket=bucket,
url=pic_url,
path=path,
timeout=60)
else:
text_to_append = xml_data.get("Content") or xml_data.get(
"Recognition") or xml_data.get("EventKey") or ""
text_to_append = text_to_append.strip()
if "\n" not in text_to_append and re.match(u"name ", text_to_append):
name = text_to_append[5:].strip()
if name:
set_name_by_wechat_user_id(wechat_user_id, name)
return u"昵称已设定为 %s" % name
if one_user_one_post:
if user_post_per_day:
today_string = get_now_from_bucket(bucket, "%Y-%m-%d")
else:
today_string = ""
if post_root:
if today_string:
post_path = "%s/%s/%s.txt" % (post_root, wechat_user_id,
today_string)
else:
post_path = "%s/%s.txt" % (post_root, wechat_user_id)
else:
if today_string:
post_path = "%s/%s.txt" % (wechat_user_id, today_string)
else:
post_path = "%s.txt" % wechat_user_id
else:
if post_root:
post_path = post_root + "/" + get_now_from_bucket(
bucket, "%Y-%m-%d.txt")
else:
post_path = get_now_from_bucket(bucket, "%Y/%Y-%m-%d.txt")
if text_to_append.strip() == "reset":
if one_user_one_post:
sync_file_by_server_side(bucket=bucket,
relative_path=post_path,
content=" ")
return u"%s 已重置" % post_path
else:
return u"只有 `One User One Post` 的时候才能使用 reset 命令。"
if text_to_append:
if add_nickname:
nickname = get_name_by_wechat_user_id(wechat_user_id)
if nickname:
text_to_append = "%s: %s" % (nickname, text_to_append)
if auto_add_date and "\n" not in text_to_append:
if text_to_append.startswith("---") or re.match(
u"\w+[:\uff1a]", text_to_append):
auto_add_date = False
if auto_add_date: # 添加时间戳
bucket_now = get_now_from_bucket(bucket, "%Y-%m-%d %H:%M:%S")
text_to_append = "%s %s" % (bucket_now, text_to_append)
is_voice = True if xml_data.get("Recognition") else False
text_to_append = compile_tag_to_wiki_link_syntax(text_to_append,
is_voice=is_voice)
# 保存文章, 5 分钟的间隔自动多一个空line
append_to_markdown_doc_and_sync(bucket=bucket,
path=post_path,
content=text_to_append,
lines_more_diff=5 * 60,
draft_by_default=draft_by_default)
if silent_reply:
return ""
else:
return u"已保存至 %s" % post_path
def append_to_markdown_doc_and_sync(bucket,
path,
content,
lines_to_append=1,
reverse=False,
do_not_repeat=True,
lines_more_diff=None,
draft_by_default=False):
# 默认检测 append 的内容是否重复
if not bucket or not path or not content:
return
if not isinstance(bucket, string_types) or not isinstance(
path, string_types) or not isinstance(content, string_types):
return
if not has_bucket(bucket):
return
if not is_a_markdown_file(path):
return
content = smart_unicode(content)
old_doc = get_record_by_path(bucket, path=path) or {}
if not isinstance(old_doc, dict):
old_doc = {}
if lines_more_diff: # 多长时间hi后，自动多一空行
if old_doc and old_doc.get('timestamp'):
try:
diff = time.time() - old_doc.get('timestamp')
if diff > lines_more_diff:
lines_to_append += 1
except:
pass
interval = '\r\n' * abs(to_int(lines_to_append, max_value=10)) # 间隔换行
if old_doc:
if get_type_from_record(old_doc) == 'post': # 目前仅支持日志类型文件的append
old_content = old_doc.get('raw_content')
if old_content == " ":
old_content = ""
if do_not_repeat:
if reverse:
if old_content.strip().startswith(content.strip()):
return ""
else:
old_content_s = old_content.strip()
appended_content_s = content.strip()
if old_content_s.endswith(
'\n' + appended_content_s
) or old_content_s == appended_content_s:
return '' # ignore, 重复的内容不处理
if reverse: # 插入头部位置
new_content = '%s%s' % (content, interval)
content = '%s%s' % (new_content, old_content)
else:
new_content = '%s%s' % (interval, content)
content = '%s%s' % (old_content, new_content)
else:
return
else: # new doc
content = content.strip()
if draft_by_default:
# 新建文档默认是 draft 的状态
if re.match(u"\w+[:\uff1a]", content): # 可能用户自己声明了 metadata
content = "status: draft\n%s" % content
else:
now = get_now_from_bucket(bucket)
content = "date: %s\nstatus: draft\n\n%s" % (now, content)
sync_file_by_server_side(bucket=bucket,
relative_path=path,
content=content)
return True # done
def get_response_for_resized_image(bucket, record, storage):
from farbox_bucket.server.helpers.file_manager import sync_file_by_server_side
if not record or not isinstance(record, dict):
return
if record.get("_get_im_failed"):
return
#if not is_doc_modified(record, date_field='mtime'): #todo ????
# return get_304_response()
if request.args.get('origin') in ['true']:
return # ignore
relative_path = record.get('path')
if not relative_path:
return
file_id = get_file_id_from_record(record)
if not file_id:
return
ext = os.path.splitext(relative_path)[-1].strip('.').lower()
if ext not in ['png', 'jpg', 'jpeg', 'bmp', 'png']:
return
if not can_resize_by_system():
return
site_settings = get_bucket_site_configs(bucket)
image_max_width = to_int(site_settings.get('image_max_width'),
default_if_fail=None)
size_from_request = request.values.get("size")
if size_from_request == "m":
image_max_width = 600
if not isinstance(image_max_width, int) or image_max_width < 100:
image_max_width = None
image_max_height = to_int(site_settings.get('image_max_height'),
default_if_fail=None)
if not isinstance(image_max_height, int) or image_max_height < 100:
image_max_height = None
if not image_max_height and not image_max_width:
return
# 处理 image_max_type 允许的值
image_max_type = (get_site_config('image_max_type')
or 'webp-jpg').lower().strip()
if not isinstance(image_max_type, string_types):
image_max_type = 'webp-jpg'
if not re.match("[a-z-]+$", image_max_type):
image_max_type = "jpg"
if "webp" in image_max_type:
request_client_accept = request.headers.get("Accept") or ""
if "image/webp" not in request_client_accept:
# webp 浏览器不支持的情况下
image_max_type = "png" if ext == "png" else "jpg"
if image_max_type == 'webp-jpg':
mimetype = 'image/webp'
else:
if '/' not in image_max_type:
mimetype = 'image/%s' % image_max_type
else:
mimetype = image_max_type
if mimetype == 'image/jpg':
mimetype = 'image/jpeg'
if "webp" not in mimetype:
# 非 webp，并且文件中能读取到 size 的，判断是否需要进行缩略图的处理
image_width = record.get("image_width")
image_height = record.get("image_height")
if image_width and image_height:
if image_max_width and image_width < image_max_width:
return
if image_max_height and image_height < image_max_height:
return
if "webp" in image_max_type:
ext = "webp"
cache_image_filepath = "_cache/images/%s/%s-%s-%s.%s" % (
file_id, image_max_type, image_max_width, image_max_height, ext)
cache_image_record = get_record_by_path(bucket=bucket,
path=cache_image_filepath)
if cache_image_record:
# 已经存在了
return storage.as_web_response(bucket=bucket,
record=cache_image_record,
mimetype=mimetype,
try_resized_image=False)
else:
# todo 这里应该增加一个 time-block，避免潜在的攻击存在
# 构建一个缩略图
raw_content = storage.get_raw_content(bucket=bucket,
record_data=record)
if not raw_content: return
try:
im = get_im(raw_content)
except:
im = None
if not im:
# 标识，下次就不会尝试了
update_record(bucket,
record_id=record.get("_id"),
_get_im_failed=True)
return
degrees = to_int(record.get("degrees"), default_if_fail=0)
if image_max_type == 'webp-jpg':
#resized_jpg_content = resize_image(im, width=image_max_width, height=image_max_height, image_type='jpg')
#resized_jpg_im = get_im(resized_jpg_content)
resized_im_content = resize_image(im,
width=image_max_width,
height=image_max_height,
image_type='webp',
degrees=degrees)
if not resized_im_content:
update_record(bucket,
record_id=record.get("_id"),
_get_im_failed=True)
return
#del im, resized_im_content # resized_jpg_im,
else:
resized_im_content = resize_image(im,
width=image_max_width,
height=image_max_height,
image_type=image_max_type,
degrees=degrees)
cache_image_record = sync_file_by_server_side(
bucket=bucket,
relative_path=cache_image_filepath,
content=resized_im_content,
is_dir=False,
is_deleted=False,
return_record=True)
if cache_image_record:
return storage.as_web_response(bucket=bucket,
record=cache_image_record,
mimetype=mimetype,
try_resized_image=False)