def load_theme_from_template_folder_for_bucket(bucket, prefix="template"):
if not has_bucket(bucket):
return
info = {}
prefix = prefix.strip().strip("/")
template_paths = get_paths_under(bucket, under=prefix)
for _relative_path in template_paths:
relative_path = _relative_path.replace("%s/"%prefix, "", 1)
raw_content = get_raw_content_by_path(bucket, _relative_path)
if not raw_content:
continue
path_without_ext, ext = os.path.splitext(relative_path)
ext = ext.lower().strip('.')
if ext not in allowed_exts:
continue
raw_content = smart_unicode(raw_content) # to unicode
info[relative_path] = raw_content
matched_compiler = server_side_template_resource_compilers.get(ext)
if matched_compiler:
new_ext, compile_func = matched_compiler
try:
compiled_content = compile_func(raw_content)
new_key = path_without_ext + '.' + new_ext.strip('.')
info[new_key] = compiled_content
except Exception as e:
pass
info["_route"] = get_templates_route_info(info)
set_bucket_configs(bucket, info, config_type='pages')
return info
def set_bucket_private_configs(bucket, configs):
if not isinstance(configs, dict):
return
if not has_bucket(bucket):
return
configs_data = json_dumps(configs)
if len(configs_data) > 500 * 1024: # 不能大于 500k
return
else:
hset("_bucket_private_configs", bucket, configs_data)
return True
def set_bucket_pages_configs_by_web_api(bucket, remote_url, timeout=3):
# True or False
if not has_bucket(bucket):
return # ignore
gevent_job = gevent.spawn(do_set_bucket_pages_configs_by_web_api, bucket,
remote_url, timeout)
try:
result = gevent_job.get(block=True, timeout=timeout)
return result
except:
gevent_job.kill(block=False)
return False
def set_bucket_api_token(bucket, db_name="_bucket_api_token"):
"""
:param bucket: 需要
:param db_name: 提供了默认值
:return: new_api_token
"""
if not is_valid_bucket_name(bucket):
return ""
if not has_bucket(bucket):
return ""
new_token = get_a_random_api_token()
hset(db_name, bucket, new_token)
return new_token
def mark_bucket_login_by_private_key(private_key):
if not private_key:
return False
bucket = get_bucket_by_private_key(private_key)
if not bucket:
return False
if not has_bucket(bucket):
return False
login_key = get_bucket_login_key(bucket)
utoken_for_cookie = '%s-%s' % (bucket, login_key)
set_cookie('utoken', utoken_for_cookie,
max_age=120 * 24 * 60 * 60) # 120 days
set_logined_bucket_in_request(bucket, checked=True)
return True
def do_download_from_internet_and_sync(bucket, path, url, timeout=10):
# 下载行为， 这里要获得site，因为如果是异步的话，g.site是无效的
if not has_bucket(bucket):
return
try:
response = requests.get(url, timeout=timeout, verify=False)
if response.status_code > 300:
return # ignore
response_content = smart_str(response.content)
if not response_content:
return
sync_file_by_server_side(bucket=bucket, relative_path=path, content=response_content) # 进行同步
return True
except:
pass
def check_wechat_bind_code(bind_code):
# 校验成功后，比较偷懒，返回的不是 True，而是 bucket
if not isinstance(bind_code, string_types):
return False
if "-" not in bind_code:
return False
bucket, signature = bind_code.strip().split("-", 1)
if not check_signature_for_bucket(
bucket, signature=signature, hours=1, salt="wechat"): # 1 小时有效期
return False
else:
if not has_bucket(bucket):
return False
else:
return bucket
def update_record_directly(bucket, record):
# 通过 record 来更新 record，不需要额外的编译
# 如果有 path 路径，不要去更改它
if not isinstance(record, dict):
return False
if not has_bucket(bucket):
return False
record['_auto_clean_bucket'] = True
# 创建新的 record
create_record(bucket, record)
object_id = record.get("_id")
if object_id:
update_path_related_when_record_changed(bucket, object_id, record)
return True
def do_set_bucket_pages_configs_by_web_api(bucket, remote_url, timeout=3):
if not has_bucket(bucket):
return
if not isinstance(remote_url, string_types):
return
if "://" not in remote_url:
remote_url = "http://" + remote_url
try:
response = requests.get(remote_url, timeout=timeout)
raw_pages_configs = response.json()
if not isinstance(raw_pages_configs, dict):
return
if not raw_pages_configs.get("_route"):
return
raw_pages_configs["can_copy"] = False
set_bucket_configs(bucket, raw_pages_configs, config_type="pages")
return True
except:
pass
def change_bucket_date():
if not get_logined_admin_bucket():
abort(404, "not admin")
info = ""
bucket = request.values.get("bucket")
date = request.values.get("date")
if request.method == "POST":
# change the expired date of bucket
if not has_bucket(bucket):
info = "no bucket found"
elif not date:
info = "no date to set"
else:
change_bucket_expired_date(bucket, date)
service_info = get_bucket_service_info(bucket)
html = render_api_template("page_admin_bucket_expired_date.jade",
info=info,
service_info=service_info)
return Response(html)
def create_record_by_sync(bucket, record, check_bucket=False):
if not isinstance(record, dict):
return 'record is not a dict'
record_id = record.pop('_id', None)
if not record_id:
return 'record_id is missing'
if not is_object_id(record_id):
return 'record_id is not a valid ObjectID'
error_info = get_record_data_error_info(record)
if error_info:
return error_info
if check_bucket: # current node has the bucket or not
if not has_bucket(bucket):
return 'no bucket matched'
py_record_data = ssdb_data_to_py_data(record)
saved = hset(bucket, record_id, py_record_data, ignore_if_exists=True)
if saved:
after_record_created(bucket, py_record_data, object_id=record_id, )
def create_bucket_by_public_key(public_key,
init_configs=None,
force_to_create=False):
# 本质上是创建一个 创世configs
if not force_to_create:
# todo 需要校验邀请码？
# 非强制需要创建的，需要被邀请才能创建 bucket
return False
bucket = get_bucket_by_public_key(public_key)
if not bucket:
return False
if has_bucket(bucket): # 已经存在了，不允许重新创建, 认为已经创建成功了
return True
now = int(time.time())
now_date = datetime.datetime.utcfromtimestamp(now)
now_date_string = now_date.strftime('%Y-%m-%d %H:%M:%S UTC')
bucket_info = dict(
public_key=public_key,
created_at=now,
created_date=now_date_string,
)
if init_configs and isinstance(init_configs, dict):
allowed_to_update = True
init_configs_bytes = json_dumps(init_configs)
if len(init_configs_bytes) > 10 * 1024:
allowed_to_update = False
if allowed_to_update:
init_configs.update(bucket_info)
bucket_info = init_configs
# 创建 init config, 这是不可修改的
hset(bucket, zero_id, bucket_info, ignore_if_exists=True)
# 创建的时候，也进行这个操作，方便知道 buckets 的总数相关数据
set_bucket_into_buckets(bucket)
# 创建的时候，给 30 days 的有效期
change_bucket_expired_date(bucket)
return True
def init_server_status_bucket():
if 'utc_offset' not in os.environ:
# 统计系统信息时候，可读性使用的 utc_offset
utc_offset = get_env('utc_offset')
if utc_offset is None:
utc_offset = 8
try:
utc_offset = str(utc_offset)
except:
utc_offset = '8'
os.environ['utc_offset'] = utc_offset
configs = get_server_status_bucket_configs()
bucket = configs['bucket']
public_key = configs['public_key']
if has_bucket(bucket):
return
create_bucket_by_public_key(public_key)
set_bucket_configs(bucket,
config_type='pages',
configs=bucket_web_template)
def create_bucket_by_web_request(invitation_code=None):
# return None or error_info
# token, bucket(check by private_key), private_key, domain,
if not request.method == 'POST':
return 'invalid request Method'
private_key = request.values.get('private_key', '').strip()
try:
bucket = get_bucket_by_private_key(private_key)
except:
bucket = None
if not bucket:
return 'invalid private key'
public_key = get_public_key_from_private_key(private_key)
is_first_bucket = False
if not has_bucket(bucket): # bucket 如果已经创建了，不走这个流程
if get_buckets_size() == 0:
is_first_bucket = True
else: # 需要邀请码的
if not can_use_invitation(invitation_code):
# 错误或者已经使用了的邀请码
return "invitation code used or invalid"
created = create_bucket_by_public_key(public_key=public_key,
force_to_create=True)
if not created:
return 'unknown error'
else:
bucket = get_bucket_by_public_key(public_key)
if is_first_bucket: # 向系统写入 first bucket
ssdb_set("first_bucket", bucket)
gevent.spawn_later(
1, try_to_reload_web_app) # try to restart web server
elif invitation_code: # 保存已经使用的邀请码信息
use_invitation(invitation_code, bucket)
# at last, login bucket
mark_bucket_login_by_private_key(private_key)
def check_bucket(self):
if self.bucket and has_bucket(self.bucket):
return json_with_status_code(200, 'ok')
else:
return json_with_status_code(404, 'not found')
def append_to_markdown_doc_and_sync(bucket,
path,
content,
lines_to_append=1,
reverse=False,
do_not_repeat=True,
lines_more_diff=None,
draft_by_default=False):
# 默认检测 append 的内容是否重复
if not bucket or not path or not content:
return
if not isinstance(bucket, string_types) or not isinstance(
path, string_types) or not isinstance(content, string_types):
return
if not has_bucket(bucket):
return
if not is_a_markdown_file(path):
return
content = smart_unicode(content)
old_doc = get_record_by_path(bucket, path=path) or {}
if not isinstance(old_doc, dict):
old_doc = {}
if lines_more_diff: # 多长时间hi后，自动多一空行
if old_doc and old_doc.get('timestamp'):
try:
diff = time.time() - old_doc.get('timestamp')
if diff > lines_more_diff:
lines_to_append += 1
except:
pass
interval = '\r\n' * abs(to_int(lines_to_append, max_value=10)) # 间隔换行
if old_doc:
if get_type_from_record(old_doc) == 'post': # 目前仅支持日志类型文件的append
old_content = old_doc.get('raw_content')
if old_content == " ":
old_content = ""
if do_not_repeat:
if reverse:
if old_content.strip().startswith(content.strip()):
return ""
else:
old_content_s = old_content.strip()
appended_content_s = content.strip()
if old_content_s.endswith(
'\n' + appended_content_s
) or old_content_s == appended_content_s:
return '' # ignore, 重复的内容不处理
if reverse: # 插入头部位置
new_content = '%s%s' % (content, interval)
content = '%s%s' % (new_content, old_content)
else:
new_content = '%s%s' % (interval, content)
content = '%s%s' % (old_content, new_content)
else:
return
else: # new doc
content = content.strip()
if draft_by_default:
# 新建文档默认是 draft 的状态
if re.match(u"\w+[:\uff1a]", content): # 可能用户自己声明了 metadata
content = "status: draft\n%s" % content
else:
now = get_now_from_bucket(bucket)
content = "date: %s\nstatus: draft\n\n%s" % (now, content)
sync_file_by_server_side(bucket=bucket,
relative_path=path,
content=content)
return True # done
def get_wechat_bind_code_for_bucket(bucket):
if not has_bucket(bucket):
return ""
signature = get_signature_for_bucket(bucket, salt="wechat")
code = "%s-%s" % (bucket, signature)
return code
response = requests.get(url, timeout=timeout, verify=False)
if response.status_code > 300:
return # ignore
response_content = smart_str(response.content)
if not response_content:
return
sync_file_by_server_side(bucket=bucket, relative_path=path, content=response_content) # 进行同步
return True
except:
pass
def download_from_internet_and_sync(bucket, url, folder_to_save='/_data/downloads', path=None, timeout=10, force=False, async=True):
# 从互联上下载内容
if not has_bucket(bucket):
return ""
if not path: # 自动生成doc_path, 以url为md5作为filename
url_path = get_url_path(url)
ext = os.path.splitext(url_path)[-1]
url_md5 = get_md5(url)
filename = url_md5 + ext
path = '/%s/%s' % (folder_to_save.strip('/'), filename)
if not force:
# 非强制的情况下，如果文件已经存在，就不下载了
if has_record_by_path(bucket, path):
return path
if async:
gevent.spawn(do_download_from_internet_and_sync, bucket, path, url=url, timeout=timeout)
else:
do_download_from_internet_and_sync(bucket, path, url, timeout=timeout)
def update_server_status_bucket_template():
bucket = get_server_status_bucket()
if bucket and has_bucket(bucket):
set_bucket_configs(bucket,
config_type='pages',
configs=bucket_web_template)