def load_theme_from_template_folder_for_bucket(bucket, prefix="template"): if not has_bucket(bucket): return info = {} prefix = prefix.strip().strip("/") template_paths = get_paths_under(bucket, under=prefix) for _relative_path in template_paths: relative_path = _relative_path.replace("%s/"%prefix, "", 1) raw_content = get_raw_content_by_path(bucket, _relative_path) if not raw_content: continue path_without_ext, ext = os.path.splitext(relative_path) ext = ext.lower().strip('.') if ext not in allowed_exts: continue raw_content = smart_unicode(raw_content) # to unicode info[relative_path] = raw_content matched_compiler = server_side_template_resource_compilers.get(ext) if matched_compiler: new_ext, compile_func = matched_compiler try: compiled_content = compile_func(raw_content) new_key = path_without_ext + '.' + new_ext.strip('.') info[new_key] = compiled_content except Exception as e: pass info["_route"] = get_templates_route_info(info) set_bucket_configs(bucket, info, config_type='pages') return info
def set_bucket_private_configs(bucket, configs): if not isinstance(configs, dict): return if not has_bucket(bucket): return configs_data = json_dumps(configs) if len(configs_data) > 500 * 1024: # 不能大于 500k return else: hset("_bucket_private_configs", bucket, configs_data) return True
def set_bucket_pages_configs_by_web_api(bucket, remote_url, timeout=3): # True or False if not has_bucket(bucket): return # ignore gevent_job = gevent.spawn(do_set_bucket_pages_configs_by_web_api, bucket, remote_url, timeout) try: result = gevent_job.get(block=True, timeout=timeout) return result except: gevent_job.kill(block=False) return False
def set_bucket_api_token(bucket, db_name="_bucket_api_token"): """ :param bucket: 需要 :param db_name: 提供了默认值 :return: new_api_token """ if not is_valid_bucket_name(bucket): return "" if not has_bucket(bucket): return "" new_token = get_a_random_api_token() hset(db_name, bucket, new_token) return new_token
def mark_bucket_login_by_private_key(private_key): if not private_key: return False bucket = get_bucket_by_private_key(private_key) if not bucket: return False if not has_bucket(bucket): return False login_key = get_bucket_login_key(bucket) utoken_for_cookie = '%s-%s' % (bucket, login_key) set_cookie('utoken', utoken_for_cookie, max_age=120 * 24 * 60 * 60) # 120 days set_logined_bucket_in_request(bucket, checked=True) return True
def do_download_from_internet_and_sync(bucket, path, url, timeout=10): # 下载行为, 这里要获得site,因为如果是异步的话,g.site是无效的 if not has_bucket(bucket): return try: response = requests.get(url, timeout=timeout, verify=False) if response.status_code > 300: return # ignore response_content = smart_str(response.content) if not response_content: return sync_file_by_server_side(bucket=bucket, relative_path=path, content=response_content) # 进行同步 return True except: pass
def check_wechat_bind_code(bind_code): # 校验成功后,比较偷懒,返回的不是 True,而是 bucket if not isinstance(bind_code, string_types): return False if "-" not in bind_code: return False bucket, signature = bind_code.strip().split("-", 1) if not check_signature_for_bucket( bucket, signature=signature, hours=1, salt="wechat"): # 1 小时有效期 return False else: if not has_bucket(bucket): return False else: return bucket
def update_record_directly(bucket, record): # 通过 record 来更新 record,不需要额外的编译 # 如果有 path 路径,不要去更改它 if not isinstance(record, dict): return False if not has_bucket(bucket): return False record['_auto_clean_bucket'] = True # 创建新的 record create_record(bucket, record) object_id = record.get("_id") if object_id: update_path_related_when_record_changed(bucket, object_id, record) return True
def do_set_bucket_pages_configs_by_web_api(bucket, remote_url, timeout=3): if not has_bucket(bucket): return if not isinstance(remote_url, string_types): return if "://" not in remote_url: remote_url = "http://" + remote_url try: response = requests.get(remote_url, timeout=timeout) raw_pages_configs = response.json() if not isinstance(raw_pages_configs, dict): return if not raw_pages_configs.get("_route"): return raw_pages_configs["can_copy"] = False set_bucket_configs(bucket, raw_pages_configs, config_type="pages") return True except: pass
def change_bucket_date(): if not get_logined_admin_bucket(): abort(404, "not admin") info = "" bucket = request.values.get("bucket") date = request.values.get("date") if request.method == "POST": # change the expired date of bucket if not has_bucket(bucket): info = "no bucket found" elif not date: info = "no date to set" else: change_bucket_expired_date(bucket, date) service_info = get_bucket_service_info(bucket) html = render_api_template("page_admin_bucket_expired_date.jade", info=info, service_info=service_info) return Response(html)
def create_record_by_sync(bucket, record, check_bucket=False): if not isinstance(record, dict): return 'record is not a dict' record_id = record.pop('_id', None) if not record_id: return 'record_id is missing' if not is_object_id(record_id): return 'record_id is not a valid ObjectID' error_info = get_record_data_error_info(record) if error_info: return error_info if check_bucket: # current node has the bucket or not if not has_bucket(bucket): return 'no bucket matched' py_record_data = ssdb_data_to_py_data(record) saved = hset(bucket, record_id, py_record_data, ignore_if_exists=True) if saved: after_record_created(bucket, py_record_data, object_id=record_id, )
def create_bucket_by_public_key(public_key, init_configs=None, force_to_create=False): # 本质上是创建一个 创世configs if not force_to_create: # todo 需要校验邀请码? # 非强制需要创建的,需要被邀请才能创建 bucket return False bucket = get_bucket_by_public_key(public_key) if not bucket: return False if has_bucket(bucket): # 已经存在了,不允许重新创建, 认为已经创建成功了 return True now = int(time.time()) now_date = datetime.datetime.utcfromtimestamp(now) now_date_string = now_date.strftime('%Y-%m-%d %H:%M:%S UTC') bucket_info = dict( public_key=public_key, created_at=now, created_date=now_date_string, ) if init_configs and isinstance(init_configs, dict): allowed_to_update = True init_configs_bytes = json_dumps(init_configs) if len(init_configs_bytes) > 10 * 1024: allowed_to_update = False if allowed_to_update: init_configs.update(bucket_info) bucket_info = init_configs # 创建 init config, 这是不可修改的 hset(bucket, zero_id, bucket_info, ignore_if_exists=True) # 创建的时候,也进行这个操作,方便知道 buckets 的总数相关数据 set_bucket_into_buckets(bucket) # 创建的时候,给 30 days 的有效期 change_bucket_expired_date(bucket) return True
def init_server_status_bucket(): if 'utc_offset' not in os.environ: # 统计系统信息时候,可读性使用的 utc_offset utc_offset = get_env('utc_offset') if utc_offset is None: utc_offset = 8 try: utc_offset = str(utc_offset) except: utc_offset = '8' os.environ['utc_offset'] = utc_offset configs = get_server_status_bucket_configs() bucket = configs['bucket'] public_key = configs['public_key'] if has_bucket(bucket): return create_bucket_by_public_key(public_key) set_bucket_configs(bucket, config_type='pages', configs=bucket_web_template)
def create_bucket_by_web_request(invitation_code=None): # return None or error_info # token, bucket(check by private_key), private_key, domain, if not request.method == 'POST': return 'invalid request Method' private_key = request.values.get('private_key', '').strip() try: bucket = get_bucket_by_private_key(private_key) except: bucket = None if not bucket: return 'invalid private key' public_key = get_public_key_from_private_key(private_key) is_first_bucket = False if not has_bucket(bucket): # bucket 如果已经创建了,不走这个流程 if get_buckets_size() == 0: is_first_bucket = True else: # 需要邀请码的 if not can_use_invitation(invitation_code): # 错误或者已经使用了的邀请码 return "invitation code used or invalid" created = create_bucket_by_public_key(public_key=public_key, force_to_create=True) if not created: return 'unknown error' else: bucket = get_bucket_by_public_key(public_key) if is_first_bucket: # 向系统写入 first bucket ssdb_set("first_bucket", bucket) gevent.spawn_later( 1, try_to_reload_web_app) # try to restart web server elif invitation_code: # 保存已经使用的邀请码信息 use_invitation(invitation_code, bucket) # at last, login bucket mark_bucket_login_by_private_key(private_key)
def check_bucket(self): if self.bucket and has_bucket(self.bucket): return json_with_status_code(200, 'ok') else: return json_with_status_code(404, 'not found')
def append_to_markdown_doc_and_sync(bucket, path, content, lines_to_append=1, reverse=False, do_not_repeat=True, lines_more_diff=None, draft_by_default=False): # 默认检测 append 的内容是否重复 if not bucket or not path or not content: return if not isinstance(bucket, string_types) or not isinstance( path, string_types) or not isinstance(content, string_types): return if not has_bucket(bucket): return if not is_a_markdown_file(path): return content = smart_unicode(content) old_doc = get_record_by_path(bucket, path=path) or {} if not isinstance(old_doc, dict): old_doc = {} if lines_more_diff: # 多长时间hi后,自动多一空行 if old_doc and old_doc.get('timestamp'): try: diff = time.time() - old_doc.get('timestamp') if diff > lines_more_diff: lines_to_append += 1 except: pass interval = '\r\n' * abs(to_int(lines_to_append, max_value=10)) # 间隔换行 if old_doc: if get_type_from_record(old_doc) == 'post': # 目前仅支持日志类型文件的append old_content = old_doc.get('raw_content') if old_content == " ": old_content = "" if do_not_repeat: if reverse: if old_content.strip().startswith(content.strip()): return "" else: old_content_s = old_content.strip() appended_content_s = content.strip() if old_content_s.endswith( '\n' + appended_content_s ) or old_content_s == appended_content_s: return '' # ignore, 重复的内容不处理 if reverse: # 插入头部位置 new_content = '%s%s' % (content, interval) content = '%s%s' % (new_content, old_content) else: new_content = '%s%s' % (interval, content) content = '%s%s' % (old_content, new_content) else: return else: # new doc content = content.strip() if draft_by_default: # 新建文档默认是 draft 的状态 if re.match(u"\w+[:\uff1a]", content): # 可能用户自己声明了 metadata content = "status: draft\n%s" % content else: now = get_now_from_bucket(bucket) content = "date: %s\nstatus: draft\n\n%s" % (now, content) sync_file_by_server_side(bucket=bucket, relative_path=path, content=content) return True # done
def get_wechat_bind_code_for_bucket(bucket): if not has_bucket(bucket): return "" signature = get_signature_for_bucket(bucket, salt="wechat") code = "%s-%s" % (bucket, signature) return code
response = requests.get(url, timeout=timeout, verify=False) if response.status_code > 300: return # ignore response_content = smart_str(response.content) if not response_content: return sync_file_by_server_side(bucket=bucket, relative_path=path, content=response_content) # 进行同步 return True except: pass def download_from_internet_and_sync(bucket, url, folder_to_save='/_data/downloads', path=None, timeout=10, force=False, async=True): # 从互联上下载内容 if not has_bucket(bucket): return "" if not path: # 自动生成doc_path, 以url为md5作为filename url_path = get_url_path(url) ext = os.path.splitext(url_path)[-1] url_md5 = get_md5(url) filename = url_md5 + ext path = '/%s/%s' % (folder_to_save.strip('/'), filename) if not force: # 非强制的情况下,如果文件已经存在,就不下载了 if has_record_by_path(bucket, path): return path if async: gevent.spawn(do_download_from_internet_and_sync, bucket, path, url=url, timeout=timeout) else: do_download_from_internet_and_sync(bucket, path, url, timeout=timeout)
def update_server_status_bucket_template(): bucket = get_server_status_bucket() if bucket and has_bucket(bucket): set_bucket_configs(bucket, config_type='pages', configs=bucket_web_template)