def get_ssl_cert_for_domain(domain):
domain = domain.strip().lower()
doc = hget('_domain_ssl', domain) or {}
if not doc and domain.startswith("www."):
domain = domain.replace("www.", "", 1)
doc = hget('_domain_ssl', domain) or {}
if not isinstance(doc, dict):
doc = {}
return doc
def get_system_domain_from_bucket(bucket):
# 获得的是系统提供的二级域名，
domain_doc = hget('_rdomain', bucket)
if domain_doc and isinstance(domain_doc, dict):
return domain_doc.get('domain')
else:
return None
def get_bucket_configs(bucket, config_type='init'):
# 'init', 'user', 'pages'
if not bucket:
return {}
config_doc_id = bucket_config_doc_id_names.get(config_type)
if not config_doc_id:
return {}
request_var_name = 'bucket_%s_cached_%s_value' % (bucket, config_type)
if hasattr(request, request_var_name):
return getattr(request, request_var_name)
info = hget(bucket, config_doc_id)
if not info or not isinstance(info, dict):
info = {}
if config_type in ['user', 'secret'] and info:
# 需要解密的数据类型
raw_data = info.get('data')
real_info = get_normal_data_by_simple_token(bucket,
raw_data,
force_py_data=True)
if not isinstance(real_info, dict):
real_info = {}
info = real_info
if config_type == "site" and has_bucket(bucket):
# site 默认给的 settings
site_configs = default_site_configs.copy()
site_configs.update(info)
info = site_configs
try:
setattr(request, request_var_name, info)
except:
pass
return info
def get_current_node_status():
node_id = get_current_node_id() or '?'
status_info = dict(
id = node_id,
node_id = node_id,
buckets_size = zsize('buckets')
)
# todo 提供 first_bucket & last_bucket，会不会造成潜在的隐私泄露？
first_bucket_timestamp = zget_min('buckets')
if first_bucket_timestamp:
first_bucket, first_bucket_timestamp = first_bucket_timestamp
status_info['first_bucket'] = first_bucket
status_info['first_bucket_date'] = db_timestamp_to_date_string(first_bucket_timestamp)
last_bucket_timestamp = zget_max('buckets')
if last_bucket_timestamp:
last_bucket, last_bucket_timestamp = last_bucket_timestamp
status_info['last_bucket'] = last_bucket
status_info['last_bucket_date'] = db_timestamp_to_date_string(last_bucket_timestamp)
records_count = hget('_records_count', 'all') or 0
try:
records_count = int(records_count)
except:
pass
status_info['records_count'] = records_count
status_info['db'] = get_db_system_status()
now = datetime.datetime.utcnow()
status_info['date'] = now.strftime('%Y-%m-%d %H:%M:%S UTC')
return status_info
def get_record(bucket, record_id, zero_ids_allowed=False, force_dict=False):
if not zero_ids_allowed and record_id in zero_ids:
return None
if not bucket:
return {}
record = hget(bucket, record_id, force_dict=force_dict)
return record
def after_path_related_record_deleted(bucket, record_data):
path = get_path_from_record(record_data)
if not path:
return
bucket_name_for_id = get_bucket_name_for_path(bucket)
bucket_name_for_url = get_bucket_name_for_url(bucket)
bucket_name_for_slash = get_bucket_name_for_slash(bucket)
bucket_name_for_order = get_bucket_name_for_order_by_record(
bucket, record_data)
#bucket_name_for_order_file_type = "%s_file_order" % bucket
# path 上清掉
hdel(bucket_name_for_id, path)
# 数据类型，对应的排序
if bucket_name_for_order:
zdel(bucket_name_for_order, path)
#if bucket_name_for_order != bucket_name_for_order_file_type:
# zdel(bucket_name_for_order_file_type, path)
# slash 上清掉
zdel(bucket_name_for_slash, path)
# url 上两个进行删除
url_path = get_url_path(record_data) or hget(bucket_name_for_url, path)
if url_path:
hdel(bucket_name_for_url, url_path)
hdel(bucket_name_for_url, path)
# 删除文件, 使用 storage 来处理这里的逻辑
storage.when_record_deleted(bucket, record_data)
def get_record_id_by_path(bucket, path):
path_bucket = get_bucket_name_for_path(bucket)
path = path.strip().lstrip('/').lower()
record_object_id = hget(path_bucket, path)
if record_object_id:
record_object_id = just_get_record_id(record_object_id)
record_object_id = record_object_id.lstrip('#')
return record_object_id
def get_bucket_usage(bucket, for_human=False):
try:
usage = int(hget('_bucket_usage', bucket))
except:
usage = 0
if for_human:
usage = bytes2human(usage)
return usage
def get_bucket_domains(bucket):
if not bucket:
return []
domains = hget("_domains", bucket, force_dict=False)
if not isinstance(domains, (list, tuple)):
return []
else:
return domains
def get_bucket_private_configs(bucket):
# 只允许在服务端中获取，不允许返回给 client 查看
cache_key = "%s_private_configs" % bucket
cached_value = get_context_value_from_request(cache_key)
if cached_value is not None:
return cached_value
configs = hget("_bucket_private_configs", bucket)
if not isinstance(configs, dict):
configs = {}
set_context_value_from_request(cache_key, configs)
return configs
def get_post_visits_count(doc, field='visits'):
# field is in ['visits', 'visitors']
bucket = get_bucket_in_request_context()
if not bucket:
return 0
visits_db_name = get_visits_db_name_for_bucket(bucket)
doc_path = doc.get('path')
if not doc_path:
return 0
key = get_visits_key(doc_path, field=field)
count = hget(visits_db_name, key) or 0
count = to_int(count, default_if_fail=0)
return count
def register_bucket_domain_from_system(bucket, domain, is_admin=False):
# 注册系统提供的二级域名
# 一个 bucket 只允许注册一个系统提供的二级域名
# 返回 None or 错误信息
domain = domain.strip().lower()
if not is_valid_bucket_name(bucket):
return 'invalid bucket'
domain_info = get_domain_basic_info(domain, is_admin=is_admin)
is_system_domain = domain_info.get('is_system_domain', False)
is_allowed = domain_info.get('allowed', False)
if is_system_domain and is_allowed:
r_domain_info = hget('_rdomain', bucket) # 当前 bucket 是否已经域名绑定了
parked_domain_info = hget('_domain', domain) # 这个域名是否已经被其它 bucket 绑定了
if parked_domain_info: # domain 已经注册过了
if parked_domain_info.get('bucket') == bucket:
return None
return '%s is used by other bucket' % domain
if r_domain_info: # bucket 已经绑定过 domain 了
master_old_domain = r_domain_info.get('domain')
if master_old_domain == domain:
return None
else:
# 一个 bucket 只能绑定一个系统的二级域名，就会删除之前的一个
hdel('_rdomain', bucket)
hdel('_domain', master_old_domain)
pull_domain_from_bucket(bucket,
master_old_domain) # 汇总 domains
# 一个 _rdomain, 作为一个反向的对应, 一个 bucket 只能有一个系统级的域名 ?
domain_doc = dict(bucket=bucket, domain=domain, created_at=time.time())
hset('_domain', domain, domain_doc)
hset('_rdomain', bucket, domain_doc)
push_domain_to_bucket(bucket, domain) # 汇总 domains
return None
else:
return '%s is not allowed for bucket:%s' % (domain, bucket)
def get_bucket_status(bucket):
status_info = {}
size = get_bucket_size(bucket)
status_info['size'] = size
last_updated_at = zget('buckets', bucket)
if last_updated_at:
status_info['date'] = db_timestamp_to_date_string(last_updated_at)
max_record_id = hget('_bucket_max_id', bucket)
delta_record_id = hget('_bucket_delta_id', bucket)
status_info['max_record_id'] = max_record_id
status_info['delta_record_id'] = delta_record_id
status_info['max_record_date'] = record_id_to_date_string(max_record_id)
status_info['delta_record_date'] = record_id_to_date_string(delta_record_id)
try:
usage = int(hget('_bucket_usage', bucket) or 0)
except:
usage = 0
status_info['usage'] = usage
status_info['usage_for_human'] = bytes2human(usage)
if usage and size:
usage_per_record = round(usage/float(size), 2)
status_info['usage_per_record'] = usage_per_record
status_info['usage_per_record_for_human'] = bytes2human(usage_per_record)
return status_info
def get_record_by_path(bucket, path, force_dict=False):
if not bucket:
return None
if not path:
return None
if not isinstance(path, string_types):
return None
record_object_id = get_record_id_by_path(bucket, path)
if record_object_id:
record = hget(bucket, record_object_id, force_dict=force_dict)
if isinstance(record, dict) and not record.get("_id"):
record["_id"] = record_object_id
else:
record = None
return record
def get_bucket_from_domain(domain):
if not domain:
return
domain = smart_unicode(domain)
domain = domain.strip().lower()
if ADMIN_BUCKET and domain in WEBSITE_DOMAINS:
return ADMIN_BUCKET
maybe_domains = [domain]
if domain.startswith('www.'):
maybe_domains.append(domain.replace('www.', '', 1))
else:
maybe_domains.append('www.%s' % domain)
for d in maybe_domains:
db_domain_info = hget('_domain', d)
if db_domain_info and isinstance(db_domain_info, dict):
return db_domain_info.get('bucket')
def show_avatar(avatar_id):
avatar_id = get_avatar_id(avatar_id)
avatar_doc = hget('_avatar', avatar_id)
now = time.time()
if avatar_doc:
avatar_date = avatar_doc.get('date')
avatar_image_content = avatar_doc.get('content')
to_clear = False
if not avatar_date:
to_clear = True
elif (now - avatar_date) > 5 * 24 * 60 * 60: # 5days
to_clear = True
elif (
now - avatar_date
) > 1 * 24 * 60 * 60 and not avatar_image_content: # 1day for empty avatar image
to_clear = True
if to_clear:
# avatar_doc 缓存 5 天
hdel('_avatar', avatar_id)
avatar_doc = None
if not avatar_doc:
avatar_image_content = get_gavatar_image_content(avatar_id) or ''
if avatar_image_content:
avatar_image_content = base64.b64encode(avatar_image_content)
avatar_doc = dict(date=now, content=avatar_image_content)
hset('_avatar', avatar_id, avatar_doc)
if not is_doc_modified(avatar_doc, date_field='date'):
return get_304_response()
else:
avatar_image_content = avatar_doc.get('content') or ''
if avatar_image_content:
avatar_image_content = base64.b64decode(avatar_image_content)
response = Response(avatar_image_content, mimetype='image/png')
set_304_response_for_doc(avatar_doc, response, date_field='date')
return response
else:
# 默认的 url
r_response = send_static_file('defaults/avatar.png')
if r_response:
return r_response
# at last
abort(404)
def delete_bucket_domain(domain, bucket=None):
# 删除 bucket 上的 domain， 支持独立域名以及系统二级域名
# 这里未做是否有权限 unregister 的校验， 需要前置校验
# 未指定 bucket，就会删除 domain 对应的唯一 bucket；指定了，则进行 bucket 一致性校验
# 返回 None or 错误信息
domain = domain.strip().lower()
if bucket and not is_valid_bucket_name(bucket):
return 'invalid bucket'
domain_record = hget('_domain', domain)
if domain_record:
to_delete = False
bucket_in_db = domain_record.get('bucket')
if bucket and bucket_in_db == bucket:
to_delete = True
elif not bucket and bucket_in_db:
to_delete = True
if to_delete:
hdel('_domain', domain)
hdel('_rdomain',
bucket_in_db) # 尝试删除 rdomain 上的信息, 这样下次 bucket 还能再注册一个系统二级域名
pull_domain_from_bucket(bucket_in_db, domain) # 汇总 domains
return None
return 'can not find the matched domain record to remove for %s' % domain
def register_bucket_independent_domain(bucket, domain):
# 注册独立域名， 这个前提是域名已经 park 到当前节点，并且已经做了必要的校验
# 返回 None or 错误信息
domain = domain.strip().lower()
if not is_valid_domain(domain):
return 'domain format error or not supported'
if not is_valid_bucket_name(bucket):
return 'invalid bucket'
if not has_bucket(bucket):
return 'current node does not have bucket:%s' % bucket
bucket = bucket.strip()
old_domain_info = hget('_domain', domain) or {}
old_matched_bucket = old_domain_info.get('bucket')
if old_matched_bucket == bucket: # # 已经注册过了，不做处理
return None
#return 'registered already for this bucket, no need to change'
if domain == "thisisadomainfortest.com": # for test only
bucket_in_domain_text = bucket
else:
bucket_in_domain_text = get_domain_text_record(domain)
if bucket_in_domain_text == bucket:
# 比如 A 曾经注册过，后来 domain 是 B 的了，那么 B 修改了 TXT 记录，就可以重新注册了。
# at last, create or modify
hset('_domain', domain, dict(
bucket=bucket,
created_at=time.time(),
))
push_domain_to_bucket(bucket, domain) # 汇总 domains
return None # done
else:
if bucket_in_domain_text:
if not is_valid_bucket_name(bucket_in_domain_text):
return 'invalid bucket format in TXT record: %s' % bucket_in_domain_text
else:
return 'TXT record is not matched to %s' % bucket
else:
return 'should set TXT record for domain first'
def get_records_for_bucket(bucket,
start_record_id=None,
end_record_id=None,
limit=1000,
includes_start_record_id=False,
reverse_scan=False,
raw=False):
# start_record_id means start here but does not include itself
# 在两个地方用到： url 中 list bucket 的 & 模板引擎中调用 records 的
# 模板引擎中调用的话，一般都会指定 start_record_id，避免一些 zero_ids 在实际呈现无意义的数据被显示出来
# 一般是不 includes_start_record_id， 上一次获得的 list 的最后一个 record 的 id 会作为 cursor (start_id)，
# 也就是说 start_id 不应该在当前获取的 list 内
records = hscan(bucket,
key_start=start_record_id,
key_end=end_record_id,
limit=limit,
reverse_scan=reverse_scan)
if includes_start_record_id and start_record_id:
start_record = hget(bucket, start_record_id)
if start_record:
records.insert(0, [start_record_id, start_record])
if not raw:
records = to_py_records_from_raw_ssdb_records(records)
return records
def get_bucket_by_wechat_user_id(wechat_user_id):
if not isinstance(wechat_user_id, string_types):
return None
return hget("wechat_accounts", wechat_user_id)
def get_name_by_wechat_user_id(wechat_user_id):
if not isinstance(wechat_user_id, string_types):
return None
return hget("wechat_names", wechat_user_id)
def get_invitation(code):
# 邀请码是 code 也是 id
record = hget('_bucket_invite', code) or {}
return record
def get_buckets_cursor_for_remote_node(node):
cursor = hget('_remote_buckets_cursor', node) or ''
return cursor
def get_simple_bucket_token(bucket):
token = hget('_simple_bucket_token', bucket) or ''
return token
def get_bucket_delta_id(bucket):
delta_id = hget('_bucket_delta_id', bucket) or ''
return delta_id
def get_bucket_max_id(bucket):
max_id = hget('_bucket_max_id', bucket) or ''
return max_id
def do_get_record_by_url(bucket, url_path):
url_path = url_path.strip().lower()
bucket_for_url = '%s_url' % bucket
path_data_id = hget(bucket_for_url, url_path)
if path_data_id:
return get_record_by_path(bucket, path_data_id)
def get_bucket_service_info(bucket):
if not bucket:
return {}
if not is_valid_bucket_name(bucket):
return {}
return hget("_bucket_info", bucket, force_dict=True)
def get_bucket_last_record_id_computed(bucket):
record_id = hget("buckets_file_cursor_computed", bucket) or None
return record_id
def allowed_to_create_record_in_bucket(bucket, record_md5):
last_md5 = hget('_bucket_max_md5', bucket)
if last_md5 and last_md5 == record_md5:
return False
else:
return True
