def get_ssl_cert_for_domain(domain): domain = domain.strip().lower() doc = hget('_domain_ssl', domain) or {} if not doc and domain.startswith("www."): domain = domain.replace("www.", "", 1) doc = hget('_domain_ssl', domain) or {} if not isinstance(doc, dict): doc = {} return doc
def get_system_domain_from_bucket(bucket): # 获得的是系统提供的二级域名, domain_doc = hget('_rdomain', bucket) if domain_doc and isinstance(domain_doc, dict): return domain_doc.get('domain') else: return None
def get_bucket_configs(bucket, config_type='init'): # 'init', 'user', 'pages' if not bucket: return {} config_doc_id = bucket_config_doc_id_names.get(config_type) if not config_doc_id: return {} request_var_name = 'bucket_%s_cached_%s_value' % (bucket, config_type) if hasattr(request, request_var_name): return getattr(request, request_var_name) info = hget(bucket, config_doc_id) if not info or not isinstance(info, dict): info = {} if config_type in ['user', 'secret'] and info: # 需要解密的数据类型 raw_data = info.get('data') real_info = get_normal_data_by_simple_token(bucket, raw_data, force_py_data=True) if not isinstance(real_info, dict): real_info = {} info = real_info if config_type == "site" and has_bucket(bucket): # site 默认给的 settings site_configs = default_site_configs.copy() site_configs.update(info) info = site_configs try: setattr(request, request_var_name, info) except: pass return info
def get_current_node_status(): node_id = get_current_node_id() or '?' status_info = dict( id = node_id, node_id = node_id, buckets_size = zsize('buckets') ) # todo 提供 first_bucket & last_bucket,会不会造成潜在的隐私泄露? first_bucket_timestamp = zget_min('buckets') if first_bucket_timestamp: first_bucket, first_bucket_timestamp = first_bucket_timestamp status_info['first_bucket'] = first_bucket status_info['first_bucket_date'] = db_timestamp_to_date_string(first_bucket_timestamp) last_bucket_timestamp = zget_max('buckets') if last_bucket_timestamp: last_bucket, last_bucket_timestamp = last_bucket_timestamp status_info['last_bucket'] = last_bucket status_info['last_bucket_date'] = db_timestamp_to_date_string(last_bucket_timestamp) records_count = hget('_records_count', 'all') or 0 try: records_count = int(records_count) except: pass status_info['records_count'] = records_count status_info['db'] = get_db_system_status() now = datetime.datetime.utcnow() status_info['date'] = now.strftime('%Y-%m-%d %H:%M:%S UTC') return status_info
def get_record(bucket, record_id, zero_ids_allowed=False, force_dict=False): if not zero_ids_allowed and record_id in zero_ids: return None if not bucket: return {} record = hget(bucket, record_id, force_dict=force_dict) return record
def after_path_related_record_deleted(bucket, record_data): path = get_path_from_record(record_data) if not path: return bucket_name_for_id = get_bucket_name_for_path(bucket) bucket_name_for_url = get_bucket_name_for_url(bucket) bucket_name_for_slash = get_bucket_name_for_slash(bucket) bucket_name_for_order = get_bucket_name_for_order_by_record( bucket, record_data) #bucket_name_for_order_file_type = "%s_file_order" % bucket # path 上清掉 hdel(bucket_name_for_id, path) # 数据类型,对应的排序 if bucket_name_for_order: zdel(bucket_name_for_order, path) #if bucket_name_for_order != bucket_name_for_order_file_type: # zdel(bucket_name_for_order_file_type, path) # slash 上清掉 zdel(bucket_name_for_slash, path) # url 上两个进行删除 url_path = get_url_path(record_data) or hget(bucket_name_for_url, path) if url_path: hdel(bucket_name_for_url, url_path) hdel(bucket_name_for_url, path) # 删除文件, 使用 storage 来处理这里的逻辑 storage.when_record_deleted(bucket, record_data)
def get_record_id_by_path(bucket, path): path_bucket = get_bucket_name_for_path(bucket) path = path.strip().lstrip('/').lower() record_object_id = hget(path_bucket, path) if record_object_id: record_object_id = just_get_record_id(record_object_id) record_object_id = record_object_id.lstrip('#') return record_object_id
def get_bucket_usage(bucket, for_human=False): try: usage = int(hget('_bucket_usage', bucket)) except: usage = 0 if for_human: usage = bytes2human(usage) return usage
def get_bucket_domains(bucket): if not bucket: return [] domains = hget("_domains", bucket, force_dict=False) if not isinstance(domains, (list, tuple)): return [] else: return domains
def get_bucket_private_configs(bucket): # 只允许在服务端中获取,不允许返回给 client 查看 cache_key = "%s_private_configs" % bucket cached_value = get_context_value_from_request(cache_key) if cached_value is not None: return cached_value configs = hget("_bucket_private_configs", bucket) if not isinstance(configs, dict): configs = {} set_context_value_from_request(cache_key, configs) return configs
def get_post_visits_count(doc, field='visits'): # field is in ['visits', 'visitors'] bucket = get_bucket_in_request_context() if not bucket: return 0 visits_db_name = get_visits_db_name_for_bucket(bucket) doc_path = doc.get('path') if not doc_path: return 0 key = get_visits_key(doc_path, field=field) count = hget(visits_db_name, key) or 0 count = to_int(count, default_if_fail=0) return count
def register_bucket_domain_from_system(bucket, domain, is_admin=False): # 注册系统提供的二级域名 # 一个 bucket 只允许注册一个系统提供的二级域名 # 返回 None or 错误信息 domain = domain.strip().lower() if not is_valid_bucket_name(bucket): return 'invalid bucket' domain_info = get_domain_basic_info(domain, is_admin=is_admin) is_system_domain = domain_info.get('is_system_domain', False) is_allowed = domain_info.get('allowed', False) if is_system_domain and is_allowed: r_domain_info = hget('_rdomain', bucket) # 当前 bucket 是否已经域名绑定了 parked_domain_info = hget('_domain', domain) # 这个域名是否已经被其它 bucket 绑定了 if parked_domain_info: # domain 已经注册过了 if parked_domain_info.get('bucket') == bucket: return None return '%s is used by other bucket' % domain if r_domain_info: # bucket 已经绑定过 domain 了 master_old_domain = r_domain_info.get('domain') if master_old_domain == domain: return None else: # 一个 bucket 只能绑定一个系统的二级域名,就会删除之前的一个 hdel('_rdomain', bucket) hdel('_domain', master_old_domain) pull_domain_from_bucket(bucket, master_old_domain) # 汇总 domains # 一个 _rdomain, 作为一个反向的对应, 一个 bucket 只能有一个系统级的域名 ? domain_doc = dict(bucket=bucket, domain=domain, created_at=time.time()) hset('_domain', domain, domain_doc) hset('_rdomain', bucket, domain_doc) push_domain_to_bucket(bucket, domain) # 汇总 domains return None else: return '%s is not allowed for bucket:%s' % (domain, bucket)
def get_bucket_status(bucket): status_info = {} size = get_bucket_size(bucket) status_info['size'] = size last_updated_at = zget('buckets', bucket) if last_updated_at: status_info['date'] = db_timestamp_to_date_string(last_updated_at) max_record_id = hget('_bucket_max_id', bucket) delta_record_id = hget('_bucket_delta_id', bucket) status_info['max_record_id'] = max_record_id status_info['delta_record_id'] = delta_record_id status_info['max_record_date'] = record_id_to_date_string(max_record_id) status_info['delta_record_date'] = record_id_to_date_string(delta_record_id) try: usage = int(hget('_bucket_usage', bucket) or 0) except: usage = 0 status_info['usage'] = usage status_info['usage_for_human'] = bytes2human(usage) if usage and size: usage_per_record = round(usage/float(size), 2) status_info['usage_per_record'] = usage_per_record status_info['usage_per_record_for_human'] = bytes2human(usage_per_record) return status_info
def get_record_by_path(bucket, path, force_dict=False): if not bucket: return None if not path: return None if not isinstance(path, string_types): return None record_object_id = get_record_id_by_path(bucket, path) if record_object_id: record = hget(bucket, record_object_id, force_dict=force_dict) if isinstance(record, dict) and not record.get("_id"): record["_id"] = record_object_id else: record = None return record
def get_bucket_from_domain(domain): if not domain: return domain = smart_unicode(domain) domain = domain.strip().lower() if ADMIN_BUCKET and domain in WEBSITE_DOMAINS: return ADMIN_BUCKET maybe_domains = [domain] if domain.startswith('www.'): maybe_domains.append(domain.replace('www.', '', 1)) else: maybe_domains.append('www.%s' % domain) for d in maybe_domains: db_domain_info = hget('_domain', d) if db_domain_info and isinstance(db_domain_info, dict): return db_domain_info.get('bucket')
def show_avatar(avatar_id): avatar_id = get_avatar_id(avatar_id) avatar_doc = hget('_avatar', avatar_id) now = time.time() if avatar_doc: avatar_date = avatar_doc.get('date') avatar_image_content = avatar_doc.get('content') to_clear = False if not avatar_date: to_clear = True elif (now - avatar_date) > 5 * 24 * 60 * 60: # 5days to_clear = True elif ( now - avatar_date ) > 1 * 24 * 60 * 60 and not avatar_image_content: # 1day for empty avatar image to_clear = True if to_clear: # avatar_doc 缓存 5 天 hdel('_avatar', avatar_id) avatar_doc = None if not avatar_doc: avatar_image_content = get_gavatar_image_content(avatar_id) or '' if avatar_image_content: avatar_image_content = base64.b64encode(avatar_image_content) avatar_doc = dict(date=now, content=avatar_image_content) hset('_avatar', avatar_id, avatar_doc) if not is_doc_modified(avatar_doc, date_field='date'): return get_304_response() else: avatar_image_content = avatar_doc.get('content') or '' if avatar_image_content: avatar_image_content = base64.b64decode(avatar_image_content) response = Response(avatar_image_content, mimetype='image/png') set_304_response_for_doc(avatar_doc, response, date_field='date') return response else: # 默认的 url r_response = send_static_file('defaults/avatar.png') if r_response: return r_response # at last abort(404)
def delete_bucket_domain(domain, bucket=None): # 删除 bucket 上的 domain, 支持独立域名以及系统二级域名 # 这里未做是否有权限 unregister 的校验, 需要前置校验 # 未指定 bucket,就会删除 domain 对应的唯一 bucket;指定了,则进行 bucket 一致性校验 # 返回 None or 错误信息 domain = domain.strip().lower() if bucket and not is_valid_bucket_name(bucket): return 'invalid bucket' domain_record = hget('_domain', domain) if domain_record: to_delete = False bucket_in_db = domain_record.get('bucket') if bucket and bucket_in_db == bucket: to_delete = True elif not bucket and bucket_in_db: to_delete = True if to_delete: hdel('_domain', domain) hdel('_rdomain', bucket_in_db) # 尝试删除 rdomain 上的信息, 这样下次 bucket 还能再注册一个系统二级域名 pull_domain_from_bucket(bucket_in_db, domain) # 汇总 domains return None return 'can not find the matched domain record to remove for %s' % domain
def register_bucket_independent_domain(bucket, domain): # 注册独立域名, 这个前提是域名已经 park 到当前节点,并且已经做了必要的校验 # 返回 None or 错误信息 domain = domain.strip().lower() if not is_valid_domain(domain): return 'domain format error or not supported' if not is_valid_bucket_name(bucket): return 'invalid bucket' if not has_bucket(bucket): return 'current node does not have bucket:%s' % bucket bucket = bucket.strip() old_domain_info = hget('_domain', domain) or {} old_matched_bucket = old_domain_info.get('bucket') if old_matched_bucket == bucket: # # 已经注册过了,不做处理 return None #return 'registered already for this bucket, no need to change' if domain == "thisisadomainfortest.com": # for test only bucket_in_domain_text = bucket else: bucket_in_domain_text = get_domain_text_record(domain) if bucket_in_domain_text == bucket: # 比如 A 曾经注册过,后来 domain 是 B 的了,那么 B 修改了 TXT 记录,就可以重新注册了。 # at last, create or modify hset('_domain', domain, dict( bucket=bucket, created_at=time.time(), )) push_domain_to_bucket(bucket, domain) # 汇总 domains return None # done else: if bucket_in_domain_text: if not is_valid_bucket_name(bucket_in_domain_text): return 'invalid bucket format in TXT record: %s' % bucket_in_domain_text else: return 'TXT record is not matched to %s' % bucket else: return 'should set TXT record for domain first'
def get_records_for_bucket(bucket, start_record_id=None, end_record_id=None, limit=1000, includes_start_record_id=False, reverse_scan=False, raw=False): # start_record_id means start here but does not include itself # 在两个地方用到: url 中 list bucket 的 & 模板引擎中调用 records 的 # 模板引擎中调用的话,一般都会指定 start_record_id,避免一些 zero_ids 在实际呈现无意义的数据被显示出来 # 一般是不 includes_start_record_id, 上一次获得的 list 的最后一个 record 的 id 会作为 cursor (start_id), # 也就是说 start_id 不应该在当前获取的 list 内 records = hscan(bucket, key_start=start_record_id, key_end=end_record_id, limit=limit, reverse_scan=reverse_scan) if includes_start_record_id and start_record_id: start_record = hget(bucket, start_record_id) if start_record: records.insert(0, [start_record_id, start_record]) if not raw: records = to_py_records_from_raw_ssdb_records(records) return records
def get_bucket_by_wechat_user_id(wechat_user_id): if not isinstance(wechat_user_id, string_types): return None return hget("wechat_accounts", wechat_user_id)
def get_name_by_wechat_user_id(wechat_user_id): if not isinstance(wechat_user_id, string_types): return None return hget("wechat_names", wechat_user_id)
def get_invitation(code): # 邀请码是 code 也是 id record = hget('_bucket_invite', code) or {} return record
def get_buckets_cursor_for_remote_node(node): cursor = hget('_remote_buckets_cursor', node) or '' return cursor
def get_simple_bucket_token(bucket): token = hget('_simple_bucket_token', bucket) or '' return token
def get_bucket_delta_id(bucket): delta_id = hget('_bucket_delta_id', bucket) or '' return delta_id
def get_bucket_max_id(bucket): max_id = hget('_bucket_max_id', bucket) or '' return max_id
def do_get_record_by_url(bucket, url_path): url_path = url_path.strip().lower() bucket_for_url = '%s_url' % bucket path_data_id = hget(bucket_for_url, url_path) if path_data_id: return get_record_by_path(bucket, path_data_id)
def get_bucket_service_info(bucket): if not bucket: return {} if not is_valid_bucket_name(bucket): return {} return hget("_bucket_info", bucket, force_dict=True)
def get_bucket_last_record_id_computed(bucket): record_id = hget("buckets_file_cursor_computed", bucket) or None return record_id
def allowed_to_create_record_in_bucket(bucket, record_md5): last_md5 = hget('_bucket_max_md5', bucket) if last_md5 and last_md5 == record_md5: return False else: return True