def get_pay_request_data(self,
price,
title,
content,
notify_url='',
return_url='',
out_trade_no='',
note=''):
price = to_float(price)
if not price:
return # ignore
price = '%.2f' % price
out_trade_no = smart_unicode(out_trade_no)
if len(out_trade_no) < 10: # 作为前缀，自动生成
out_trade_no = self.get_out_trade_no(prefix=out_trade_no)
# 不限制 subject、 body， 反正太长了自己会出错
data = dict(
total_amount=price,
subject=smart_unicode(title), # max length 256
body=smart_unicode(content), # max length 128
out_trade_no=out_trade_no,
)
if note:
data['passback_params'] = note
if notify_url: # alipay 会进行通知
data['notify_url'] = notify_url
if return_url: # url 跳转回来，成功支付后
data['return_url'] = return_url
return data
def pay(self,
price,
title,
content,
callback_url='',
notify_url='',
return_url='',
order_id='',
note=''):
if not self.alipay_api:
return
notify_url = notify_url or callback_url
return_url = return_url or callback_url
price = to_float(price)
if not price or price < 0.01:
price = 0.01
redirect_url = self.alipay_api.pay(
price,
title,
content,
notify_url=notify_url,
return_url=return_url,
out_trade_no=order_id,
note=note,
)
return redirect_url
def get_paths_by_type(bucket,
data_type,
offset=0,
limit=10000,
reverse=False,
prefix_to_ignore=None,
date_start=None,
date_end=None):
paths = []
if not bucket:
return []
site_configs = get_bucket_site_configs(bucket)
utc_offset = to_float(site_configs.get('utc_offset', 8), default_if_fail=8)
if date_start:
date_start = date_to_timestamp(to_date(date_start),
utc_offset=utc_offset)
if date_end:
date_end = date_to_timestamp(to_date(date_end), utc_offset=utc_offset)
should_hit_date = False
if date_start or date_end:
should_hit_date = True
if should_hit_date and not prefix_to_ignore:
prefix_to_ignore = '_'
if prefix_to_ignore:
prefix_to_ignore = smart_unicode(prefix_to_ignore)
order_bucket = get_bucket_name_for_order(bucket, data_type)
result = zrange(order_bucket, offset=offset, limit=limit, reverse=reverse)
for path, order_value in result:
if should_hit_date:
order_value = to_float(order_value)
if order_value < 10000000:
continue
if date_start and order_value < date_start:
continue
if date_end and order_value > date_end:
continue
path = smart_unicode(path)
if prefix_to_ignore and path.startswith(prefix_to_ignore):
continue
paths.append(path)
return paths
def post_order_value(self):
# 手工设定的 position， 统一扩大 1000 倍，避免一些浮点数在某些场景下被 int 处理，而颗粒度失真
order_fields = ['sort', 'order', 'position']
for field in order_fields:
order_value = to_float(self.metadata.get(field))
if order_value is not None:
# 统一扩大 1000 倍
return order_value * 1000
# at last, choose date timestamp as order
order_value = self.post_timestamp
return order_value
def utc_date_parse(timestr, parserinfo=None, utc_offset=None, **kwargs):
# 转为 datetime，但已经偏移为 utc 的时间了
if utc_offset is None:
utc_offset = get_local_utc_offset()
if isinstance(utc_offset, string_types):
utc_offset = to_float(utc_offset, 8)
if not isinstance(timestr, string_types): #非字符串的，不处理
#if hasattr(timestr, 'tzinfo') and timestr.tzinfo is None: # 时间格式且无时区
# timestr -= datetime.timedelta(0, utc_offset*3600) # 调整时区偏差
# 不需要调整时区，但这种直接传入的date，需确保是utcnow()获取的
return timestr
else:
timestr = timestr.strip()
if not timestr: # 空字符不处理
return timestr
# 网易邮箱的 Date...
# Fri, 28 Apr 2017 20:09:08 +0800 (GMT+08:00)
if ' +' in timestr:
p1, p2 = timestr.split(' +', 1)
timezone_str = p2.split(' ')[0]
timestr = '%s +%s' % (p1, timezone_str)
if isinstance(timestr, unicode):
timestr = timestr.replace(u'\uff1a', ':')
date = parse(timestr, parserinfo, **kwargs)
date_offset_done = False
if date.tzinfo is not None: # 保证数据有效性
try:
diff = date.utcoffset()
date = date.replace(tzinfo=None)
date = date - diff
date_offset_done = True
except ValueError: # 比如exif中的信息有错，则不处理；但是mongodb又会转化，所以，要保证数据格式可用
date = date.replace(tzinfo=None)
# utc_offset now
if date.tzinfo is None and not date_offset_done: # 如果不包含时区信息，则偏移，比如dropbox api提供的都是有时区信息的
try:
date -= datetime.timedelta(0, utc_offset * 3600)
except:
# 囧, 可能会导致 date 的时间工差 <0 ....
pass
return date
def dump_comments_to_csv_content(parent_obj_doc, comments):
comment_keys = ['author', 'content', 'email', 'site', 'date', 'ip', 'reply']
utc_offset = to_float(parent_obj_doc.get('_utc_offset'), default_if_fail=8)
#doc_path = to_doc_path(parent_obj_doc)
#comments_path = doc_path_to_comments_path(doc_path) # 评论的文档存储路径
csv_records = [comment_keys]
for comment in comments:
csv_record = []
for key in comment_keys:
value = comment.get(key) or ''
if isinstance(value, datetime.datetime):
value = value + datetime.timedelta(0, utc_offset * 3600)
value = value.strftime("%Y-%m-%d %H:%M:%S")
csv_record.append(value)
csv_records.append(csv_record)
csv_content = dump_csv(csv_records)
return csv_content
def update_record_order_value_to_related_db(bucket, record_data, force_value=None):
# 设定排序, 如果没有排序逻辑的，实际上根据 get_data(type) 的逻辑是无法取出内容的
path = get_path_from_record(record_data)
if not path:
return
path = path.strip('/')
if not path:
return
bucket_name_for_order = get_bucket_name_for_order_by_record(bucket, record_data)
data_type = get_data_type(record_data)
data_order = record_data.get("_order") or record_data.get("order")
if not data_order and data_type in ["file", "post", "folder"]: # 不应该出现的情况
data_order = time.time()
if data_order is not None:
data_order = to_float(data_order, default_if_fail=None)
if force_value is not None:
data_order = force_value
if data_order is not None and bucket_name_for_order:
zset(bucket_name_for_order, path, data_order)
def __init__(self):
self.ip = get_visitor_ip()
self.email = safe_get('email')
self.site = safe_get('site') # url, not just domain
self.domain = request.form.get('site', '').lower().strip().replace(
'http://', '').replace('https://', '').strip('/')
self.parent_path = request.values.get('path', '') # 所评论的文档路径
self.utc_offset = to_float(self.parent_obj.get('_utc_offset'),
default_if_fail=8)
self.now = datetime.datetime.utcnow() + datetime.timedelta(
0, self.utc_offset * 3600)
self.date = self.now.strftime(
"%Y-%m-%d %H:%M:%S") # 用来构建 comment_id 的逻辑
self.raw_comment_content = smart_unicode(
request.form.get('content', '')) # 去掉html tag
self.content = self.raw_comment_content[:1000].strip() # 评论最多1k字
self.error_info = ''
def get_comments(parent_doc, bucket=None, as_tree=None):
bucket = bucket or get_bucket_in_request_context() or request.values.get('bucket')
if not bucket:
return []
path = to_doc_path(parent_doc)
comments_doc = get_comments_record(bucket, path)
site_configs = get_bucket_site_configs(bucket)
if not get_value_from_data(site_configs, "comments", default=True):
return []
if as_tree is None: # 自动匹配, 网站设置中对应
comments_type = get_value_from_data(site_configs, 'comments_type') or 'tree'
if comments_type in ['tree']:
as_tree = True
else:
as_tree = False
utc_offset = to_float(parent_doc.get('_utc_offset'), 8)
return get_comments_by_comments_doc(comments_doc, as_tree=as_tree, utc_offset=utc_offset)
def get_grid_factor(k, base=24, odd_base=5):
# 一般有两套，比如 1/5 就对应不到 24 中
k = special_grid_factors.get(k) or k
if not k:
return k, base
if isinstance(k, (str, unicode)):
if '-' in k or '/' in k:
v1, v2 = re.split(r'[-/]', k, maxsplit=1)
v1 = to_int(v1)
v2 = to_int(v2)
if v1 and v2:
small_one = min(v1, v2)
big_one = max(v1, v2)
if big_one == odd_base:
base = odd_base
k = float(small_one) / float(big_one)
elif '.' in k:
k = to_float(k)
else: # 整数
k = to_int(k) or 1
if k and isinstance(k, int) and k > 1:
k = float(k) / base
if isinstance(k, (float, int)) and k <= 1:
# 之前全部处理为分数了, 这样 1 就是全值了
k *= base
# 处理最终的 k 值
k = to_int(round(k)) or base # 四舍五入并 int
if k > base:
k = base
if k < 1:
k = 1
return k, base
def update_post_visits_for_response(response):
doc = get_doc_in_request()
if doc:
doc_type = doc.get('_type')
doc_path = doc.get('path')
else:
doc_type = get_doc_type_in_request()
doc_path = get_doc_path_in_request()
bucket = get_bucket_in_request_context()
if bucket and doc_type == 'post' and doc_path:
pass
else:
return
visits_key = get_visits_key(doc_path, field='visits')
visitors_key = get_visits_key(doc_path, field='visitors')
now = time.time()
# 直接在这里处理cookie
# set cookie to know it's not a new visitor
response.set_cookie('last_visited_at', str(now), max_age=one_year_seconds)
last_visited_at = to_float(request.cookies.get('last_visited_at', 0)) or 0
diff = now - last_visited_at
if diff > 24 * 60 * 60: # 24小时内的，认为是同一个visitor
is_visitor = True
else:
is_visitor = False
# 异步更新到数据库
spawn(async_update_visits,
bucket,
visits_key,
visitors_key,
is_visitor=is_visitor)
def float(self):
# 类似 int 的用法
return to_float(self.core)
ADMIN_BUCKET = get_env('ADMIN_BUCKET') or ''
DEBUG = bool(get_env('DEBUG'))
WEBSOCKET = bool(get_env('WEBSOCKET'))
# 允许服务的提供者，直接使用 py 脚本进行处理
bucket_scripts_root = '/mt/web/configs/bucket_scripts'
if DEBUG:
bucket_scripts_root = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'test_bucket_scripts')
if os.path.isdir(bucket_scripts_root):
sys.path.append(bucket_scripts_root)
BUCKET_PRICE = to_float(get_env("bucket_price"), default_if_fail=128) or 0
BUCKET_PRICE2 = to_float(get_env("bucket_price2"), default_if_fail=0) or 0
def get_domains_from_env(key):
raw_domains = get_env(key) or ''
domain_list = re.split('[,\n]', raw_domains)
domains_got = []
for domain in domain_list:
domain = domain.strip().lower()
if domain not in domains_got and domain:
domains_got.append(domain)
return domains_got
SYSTEM_DOMAINS = get_domains_from_env('DOMAINS')
def get_bucket_utc_offset(bucket=None):
bucket = bucket or get_bucket_in_request_context()
site_configs = get_bucket_site_configs(bucket) or {}
utc_offset = to_float(site_configs.get('utc_offset', 8), default_if_fail=8)
return utc_offset
def to_form_fields_obj(data_obj, keys, formats=None, extra_handler_func=None):
# 将一个 dict, 预处理为可HTML 格式进行编辑的数据，一般是处理 json 的可编辑性
field_objs_list = []
formats = formats or {}
if not isinstance(formats, dict): # 必须是dict类型
formats = {}
if basic_form_formats:
new_formats = basic_form_formats.copy()
new_formats.update(formats)
formats = new_formats
if not isinstance(keys, (tuple, list)):
return field_objs_list
for key in keys:
# 先提取 placeholder, 反引号的方式（头尾边界比较容易判断）
# placeholder也可能用作他用，比如select的options
if isinstance(key, dict): # 直接传入的是 dict 类型,不需要特别处理, 但是比较少见
field_objs_list.append(key)
continue
key = smart_unicode(key) # 必须是 unicode
placeholder = ''
p_c = re.search('`(.*?)`', key)
if p_c and 'form_keys=' not in key:
# form_keys= 不处理 ```, 因为其子元素需要这些信息
placeholder = p_c.group(1)
key = re.sub(r'`.*?`', '', key)
key, extra_info = extract_extra_info(
key) # extra_info 是从单个key的括号内提取的内容
if 'placeholder' in extra_info:
placeholder = extra_info['placeholder']
field_type = extra_info.get('type') or '' # 默认, 可能没有提取信息来（单行的话）
if '@' in key: # 使用@方式，优先级更高
key, field_type = key.rsplit('@', 1)
if not field_type and 'password' in key:
field_type = 'password'
# key_matched_data 实际上就是formats
key_matched_data = formats.get(key) or {} # 先处理 format
if not isinstance(key_matched_data, dict):
key_matched_data = {}
key_matched_data = key_matched_data.copy() # copy 避免产生不必要的混乱
key_matched_data['key'] = key
field_type = field_type or key_matched_data.get(
'type') # 原先没有定义 field_type, 从 formats中获得
if placeholder:
key_matched_data['placeholder'] = placeholder
if '.' in key:
key_title = key.split('.')[-1].replace('_', ' ').title()
else:
key_title = key.replace('_', ' ').title()
if 'Id' in key_title:
key_title = re.sub(r'( |^)(Id)( |$)', '\g<1>ID\g<3>', key_title)
key_matched_data['title'] = key_matched_data.get('title') or key_title
# set value
default_value = extra_info.get("default") # 默认值
if default_value is None:
default_value = key_matched_data.get('default', '')
if default_value is None: default_value = ''
value = get_value_from_data(data_obj, key, default=default_value)
if default_value and isinstance(default_value,
string_types) and value == '':
# 有值，但是空字符串的时候，使用 default_value 来处理
value = default_value
if value is None:
value = ''
key_matched_data['value'] = value
if field_type == 'timezone':
field_type = 'select'
key_without_dot = key.split('.')[-1]
if field_type == 'select' and key_without_dot in DEFAULT_OPTIONS:
key_matched_data['options'] = DEFAULT_OPTIONS[key_without_dot]
# 类型转换
if field_type == 'bool': # bool 转为 select 类型
field_type = 'select'
key_matched_data['options'] = [('yes', 'Yes'), ('no', 'No')]
key_matched_data['value'] = 'yes' if key_matched_data.get(
'value') else 'no'
elif field_type == 'select':
if placeholder and not key_matched_data.get('options'):
# 通过placeholder 计算 options
option_values = [
value.strip() for value in placeholder.split(',')
if value.strip()
]
if not extra_info.get('index_value', True): # value不为索引
key_matched_data['options'] = [(v, v.replace('_', " "))
for v in option_values]
else: # value为索引值
# display_text@key
options = []
for i, v in enumerate(option_values):
if isinstance(v, (str, unicode)):
v = v.strip()
option = (i + 1, v)
if v.endswith('@'): # 相当于 显示的内容，即 value本身
v = v[:-1] # remove @
if isinstance(v, string_types) and len(
v) < 100 and re.match(
r'[.a-z0-9+_-]+$', v, flags=re.I):
option = (v, v)
if isinstance(v, string_types) and '@' in v:
display_text, k = v.split('@', 1)
option = (k.strip(), display_text.strip())
options.append(option)
key_matched_data['options'] = options
elif not key_matched_data.get('options') and isinstance(
value, (list, tuple)):
# 直接从 list 类型 的 value 中提取
options = []
for row in value:
if isinstance(row, string_types):
options.append([row, row])
key_matched_data['options'] = options
# value 可能是整数，就先转为 int、 float 的类型
if isinstance(value, string_types):
if re.match(r'\d+$', value):
value = int(value)
key_matched_data['value'] = value
elif re.match(r'\d+\.\d+$', value):
value = to_float(value)
key_matched_data['value'] = value
elif field_type == 'list': # list 的转为 text，可以用 textarea 来渲染
key_matched_data[
'key'] = key + '@list' # 这样 backend 在重新处置这个字段的时候，会转为 list 的类型
field_type = 'text' # text == textarea
old_value = key_matched_data.get('value')
if isinstance(old_value, (list, tuple)):
value = '\n'.join(old_value)
key_matched_data['value'] = value
elif field_type == 'file':
# 将 placeholder 的内容取出来作为 filepath
key_matched_data[
'placeholder'] = 'drag file here to upload/replace'
# 额外的扩充， 由程序的逻辑控制
if extra_handler_func and hasattr(extra_handler_func, '__call__'):
field_type, key_matched_data = extra_handler_func(
field_type, key_matched_data)
if extra_info:
key_matched_data.update(extra_info)
# 设定了dom的固定宽度、高度
for w_h_field in ['height', 'width']:
w_h_value = key_matched_data.get(w_h_field)
if w_h_value:
w_h_value = smart_unicode(w_h_value)[:30].strip()
if re.match('[\d.]+$', w_h_value):
w_h_value += 'px'
key_matched_data[w_h_field] = w_h_value
# 某些file_type 比如 category / list 最后都转为HTML类型的field_type
if field_type:
key_matched_data['type'] = field_type
else:
key_matched_data['type'] = 'default'
if key_matched_data.get('type') == 'list': # list 类型的value的处理
if isinstance(key_matched_data.get('value', None), (list, tuple)):
key_matched_data['value'] = '\n'.join(
key_matched_data['value'])
field_objs_list.append(key_matched_data)
return field_objs_list
def __float(obj):
if isinstance(obj, (int, float, str, unicode)):
return to_float(obj, default_if_fail=int)
return obj