def get_json_content_by_path(bucket, path, force_dict=False):
raw_content = get_raw_content_by_path(bucket, path)
if not raw_content:
return {}
try:
result = json_loads(raw_content)
if force_dict and not isinstance(result, dict):
return {}
return result
except:
return {}
def from_json(obj):
# 将一个字符串，重新载入为一个dict/list等类型的py 数据
if not obj:
return obj
if not isinstance(obj, string_types) and hasattr(obj, 'core'):
# 可能是 Text/ Date 等wrap后的数据对象
obj = obj.core
if isinstance(obj, string_types):
try:
return json_loads(obj)
except:
return obj
return obj
def sync(self):
# one record only, push to server now
# return sync_status
json_data = self.json_data_for_sync
if not json_data:
return None # ignore
py_data = json_loads(json_data)
sync_status = self.send_message(action='create', message=json_data)
# 上传文件
if py_data.get('_type') in ['file', 'image']:
raw_file_data = py_data.get('raw_content')
if not raw_file_data:
# 直接上传文件, 如果 version 一样，或者 size 超了，都会返回 no 的结果
reply = self.send_message(action='should_upload_file',
message=json_data)
if reply.get('message') == 'yes' and os.path.isfile(
self.filepath):
file_size = os.path.getsize(self.filepath)
request_timeout = 120
if file_size:
file_mb = int(file_size / 1024. / 1024)
if file_mb > 2:
request_timeout = file_mb * 60 # 1mb 就是多 1 分钟的 timeout
request_timeout = min(request_timeout,
30 * 60) # 不能超过 30 分钟
with open(self.filepath, 'rb') as f:
sync_status = self.send_message(
action='upload_file',
message=json_data,
file_to_post=f,
timeout=request_timeout,
)
if DEBUG:
info = 'is_deleted=%s, sync %s, sync_status: %s' % (
self.is_deleted, self.relative_path, sync_status)
print(info)
return sync_status
def ssdb_data_to_py_data(ssdb_data, hit_cache=False):
# 为了避免从 ssdb 中获得数据，反复转为 python 中使用， 增加了 cache_id 的逻辑，以避免重复计算性能消耗的问题
if not isinstance(ssdb_data, string_types):
return ssdb_data
if hit_cache:
data_cache_key = to_md5(ssdb_data)
cached_value = ssdb_data_to_py_data_cache.get(data_cache_key)
if cached_value:
return cached_value
else:
data_cache_key = None
if re.match('\s*[\[\{\(]', ssdb_data): # dict list tuple
try:
py_data = json_loads(ssdb_data)
if data_cache_key:
ssdb_data_to_py_data_cache[data_cache_key] = py_data
except:
py_data = to_unicode(ssdb_data)
else:
py_data = to_unicode(ssdb_data)
return py_data
def get_compiler_data_directly(relative_path,
content=None,
is_deleted=False,
is_dir=False,
real_relative_path=None):
if not relative_path:
return
if content and len(content) > MAX_RECORD_SIZE:
# 避免不必要的内存占用，此时的 content 必然不会存储在 raw_content 这个字段中
content = None
compiler_worker = FarBoxSyncCompilerWorker(
server_node=None,
root=None,
filepath=None,
relative_path=relative_path,
raw_content=content,
is_deleted=is_deleted,
is_dir=is_dir,
real_relative_path=real_relative_path)
json_data = compiler_worker.json_data_for_sync
if json_data:
data = json_loads(json_data)
return data
def get_normal_data_by_simple_token(bucket,
encrypted_data,
force_py_data=True,
default_if_failed=None):
# force_py_data = True 的时候，必须是 list/tuple/dict 的数据类型
if not encrypted_data:
return encrypted_data
token = get_simple_bucket_token(bucket)
if token and isinstance(encrypted_data, string_types):
result = simple_decrypt(encrypted_data, token)
try:
result = json_loads(result)
except:
pass
else:
result = encrypted_data
if force_py_data:
if isinstance(result, (list, tuple, dict)):
return result
else:
return default_if_failed
else:
return result
def create_record(bucket,
record_data,
avoid_repeated=True,
auto_id=True,
file_content=None,
return_record=False):
# make sure the bucket is correct before create record
# 如果返回数据，就是 error_info
# avoid_repeated 就是避免跟最后一条数据 body 是一样的
error_info = get_record_data_error_info(record_data)
if error_info:
return error_info
py_record_data = ssdb_data_to_py_data(record_data)
byte_record_data = py_data_to_ssdb_data(record_data)
if auto_id:
object_id = str(ObjectId())
if '_id' not in py_record_data and isinstance(
py_record_data, dict): # record data 如有必要自动填充 _id
py_record_data['_id'] = object_id
else:
object_id = py_record_data.get('_id') or py_record_data.get('id')
avoid_repeated = False # 指定的 id 的，不做 repeated 的校验
if not object_id:
return 'auto_id disabled, should pass id in the record data'
if avoid_repeated: # 避免最后一条记录的重复
record_md5 = to_md5(byte_record_data)
if not allowed_to_create_record_in_bucket(bucket, record_md5):
error_info = 'current data is repeated to latest record @%s' % bucket
if isinstance(py_record_data, dict):
path_in_record = py_record_data.get('path')
if path_in_record:
error_info += smart_unicode(', the path is %s' %
path_in_record)
return error_info
else:
update_bucket_last_record_md5(bucket, record_md5)
# '_auto_clean_bucket' in record_data and is `True`
# 如果是 delete 的直接删除 （break），反之则是完全的 update，相当于新的 record 代替 旧的 record
auto_clean_status = auto_clean_record_before_handle_path_related_record(
bucket, py_record_data)
if auto_clean_status == 'break':
return
# store pre_object_id
# 获得上一个对象的 id， 将当前的 data 转为 dict （如果是），存储 _pre_id 这个字段
pre_object_id = get_bucket_max_id(bucket)
if pre_object_id:
if isinstance(py_record_data, dict):
py_record_data['_pre_id'] = pre_object_id
# 存储 record， 并且更新 bucket 上 max_id 的信息
# 由于 record_id 是随机生成，本质上不会重复，故 ignore_if_exists=False, 避免一次校验的过程
hset(bucket, object_id, py_record_data, ignore_if_exists=False)
after_record_created(bucket,
py_record_data,
object_id=object_id,
should_update_bucket_max_id=True)
# 更新 buckets 的信息，表示当前 bucket 刚刚被更新过了
set_bucket_into_buckets(bucket)
if py_record_data.get("path"):
# path 相关的，因为有 delete 的关系，单独进行 last_record_id 的存储，不然直接 hget_max 就可以了
set_bucket_last_record_id(bucket, object_id)
if file_content and not py_record_data.get("raw_content"):
# 指定了要存储的 file content，并且 record 中并没有 raw_content 这个字段，进行文件的存储
storage.accept_upload_file_from_client(
bucket, py_record_data, get_raw_content_func=file_content)
if py_record_data.get("path") == "settings.json" and py_record_data.get(
"raw_content"):
try:
site_settings = json_loads(py_record_data.get("raw_content"))
if isinstance(site_settings, dict):
set_bucket_configs(bucket, site_settings, config_type="site")
except:
pass
if return_record:
return py_record_data
