def do_record_sync_log(self, log):
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
log = to_bytes('%s %s\n\n' % (now, log))
sync_log_filepath = join(self.root, '.sync/%s_sync.log' % self.app_name_for_sync)
try:
make_sure_path(sync_log_filepath)
with open(sync_log_filepath, 'a') as f:
f.write(log)
except:
pass
def store_sync_from_log(root, log):
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
log = smart_str('%s %s\n\n' % (now, log))
sync_log_filepath = join(root, '.sync/farbox_sync_from.log')
try:
make_sure_path(sync_log_filepath)
with open(sync_log_filepath, 'a') as f:
f.write(log)
except:
pass
def store_nginx_server_cert(ssl_key, ssl_cert):
if not ssl_key or not ssl_cert:
return
old_ssl_key = read_file(app_nginx_server_ssl_key_filepath)
old_ssl_cert = read_file(app_nginx_server_ssl_cert_filepath)
if old_ssl_key != ssl_key or old_ssl_cert != ssl_cert:
make_sure_path(app_nginx_server_ssl_cert_filepath, is_file=True)
write_file(app_nginx_server_ssl_key_filepath, ssl_key)
write_file(app_nginx_server_ssl_cert_filepath, ssl_cert)
# reload nginx
c_f = os.popen("/usr/nginx/sbin/nginx -s reload")
try:
c_f.read()
except:
pass
def sync_from_farbox(root,
private_key,
node,
get_cursor_func=None,
save_cursor_func=None,
before_file_sync_func=None,
after_file_sync_func=None,
per_page=30):
get_cursor_func = get_cursor_func or partial(default_get_cursor_func, root)
save_cursor_func = save_cursor_func or partial(default_set_cursor_func,
root)
cursor = get_cursor_func()
message = dict(per_page=per_page)
if cursor:
message["cursor"] = cursor
records = send_message(node,
private_key,
action="show_records",
message=message)
if not isinstance(records, (list, tuple)):
if settings.DEBUG:
print("error:records from node is not list/tuple type")
return
#if settings.DEBUG:
# print("get %s records from node" % len(records))
last_cursor = None
error_happened = False
will_continue = False
if len(records) == per_page:
will_continue = True
for record in records:
if not isinstance(record, dict):
continue
record_id = record.get("_id")
server_side_file_version = record.get("version")
last_cursor = record_id
path = record.get("path")
if not record_id or not path:
continue
is_dir = record.get("is_dir")
if is_dir:
continue
is_deleted = record.get("is_deleted")
if is_deleted:
# 为了避免误删除，忽略 is_deleted 的逻辑
continue
abs_filepath = join(root, path)
if server_side_file_version and os.path.isfile(abs_filepath):
if get_md5_for_file(abs_filepath) == server_side_file_version:
# 文件已经存在且重复了
#if settings.DEBUG:
#print("has same file on server side for %s" % abs_filepath)
continue
# 开始下载文件
# 302 的跳转会自动处理，从而获得 200 的最终结果
response = send_message(node,
private_key,
action="download_file",
message=dict(record_id=record_id),
timeout=120,
return_response=True)
if not response:
error_happened = True
continue
if response.status_code == 404: # 404 就直接忽略
continue
elif response.status_code not in [200, 201]:
error_happened = True
continue
raw_file_content = response.content
if not raw_file_content:
continue
# 存储前的 hook, 比如做版本的存储
if before_file_sync_func and hasattr(before_file_sync_func,
"__call__"):
before_file_sync_func(abs_filepath)
# 如果文件已经存在，保存之前，先放到回收站了，给用户多一个反悔的可能；另外 Windows 上 send2trash 并不总是正确的，except 就直接 pass
if os.path.isfile(abs_filepath) and send2trash is not None:
try:
send2trash.send2trash(abs_filepath)
except:
pass
try:
make_sure_path(abs_filepath)
with open(abs_filepath, "wb") as f:
f.write(smart_str(raw_file_content))
if settings.DEBUG:
print("downloaded %s" % abs_filepath)
except:
if settings.DEBUG:
print_error()
error_happened = True
# 存储后的 hook
if after_file_sync_func and hasattr(after_file_sync_func, "__call__"):
after_file_sync_func(abs_filepath)
# 存储日志
store_sync_from_log(root, abs_filepath)
if not error_happened and last_cursor:
# 没有错误发生，才会保存 cursor，确保同步的数据尽可能保持一致性
cursor_saved = save_cursor_func(last_cursor)
if cursor_saved and will_continue:
# 继续调用
sync_from_farbox(root=root,
node=node,
private_key=private_key,
get_cursor_func=get_cursor_func,
save_cursor_func=save_cursor_func,
before_file_sync_func=before_file_sync_func,
after_file_sync_func=after_file_sync_func,
per_page=per_page)
else:
if cursor_saved:
store_sync_from_log(root, "records updated")
else:
store_sync_from_log(root, "sync finished, no need to update")
if settings.DEBUG:
print("sync-from finished, %s records" % len(records))