def get_sync_cursor(root):
sync_folder = join(root, '.sync')
cursor_path = join(sync_folder, '%s.cursor' % app_name)
if os.path.isfile(cursor_path):
with open(cursor_path) as f:
return f.read()
return ''
def delete_sync_cursor(root):
sync_folder = join(root, '.sync')
cursor_path = join(sync_folder, '%s.cursor' % app_name)
try:
delete_file(cursor_path, to_trash=False)
except:
pass
def show_info(site_folder):
site_token = get_site_token(site_folder)
print_with_color('Version: %s' % version, 'green')
if not site_token:
print_with_color('current folder is not a site of Bitcron', 'red')
else:
print_with_color(
'Site Token: %s (do not tell anyone else!)' % site_token,
'magenta')
site_sync_configs = get_sync_configs(site_folder, site_token)
if site_sync_configs:
domain = site_sync_configs.get('domain')
main_node = site_sync_configs.get('main_node')
if domain:
print_with_color('Domain: %s' % domain, 'green')
if main_node:
print_with_color('MainNode: %s' % main_node, 'yellow')
site_node_config_filepath = join(site_folder, site_node_filename)
try:
with open(site_node_config_filepath) as f:
user_node_info = f.read().strip().strip('/').split('/')[-1]
if user_node_info:
print_with_color(
'MyNode: %s (current server)' % user_node_info,
'cyan')
except:
pass
def get_sync_configs(root, site_token):
site_node_config_filepath = join(root, site_node_filename)
user_site_node = None
try:
with open(site_node_config_filepath) as f:
user_site_node = f.read().strip()
user_site_node = user_site_node.strip('/').split('/')[-1]
except:
pass
site_info = get_sync_site_info(site_token)
if site_info:
configs = {}
main_node = site_info.get('main_node')
site_domain = site_info.get('domain')
url_fields = [
'sync_url', 'sync_list_url', 'sync_content_url', 'sync_should_url'
]
for url_field in url_fields:
url = site_info.get(url_field)
if not url:
return {}
if user_site_node:
url = url.replace(main_node, user_site_node)
configs[url_field] = url
configs['domain'] = site_domain
configs['main_node'] = main_node
return configs
return {}
def make_sure_sync_data_folder(root):
sync_folder = join(root, '.sync')
data_folder = get_sync_data_folder(root)
if not os.path.isdir(data_folder):
os.makedirs(data_folder)
hide_a_path(sync_folder)
return data_folder
def delete_site_folder_token(site_folder):
if not os.path.isdir(site_folder):
return # ignore
site_token_filepath = join(site_folder, site_token_filename)
try:
os.remove(site_token_filepath)
except:
pass
def get_site_token(site_folder):
if not site_folder:
return
token_filepath = join(site_folder, site_token_filename)
if os.path.isfile(token_filepath):
try:
with open(token_filepath) as f:
token_content = f.read().strip()
if len(token_content) == site_token_length:
return token_content
except:
pass
def find_files_to_delete(root_path):
sync_data_folder = get_sync_data_folder(root_path)
if not os.path.isdir(sync_data_folder): # never synced before
return []
files = loop_local_filesystem(root_path,
check_md5=False) # same_path already
data_filenames = os.listdir(sync_data_folder)
old_file_paths = []
for data_filename in data_filenames:
data_filepath = join(sync_data_folder, data_filename)
try:
with open(data_filepath) as f:
data = json.loads(f.read())
filepath = data.get('filepath')
if data.get('is_relative'):
filepath = join(root_path, filepath)
if filepath:
old_file_paths.append(same_slash(filepath))
except:
pass
return list(set(old_file_paths) - set(files))
def get_sync_data(filepath, root):
# get the synced information for a filepath
# 根据文件的路径，获得对应 md5 文件，里面存储了必要的信息（md5 * synced_at），用于判断当前文件是否需要同步
filepath = same_slash(filepath)
data_path = get_sync_data_filepath(filepath, root)
if os.path.isfile(data_path):
try:
with open(data_path) as f:
data = json.loads(f.read())
if data.get('is_relative'):
# 相对路径，转为绝对路径
data['filepath'] = join(root, data['filepath'])
if isinstance(data, dict):
return data
except:
pass
return {} # final
def loop_local_filesystem(root_path, check_md5=True):
root_path = same_slash(root_path)
if not os.path.isdir(root_path): # 根目录不存在，不处理
return []
file_paths = []
for parent, folders, files in os.walk(root_path):
if is_a_hidden_path(parent):
continue
elif not is_real(parent): # link类型的不处理
continue
for fs in [files, folders]:
for filename in fs:
filepath = join(parent, filename)
# 看是否已经在本地数据库了
if not should_sync(filepath, root_path, check_md5):
continue
file_paths.append(filepath)
return file_paths
def store_sync_cursor(root, cursor):
make_sure_sync_data_folder(root)
sync_folder = join(root, '.sync')
cursor_path = join(sync_folder, '%s.cursor' % app_name)
with open(cursor_path, 'w') as f:
f.write(cursor)
def make_sure_sync_log_path(root):
make_sure_sync_data_folder(root)
sync_folder = join(root, '.sync')
log_path = join(sync_folder, '%s.log' % app_name)
return log_path
def get_sync_log_path(root):
sync_folder = join(root, '.sync')
log_path = join(sync_folder, '%s.log' % app_name)
return log_path
def main():
current_folder = os.getcwd()
argv = sys.argv
if len(argv) == 2 and argv[1].strip('-') in ['help', 'h']:
return show_help()
if len(argv) == 2 and len(argv[1].strip()) == site_token_length:
# set token
site_token = argv[1].strip()
site_token_filepath = join(current_folder, site_token_filename)
make_sure_path(site_token_filepath)
with open(site_token_filepath, 'w') as f:
f.write(site_token)
info = "site token is stored in %s now, do not public this file to anyone else." % site_token_filepath
print_with_color(info, 'yellow')
else:
if not is_site_folder(current_folder):
# try to match parents
current_folder = get_site_folder_from_parents(current_folder)
if not is_site_folder(current_folder):
print_with_color(
"please run `bitcron TOKEN` first, you can find site TOKEN from your sites list page.",
'red')
else:
print_with_color('under %s now' % current_folder, 'green')
# set site node
if len(argv) == 2 and argv[1].strip().endswith('.bitcron.com'):
site_node = argv[1].strip()
site_node_config_filepath = join(current_folder,
site_node_filename)
with open(site_node_config_filepath, 'w') as f:
f.write(site_node)
print_with_color('current node on %s now' % site_node, 'green')
return
action = ''
if len(argv) == 1:
action = 'sync_to' # sync to server
elif len(argv) == 2:
raw_action = argv[1]
if raw_action in ['sync', 'sync_from', 'sync-from']:
action = 'sync_from'
elif raw_action in ['logout']:
action = 'logout'
elif raw_action in ['reset']:
action = 'reset'
else:
action = raw_action
if action == 'info':
show_info(current_folder)
elif action in ['site', 'open']:
opened_url = open_site_in_browser(current_folder)
if not opened_url:
print_with_color("can't handle this command", 'red')
else:
print_with_color(
"%s is opened in your browser now" % opened_url,
'cyan')
elif action == 'sync_to':
do_sync_to(token=get_site_token(current_folder),
root=current_folder)
elif action == 'sync_from':
do_sync_from(token=get_site_token(current_folder),
root=current_folder)
elif action == 'logout':
delete_site_folder_token(current_folder)
print_with_color(
"site TOKEN of current site folder is cleared on this disk.",
'red')
elif action == 'reset':
delete_sync_meta_folder(current_folder)
print_with_color(
"metadata information files of site folder are cleared on this disk.",
'red')
else:
print_with_color("can't find matched command to run", 'red')
def get_sync_data_folder(root):
data_path = join(root, '.sync/%s' % app_name)
return data_path
def delete_sync_meta_folder(site_folder):
delete_site_folder_token(site_folder)
meta_folder = join(site_folder, '.sync')
delete_file(meta_folder)
def handle_meta(token, meta, root_path, sync_content_url):
# prefix 更多是一个范围，一般是一个 site folder
root_path = same_slash(root_path).rstrip('/')
relative_path = same_slash(meta['path']).lstrip(
'/') # 这个是大小写敏感的, 相对于根目录下的相对路径
site_name = relative_path.strip('/').split('/')[0]
full_path = join(root_path, relative_path) # 本地电脑的path
version = meta.get('version', '')
is_deleted = meta.get('is_deleted', False)
is_dir = meta.get('is_dir', False)
if is_a_hidden_path(relative_path): # 隐藏文件不处理
return
if relative_path.startswith(
'_cache/') or relative_path == '_cache': # cache 文件不处理
return
if os.path.exists(join(root_path, site_name, '.sync_ignore')):
return # ignore
if is_deleted:
if os.path.exists(full_path):
delete_file(full_path)
# delete
after_sync_deleted(full_path, root=root_path)
return 'got %s from server, delete' % relative_path
elif is_dir:
if not os.path.isdir(full_path):
try:
os.makedirs(full_path)
after_synced(full_path, root=root_path)
except OSError:
return 'failed to create dir %s' % full_path
except:
pass
else: # 具体的文件
file_id = to_unicode(meta['_id'])
need_download = True
if os.path.isfile(full_path):
old_version = md5_for_file(full_path)
if old_version == version:
need_download = False
if need_download:
# 需要下载具体的文件
timeout = 20 * 60
if is_markdown(full_path):
timeout = 2 * 60
response = requests.post(sync_content_url,
data=dict(token=token, id=file_id),
timeout=timeout)
if response.status_code >= 400:
# ignore
return 'fail to get %s, status code is %s' % (
full_path, response.status_code)
content = response.content
try:
create_file(full_path, content)
except OSError:
return 'failed to create file then ignore %s' % full_path
after_synced(full_path,
root=root_path) # 保存这个状态，这样下次同步的时候，就不会认为这个文件需要同步了
time.sleep(0.2) # 避免过度请求被服务器封锁的情况
return 'got %s from server' % relative_path
else:
return '%s same file in both client side and server side' % relative_path
def get_sync_data_filepath(filepath, root):
data_folder = make_sure_sync_data_folder(root)
data_filename = get_sync_data_filename(filepath, root)
data_filepath = join(data_folder, data_filename)
return data_filepath
def do_sync_to(token, root):
# need to upload
write_logs(['start to sync now'], '', root) # init
changes_log_filepath = join(root, 'sync_changes.log')
sync_configs = get_sync_configs(root, token)
if not sync_configs:
write_logs(['site TOKEN error or network connection error'], '',
root) # init
return
sync_should_url = sync_configs['sync_should_url']
sync_url = sync_configs['sync_url']
try:
file_paths = loop_local_filesystem(root)
try:
file_paths.remove(changes_log_filepath)
except:
pass
if file_paths:
write_logs('will try to sync %s files to server' % len(file_paths),
root=root)
for filepath in file_paths:
logs = sync_file_to_server(token,
filepath,
root=root,
sync_should_url=sync_should_url,
sync_url=sync_url)
if should_break_sync(logs, root):
break
write_logs(logs, filepath, root)
if logs: # 不管日志上是否显示成功，都进行存档
after_synced(filepath, root)
files_to_delete = find_files_to_delete(root)
if files_to_delete:
write_logs('will try to delete %s files on server' %
len(files_to_delete),
root=root)
for filepath in files_to_delete:
# delete file
logs = sync_file_to_server(token,
filepath,
root=root,
sync_should_url=sync_should_url,
sync_url=sync_url)
if should_break_sync(logs, root):
break
write_logs(logs, filepath, root)
after_sync_deleted(filepath,
root) # no need to try again to delete on cloud
# 将file_paths & files_to_delete 写到 root 内的某个配置文件内，以供其它调用
if files_to_delete or file_paths:
data = dict(
date=get_now_str(),
root=root,
added=file_paths,
deleted=files_to_delete,
)
with open(changes_log_filepath, 'w') as f:
f.write(json.dumps(data))
else:
write_logs(['no need to sync to server, it is up-to-date.'],
root=root)
except Exception, e:
write_logs([to_str(e)], '', root) # init
write_logs(['unexpected errors happen, abort syncing now'], '',
root) # init
error_info = sys.exc_info()
if error_info:
e_type, value, tb = error_info[:3]
traceback.print_exception(e_type, value, tb)
