def get_relative_path(filepath, root=None): filepath = same_slash(filepath) root = same_slash(root) relative_path = os.path.split(filepath)[-1] if root and filepath.startswith(root + '/'): relative_path = filepath.replace(root + '/', '', 1) return relative_path
def update_sync_data(filepath, root, new_data): # 有时候有必要,需要存储一些特殊的字段进去 filepath = same_slash(filepath) data_path = get_sync_data_filepath(filepath, root) data = get_sync_data(filepath, root) data.update(new_data) with open(data_path, 'w') as f: f.write(json.dumps(data))
def after_sync_deleted(filepath, root): filepath = same_slash(filepath) data_path = get_sync_data_filepath(filepath, root) if os.path.isfile(data_path): try: os.remove(data_path) except: pass
def get_sync_data(filepath, root): # get the synced information for a filepath # 根据文件的路径,获得对应 md5 文件,里面存储了必要的信息(md5 * synced_at),用于判断当前文件是否需要同步 filepath = same_slash(filepath) data_path = get_sync_data_filepath(filepath, root) if os.path.isfile(data_path): try: with open(data_path) as f: data = json.loads(f.read()) if data.get('is_relative'): # 相对路径,转为绝对路径 data['filepath'] = join(root, data['filepath']) if isinstance(data, dict): return data except: pass return {} # final
def loop_local_filesystem(root_path, check_md5=True): root_path = same_slash(root_path) if not os.path.isdir(root_path): # 根目录不存在,不处理 return [] file_paths = [] for parent, folders, files in os.walk(root_path): if is_a_hidden_path(parent): continue elif not is_real(parent): # link类型的不处理 continue for fs in [files, folders]: for filename in fs: filepath = join(parent, filename) # 看是否已经在本地数据库了 if not should_sync(filepath, root_path, check_md5): continue file_paths.append(filepath) return file_paths
def find_files_to_delete(root_path): sync_data_folder = get_sync_data_folder(root_path) if not os.path.isdir(sync_data_folder): # never synced before return [] files = loop_local_filesystem(root_path, check_md5=False) # same_path already data_filenames = os.listdir(sync_data_folder) old_file_paths = [] for data_filename in data_filenames: data_filepath = join(sync_data_folder, data_filename) try: with open(data_filepath) as f: data = json.loads(f.read()) filepath = data.get('filepath') if data.get('is_relative'): filepath = join(root_path, filepath) if filepath: old_file_paths.append(same_slash(filepath)) except: pass return list(set(old_file_paths) - set(files))
def after_synced(filepath, root, **extra_data): filepath = same_slash(filepath) if not os.path.exists(filepath): return # ignore if not is_sub_path(filepath, root): return # ignore data_path = get_sync_data_filepath(filepath, root) now = datetime.datetime.now() relative_path = get_relative_path(filepath, root) sync_data = dict(filepath=relative_path, synced_at=now.strftime('%Y-%m-%d %H:%M:%S'), md5=md5_for_file(filepath), is_dir=os.path.isdir(filepath), is_relative=True) sync_data.update(extra_data) with open(data_path, 'w') as f: f.write(json.dumps(sync_data)) # store the parent folders into the local sync db parent_folder = os.path.dirname(filepath) parent_data_path = get_sync_data_filepath(parent_folder, root) if not os.path.isfile(parent_data_path): after_synced(parent_folder, root)
def sync_from_server(root, token): logs = [] root = same_slash(root) sync_configs = get_sync_configs(root, token) if not sync_configs: write_logs( "can't get the correct sync configs from server, check your network connection and your TOKEN?" ) return sync_list_url = sync_configs['sync_list_url'] sync_content_url = sync_configs['sync_content_url'] old_cursor = get_sync_cursor(root) write_logs('start sync from server', root=root) new_cursor, metas = get_metas(token, cursor=old_cursor, sync_list_url=sync_list_url) if not metas: write_logs('no need to sync from server, it is up-to-date.', root=root) return else: write_logs('will try to sync %s files from server.' % len(metas), root=root) for meta in metas: handler_log = handle_meta(token, meta, root, sync_content_url) if handler_log: write_logs(handler_log, root=root) if new_cursor: # 处理之后,再保存cursor store_sync_cursor(root, new_cursor) if new_cursor is None and not metas: logs.append('syncing maybe failed, check your sync config?') logs.append('sync ended') write_logs(logs, root=root)
def handle_meta(token, meta, root_path, sync_content_url): # prefix 更多是一个范围,一般是一个 site folder root_path = same_slash(root_path).rstrip('/') relative_path = same_slash(meta['path']).lstrip( '/') # 这个是大小写敏感的, 相对于根目录下的相对路径 site_name = relative_path.strip('/').split('/')[0] full_path = join(root_path, relative_path) # 本地电脑的path version = meta.get('version', '') is_deleted = meta.get('is_deleted', False) is_dir = meta.get('is_dir', False) if is_a_hidden_path(relative_path): # 隐藏文件不处理 return if relative_path.startswith( '_cache/') or relative_path == '_cache': # cache 文件不处理 return if os.path.exists(join(root_path, site_name, '.sync_ignore')): return # ignore if is_deleted: if os.path.exists(full_path): delete_file(full_path) # delete after_sync_deleted(full_path, root=root_path) return 'got %s from server, delete' % relative_path elif is_dir: if not os.path.isdir(full_path): try: os.makedirs(full_path) after_synced(full_path, root=root_path) except OSError: return 'failed to create dir %s' % full_path except: pass else: # 具体的文件 file_id = to_unicode(meta['_id']) need_download = True if os.path.isfile(full_path): old_version = md5_for_file(full_path) if old_version == version: need_download = False if need_download: # 需要下载具体的文件 timeout = 20 * 60 if is_markdown(full_path): timeout = 2 * 60 response = requests.post(sync_content_url, data=dict(token=token, id=file_id), timeout=timeout) if response.status_code >= 400: # ignore return 'fail to get %s, status code is %s' % ( full_path, response.status_code) content = response.content try: create_file(full_path, content) except OSError: return 'failed to create file then ignore %s' % full_path after_synced(full_path, root=root_path) # 保存这个状态,这样下次同步的时候,就不会认为这个文件需要同步了 time.sleep(0.2) # 避免过度请求被服务器封锁的情况 return 'got %s from server' % relative_path else: return '%s same file in both client side and server side' % relative_path
def sync_file_to_server(token, filepath, root, sync_should_url, sync_url): # return logs (list) # 将本地的path转为 Server side 需要的path logs = [] filepath = same_slash(filepath) root = same_slash(root) if is_a_hidden_path(filepath): return [] # ignore if not is_sub_path(filepath, root): return [] relative_path = get_relative_path(filepath, root) data = { 'source': 'cli', 'platform': PLATFORM, 'path': to_str(relative_path), 'token': token, } if os.path.exists(filepath): reason = reason_for_not_sync(filepath) if reason: # 不同步处理 return [reason] is_deleted = False is_dir = os.path.isdir(filepath) is_file = not is_dir data['client_mtime'] = str( datetime.datetime.fromtimestamp(os.path.getmtime(filepath), tz=tzutc())) else: is_deleted = True sync_data = get_sync_data(filepath, root=root) is_dir = sync_data.get('is_dir', False) is_file = False data['is_dir'] = str(is_dir).lower() # 转为str data['is_deleted'] = str(is_deleted).lower() # 转为str if not is_deleted: if is_dir: version = '' else: version = md5_for_file(filepath) data['version'] = version # 开始同步到服务器 response = None if is_file: try: timeout = 20 * 60 if is_markdown(filepath): timeout = 2 * 60 try: checker_response = requests.post(sync_should_url, data=data, timeout=timeout) if checker_response.status_code == 401: return ['bad token, break now'] if checker_response.text == 'no': # 不需要同步了,已经存在了 return ['exist on server side already'] elif checker_response.text == 'no_site': return ['no site matched, break now'] # will not sync... except: pass with open(filepath, 'rb') as f: response = requests.post(sync_url, data=data, files={'file': f}, timeout=timeout) if response.status_code == 401: return ['bad token, break now'] logs.append('uploaded to server done') except ConnectionError: logs.append('uploaded to server failed because of ConnectionError') except Timeout: logs.append('uploaded to server failed because of TimeoutError') else: # 删除文件或者是目录的更新 try: response = requests.post(sync_url, data=data) if is_deleted: logs.append('deleted on server') else: logs.append('update folder to server') except ConnectionError: if is_deleted: logs.append( 'delete on server failed because of ConnectionError') else: logs.append( 'update folder on server failed because of ConnectionError' ) if response: status_code = response.status_code if status_code == 200: if response.headers.get( 'content-type') == 'application/json' and response.json( ).get('error_code'): message = response.json().get('message') or '' error_code = response.json().get('error_code') try: error_code = int(error_code) if error_code in [503, 401, 404]: info = 'error_code:%s %s' % (error_code, message) logs.append(info) except: pass if status_code == 401: # 授权错误,删除原来保存的token logs.append('bad token') return logs