def __upload_file(self, local_file: PathLike, remote_file: PathObject = None, compile=False, arch=None, progress_callback: ProgressCallback = None): lol = convert_to_pathstr(local_file) if remote_file == None: remote_file = self.get_remote_path(local_file) rmt = convert_to_pathstr(remote_file) if compile and self.should_compile(lol) and self.should_compile(rmt): data = get_compiled_file_content(lol, arch=arch) rmt = PATTERN_PY.sub(".mpy", rmt) else: with open(lol, "rb") as f: data = f.read() self.__fe.upload(rmt, data, progress_callback=progress_callback)
def should_compile(self, path: PathObject): if isinstance(path, FileEntity): if path.type == FileEntityType.DIRECTORY: return False pathstr = convert_to_pathstr(path) for ptn in self.__pattern_compile_ignored: if len(ptn.findall(pathstr)) > 0: return False return len(PATTERN_PY.findall(pathstr)) > 0
def should_include(self, path: PathObject, ignore_hidden=True): pathstr = convert_to_pathstr(path) for ptn in self.__pattern_include: if len(ptn.findall(pathstr)) > 0: return True for ptn in self.__pattern_exclude: if len(ptn.findall(pathstr)) > 0: return False for p in PurePath(pathstr).parts: if ignore_hidden and p.startswith(".") and p != ".": return False # ignore hidden file return True
def build(self, compile=False, arch=None, ignore_hidden=True, target_folder: PathLike = ".build", progress_callback: SyncProgressCallback = None): local_files = set(self.__walk_local_like_remote(ignore_hidden)) target_folder = syspath.abspath(target_folder) if syspath.exists(target_folder): rmtree(target_folder) makedirs(target_folder) new_file_record = {} for f in local_files: # base info localpath = self.get_local_path(f) target = syspath.join( target_folder, PurePath(localpath).relative_to(self.__local)) folder = syspath.dirname(target) if f.type == FileEntityType.DIRECTORY: if not syspath.exists(folder): makedirs(folder) continue if progress_callback != None: progress_callback(0, 0, 0, 0, "build", str(f.abspath.relative_to(self.__remote))) # calc hash hash = self.__hash_local_file(f, compile) key = convert_to_pathstr(f) new_file_record[key] = hash # build if not syspath.exists(folder): makedirs(folder) if compile and self.should_compile(f): data = get_compiled_file_content(localpath, arch=arch) target = PATTERN_PY.sub(".mpy", target) else: with open(localpath, 'rb') as f: data = f.read() with open(target, 'wb') as f: f.write(data) # write hash record localpath = self.get_local_path(self.__record_file_path) target = syspath.join(target_folder, PurePath(localpath).relative_to(self.__local)) folder = syspath.dirname(target) if not syspath.exists(folder): makedirs(folder) with open(target, "wb") as f: f.write(json.dumps(new_file_record).encode("utf-8"))
def get_local_path(self, path: PathObject): pth = PurePosixPath(convert_to_pathstr(path)).relative_to( self.__remote) return self.__local.joinpath(pth)
def sync_dir_remote_with_local( self, compile=False, arch=None, ignore_hidden=True, upload_only_modified=True, delete_exist_file=True, progress_callback: SyncProgressCallback = None): self.__fe._require_device() need_close = False if self.__fe.status == FileExplorerStatus.UNKNOWN: need_close = True try: if need_close: self.__fe.init() file_record = {} new_file_record = {} try: j = self.__fe.download(self.__record_file_path).decode("utf-8") file_record = json.loads(j) except: pass # ensure target folder exist on remote if not self.__fe.exist(self.__remote): self.__fe.mkdirs(self.__remote) # get file list local_files = set(self.__walk_local_like_remote(ignore_hidden)) local_files_compiled = set() for f in local_files: if compile and self.should_compile(f): new_name = PATTERN_PY.sub(".mpy", f.name) local_files_compiled.update( [FileEntity(f.directory, new_name, f.type, f.size)]) else: local_files_compiled.update([f]) remote_files = set(self.__walk_remote()) # get files need delete exist_should_delete_files = remote_files - local_files_compiled # file to delete try: f = self.__fe.stat(self.__record_file_path) exist_should_delete_files.discard(f) except: pass # get must upload file need_upload_files = set() dir_count = 0 for local_file in local_files: if local_file.type == FileEntityType.DIRECTORY: need_upload_files.add(local_file) dir_count += 1 continue hash = self.__hash_local_file(local_file, compile) key = convert_to_pathstr(local_file) new_file_record[key] = hash if not (key in file_record and file_record[key] == hash) or ( not upload_only_modified): exist_should_delete_files.add( local_file) # delete first, and upload need_upload_files.add(local_file) # start upload total = len(need_upload_files) - dir_count if delete_exist_file: total += len(exist_should_delete_files) finished = 0 if delete_exist_file: for f in exist_should_delete_files: if progress_callback != None: progress_callback( finished, total, 0, 0, "delete", str(f.abspath.relative_to(self.__remote))) self.__fe.rmtree(f) finished += 1 for f in need_upload_files: def upload_progress_callback(sub_p, sub_t): if progress_callback != None: progress_callback( finished, total, sub_p, sub_t, "upload", str(f.abspath.relative_to(self.__remote))) if f.type == FileEntityType.DIRECTORY: self.__fe.mkdirs(f) continue # dir not count else: try: self.__upload_file( self.get_local_path(f), f, compile, arch, progress_callback=upload_progress_callback) except: key = key = convert_to_pathstr(f) del new_file_record[key] print("================") traceback.print_exc() print('========> Upload Error:', key) finished += 1 # write record self.__fe.upload(self.__record_file_path, json.dumps(new_file_record).encode("utf-8")) finally: if need_close: self.__fe.close() self.__fe._release_device()