def _compare_dir(self, localdir, remotedir, result): children = remotedir['item_collection']['entries'] server_file_map = dict((f['name'], f) \ for f in children if 'sha1' in f) server_folder_map = dict((f['name'], f) \ for f in children if 'sha1' not in f) result_item = result.start_add(remotedir) subfolders = [] for filename in os.listdir(localdir): path = os.path.join(localdir, filename) if os.path.isfile(path): node = server_file_map.pop(filename, None) if node is None: result_item.add_client_unique(True, path) else: result_item.add_compare( get_sha1(path) != node['sha1'], path, node) elif os.path.isdir(path): folder_node = server_folder_map.pop(filename, None) if folder_node is None: result_item.add_client_unique(False, path) else: subfolders.append(folder_node) result_item.add_server_unique(True, server_file_map) result_item.add_server_unique(False, server_folder_map) # compare recursively for folder in subfolders: path = os.path.join(localdir, folder['name']) folder = self.get_file_info(folder['id'], False) self._compare_dir(path, folder, result) result.end_add() return result
def _check_file_on_server(self, filepath, parent): """Check if the file already exists on the server Return `None` if not, return `True` if it does, and has the same SHA, return id if it does, but has the different SHA. """ filename = os.path.basename(filepath) files = self.list(parent)['entries'] or [] for f in files: name = f['name'] if name == filename: logger.debug(u"found same filename: {}".format(name)) if f['type'] == 'folder': logger.error(u"A folder named '{}' already exists on" \ " the server".format(name)) raise FileConflictionError() sha1 = f['sha1'] if get_sha1(filepath) == sha1: logger.debug("same sha1") return True else: logger.debug("diff sha1") return f['id'] logger.debug(u"file {} not found under the directory {}".format( filename, parent))
def _compare_dir(self, localdir, dir_tree, result): files = dir_tree.find("files") server_file_nodes = files and files.findall("file") or [] server_file_map = dict((f.attrib["file_name"], f) for f in server_file_nodes) folders = dir_tree.find("folders") server_folder_nodes = folders and folders.findall("folder") or [] server_folder_map = dict((f.attrib["name"], f) for f in server_folder_nodes) result_item = result.start_add(dir_tree) dir_subtrees = [] for filename in os.listdir(localdir): path = os.path.join(localdir, filename) if os.path.isfile(path): node = server_file_map.pop(filename, None) if node is None: result_item.add_client_unique(True, path) else: sha1 = node.attrib["sha1"] result_item.add_compare(get_sha1(path) != sha1, path, node) elif os.path.isdir(path): folder_node = server_folder_map.pop(filename, None) if folder_node is None: result_item.add_client_unique(False, path) else: dir_subtrees.append(folder_node) result_item.add_server_unique(True, server_file_map) result_item.add_server_unique(False, server_folder_map) # compare recursively for subtree in dir_subtrees: path = os.path.join(localdir, subtree.attrib["name"]) self._compare_dir(path, subtree, result) result.end_add() return result
def download_dir(self, folder_id, localdir=None, by_name=False): """Download the directory with the given id to a local directory""" self._check() if by_name: folder_id = self._convert_to_id(folder_id, False) folder_info = self.get_file_info(folder_id, False) folder_name = folder_info['name'] localdir = os.path.join(localdir or ".", folder_name) try: os.makedirs(localdir) except OSError as e: if e.errno != errno.EEXIST: raise files = folder_info['item_collection']['entries'] for f in (files or []): file_name = f['name'] file_id = f['id'] file_type = f['type'] if file_type == 'file': localfile = os.path.join(localdir, file_name) if os.path.exists(localfile): # check sha1 = f['sha1'] if get_sha1(localfile) == sha1: logger.debug("same sha1") continue # download self.download_file(file_id, localdir) elif file_type == 'folder': self.download_dir(file_id, localdir) else: logger.warn(u"unexpected file type".format(file_type))
def download_dir(self, folder_id, localdir=None, by_name=False): """Download the directory with the given id to a local directory""" self._check() if by_name: folder_id = self._convert_to_id(folder_id, False) tree = self.list(folder_id) folder_name = tree.attrib["name"] localdir = os.path.join(localdir or ".", folder_name) try: os.makedirs(localdir) except OSError as e: if e.errno != errno.EEXIST: raise files = tree.find("files") for f in files and files.findall("file") or []: file_name = f.attrib["file_name"] file_id = f.attrib["id"] localfile = os.path.join(localdir, file_name) if os.path.exists(localfile): # check sha1 = f.attrib["sha1"] if get_sha1(localfile) == sha1: logger.debug("same sha1") continue # download self.download_file(file_id, localdir) folders = tree.find("folders") for f in folders and folders.findall("folder") or []: folder_id = f.attrib["id"] self.download_dir(folder_id, localdir)
def download_dir(self, folder_id, localdir=None, by_name=False): """Download the directory with the given id to a local directory""" self._check() if by_name: folder_id = self._convert_to_id(folder_id, False) tree = self.list(folder_id) folder_name = tree.attrib['name'] localdir = os.path.join(localdir or ".", folder_name) try: os.makedirs(localdir) except OSError as e: if e.errno != errno.EEXIST: raise files = tree.find('files') for f in (files and files.findall('file') or []): file_name = f.attrib['file_name'] file_id = f.attrib['id'] localfile = os.path.join(localdir, file_name) if os.path.exists(localfile): # check sha1 = f.attrib['sha1'] if get_sha1(localfile) == sha1: logger.debug("same sha1") continue # download self.download_file(file_id, localdir) folders = tree.find('folders') for f in (folders and folders.findall('folder') or []): folder_id = f.attrib['id'] self.download_dir(folder_id, localdir)
def test_get_sha1(tmpdir): text_hash_pairs = [ ("", "da39a3ee5e6b4b0d3255bfef95601890afd80709"), ("1234567890", "01b307acba4f54f55aafc33bb06bbbf6ca803e9a"), ("1234567890\n", "12039d6dd9a7e27622301e935b6eefc78846802e"), ("abc", "a9993e364706816aba3e25717850c26c9cd0d89d"), ] for text, hash_val in text_hash_pairs: f = tmpdir.join("tmp.txt") f.write(text) assert get_sha1(str(f)) == hash_val
def _check_file_on_server(self, filepath, parent): """Check if the file with the same SHA in the server""" filename = os.path.basename(filepath) tree = self.list(parent) files = tree.find("files") for f in files and files.findall("file") or []: attrs = f.attrib name = attrs["file_name"] if name == filename: logger.debug(u"found same filename: {}".format(name)) sha1 = attrs["sha1"] if get_sha1(filepath) == sha1: logger.debug("same sha1") return True logger.debug("diff sha1") return # attrs['id'] logger.debug(u"file {} not found under the directory {}".format(filename, parent))
def _check_file_on_server(self, filepath, parent): """Check if the file with the same SHA in the server""" filename = os.path.basename(filepath) tree = self.list(parent) files = tree.find('files') for f in (files and files.findall('file') or []): attrs = f.attrib name = attrs['file_name'] if name == filename: logger.debug(u"found same filename: {}".format(name)) sha1 = attrs['sha1'] if get_sha1(filepath) == sha1: logger.debug("same sha1") return True logger.debug("diff sha1") return #attrs['id'] logger.debug(u"file {} not found under the directory {}" .format(filename, parent))
def _compare_dir(self, localdir, dir_tree, result): files = dir_tree.find('files') server_file_nodes = files and files.findall('file') or [] server_file_map = dict((f.attrib['file_name'], f) \ for f in server_file_nodes) folders = dir_tree.find('folders') server_folder_nodes = folders and folders.findall('folder') or [] server_folder_map = dict((f.attrib['name'], f) \ for f in server_folder_nodes) result_item = result.start_add(dir_tree) dir_subtrees = [] for filename in os.listdir(localdir): path = os.path.join(localdir, filename) if os.path.isfile(path): node = server_file_map.pop(filename, None) if node is None: result_item.add_client_unique(True, path) else: sha1 = node.attrib['sha1'] result_item.add_compare(get_sha1(path) != sha1, path, node) elif os.path.isdir(path): folder_node = server_folder_map.pop(filename, None) if folder_node is None: result_item.add_client_unique(False, path) else: dir_subtrees.append(folder_node) result_item.add_server_unique(True, server_file_map) result_item.add_server_unique(False, server_folder_map) # compare recursively for subtree in dir_subtrees: path = os.path.join(localdir, subtree.attrib['name']) self._compare_dir(path, subtree, result) result.end_add() return result
def compare_file(self, localfile, remotefile, by_name=False): """Compare files between server and client(as per SHA1)""" sha1 = get_sha1(localfile) info = self.get_file_info(remotefile, True, by_name) return sha1 == info['sha1']
def compare_file(self, localfile, remotefile, by_name=False): """Compare files between server and client""" sha1 = get_sha1(localfile) info = self.get_file_info(remotefile, by_name) return sha1 == info["sha1"]