def _compare_dir(self, localdir, remotedir, result):
children = remotedir['item_collection']['entries']
server_file_map = dict((f['name'], f) \
for f in children if 'sha1' in f)
server_folder_map = dict((f['name'], f) \
for f in children if 'sha1' not in f)
result_item = result.start_add(remotedir)
subfolders = []
for filename in os.listdir(localdir):
path = os.path.join(localdir, filename)
if os.path.isfile(path):
node = server_file_map.pop(filename, None)
if node is None:
result_item.add_client_unique(True, path)
else:
result_item.add_compare(
get_sha1(path) != node['sha1'], path, node)
elif os.path.isdir(path):
folder_node = server_folder_map.pop(filename, None)
if folder_node is None:
result_item.add_client_unique(False, path)
else:
subfolders.append(folder_node)
result_item.add_server_unique(True, server_file_map)
result_item.add_server_unique(False, server_folder_map)
# compare recursively
for folder in subfolders:
path = os.path.join(localdir, folder['name'])
folder = self.get_file_info(folder['id'], False)
self._compare_dir(path, folder, result)
result.end_add()
return result
def _check_file_on_server(self, filepath, parent):
"""Check if the file already exists on the server
Return `None` if not,
return `True` if it does, and has the same SHA,
return id if it does, but has the different SHA.
"""
filename = os.path.basename(filepath)
files = self.list(parent)['entries'] or []
for f in files:
name = f['name']
if name == filename:
logger.debug(u"found same filename: {}".format(name))
if f['type'] == 'folder':
logger.error(u"A folder named '{}' already exists on" \
" the server".format(name))
raise FileConflictionError()
sha1 = f['sha1']
if get_sha1(filepath) == sha1:
logger.debug("same sha1")
return True
else:
logger.debug("diff sha1")
return f['id']
logger.debug(u"file {} not found under the directory {}".format(
filename, parent))
def _compare_dir(self, localdir, dir_tree, result):
files = dir_tree.find("files")
server_file_nodes = files and files.findall("file") or []
server_file_map = dict((f.attrib["file_name"], f) for f in server_file_nodes)
folders = dir_tree.find("folders")
server_folder_nodes = folders and folders.findall("folder") or []
server_folder_map = dict((f.attrib["name"], f) for f in server_folder_nodes)
result_item = result.start_add(dir_tree)
dir_subtrees = []
for filename in os.listdir(localdir):
path = os.path.join(localdir, filename)
if os.path.isfile(path):
node = server_file_map.pop(filename, None)
if node is None:
result_item.add_client_unique(True, path)
else:
sha1 = node.attrib["sha1"]
result_item.add_compare(get_sha1(path) != sha1, path, node)
elif os.path.isdir(path):
folder_node = server_folder_map.pop(filename, None)
if folder_node is None:
result_item.add_client_unique(False, path)
else:
dir_subtrees.append(folder_node)
result_item.add_server_unique(True, server_file_map)
result_item.add_server_unique(False, server_folder_map)
# compare recursively
for subtree in dir_subtrees:
path = os.path.join(localdir, subtree.attrib["name"])
self._compare_dir(path, subtree, result)
result.end_add()
return result
def download_dir(self, folder_id, localdir=None, by_name=False):
"""Download the directory with the given id to a local directory"""
self._check()
if by_name:
folder_id = self._convert_to_id(folder_id, False)
folder_info = self.get_file_info(folder_id, False)
folder_name = folder_info['name']
localdir = os.path.join(localdir or ".", folder_name)
try:
os.makedirs(localdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
files = folder_info['item_collection']['entries']
for f in (files or []):
file_name = f['name']
file_id = f['id']
file_type = f['type']
if file_type == 'file':
localfile = os.path.join(localdir, file_name)
if os.path.exists(localfile):
# check
sha1 = f['sha1']
if get_sha1(localfile) == sha1:
logger.debug("same sha1")
continue
# download
self.download_file(file_id, localdir)
elif file_type == 'folder':
self.download_dir(file_id, localdir)
else:
logger.warn(u"unexpected file type".format(file_type))
def download_dir(self, folder_id, localdir=None, by_name=False):
"""Download the directory with the given id to a local directory"""
self._check()
if by_name:
folder_id = self._convert_to_id(folder_id, False)
tree = self.list(folder_id)
folder_name = tree.attrib["name"]
localdir = os.path.join(localdir or ".", folder_name)
try:
os.makedirs(localdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
files = tree.find("files")
for f in files and files.findall("file") or []:
file_name = f.attrib["file_name"]
file_id = f.attrib["id"]
localfile = os.path.join(localdir, file_name)
if os.path.exists(localfile):
# check
sha1 = f.attrib["sha1"]
if get_sha1(localfile) == sha1:
logger.debug("same sha1")
continue
# download
self.download_file(file_id, localdir)
folders = tree.find("folders")
for f in folders and folders.findall("folder") or []:
folder_id = f.attrib["id"]
self.download_dir(folder_id, localdir)
def download_dir(self, folder_id, localdir=None, by_name=False):
"""Download the directory with the given id to a local directory"""
self._check()
if by_name:
folder_id = self._convert_to_id(folder_id, False)
tree = self.list(folder_id)
folder_name = tree.attrib['name']
localdir = os.path.join(localdir or ".", folder_name)
try:
os.makedirs(localdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
files = tree.find('files')
for f in (files and files.findall('file') or []):
file_name = f.attrib['file_name']
file_id = f.attrib['id']
localfile = os.path.join(localdir, file_name)
if os.path.exists(localfile):
# check
sha1 = f.attrib['sha1']
if get_sha1(localfile) == sha1:
logger.debug("same sha1")
continue
# download
self.download_file(file_id, localdir)
folders = tree.find('folders')
for f in (folders and folders.findall('folder') or []):
folder_id = f.attrib['id']
self.download_dir(folder_id, localdir)
def test_get_sha1(tmpdir):
text_hash_pairs = [
("", "da39a3ee5e6b4b0d3255bfef95601890afd80709"),
("1234567890", "01b307acba4f54f55aafc33bb06bbbf6ca803e9a"),
("1234567890\n", "12039d6dd9a7e27622301e935b6eefc78846802e"),
("abc", "a9993e364706816aba3e25717850c26c9cd0d89d"),
]
for text, hash_val in text_hash_pairs:
f = tmpdir.join("tmp.txt")
f.write(text)
assert get_sha1(str(f)) == hash_val
def _check_file_on_server(self, filepath, parent):
"""Check if the file with the same SHA in the server"""
filename = os.path.basename(filepath)
tree = self.list(parent)
files = tree.find("files")
for f in files and files.findall("file") or []:
attrs = f.attrib
name = attrs["file_name"]
if name == filename:
logger.debug(u"found same filename: {}".format(name))
sha1 = attrs["sha1"]
if get_sha1(filepath) == sha1:
logger.debug("same sha1")
return True
logger.debug("diff sha1")
return # attrs['id']
logger.debug(u"file {} not found under the directory {}".format(filename, parent))
def _check_file_on_server(self, filepath, parent):
"""Check if the file with the same SHA in the server"""
filename = os.path.basename(filepath)
tree = self.list(parent)
files = tree.find('files')
for f in (files and files.findall('file') or []):
attrs = f.attrib
name = attrs['file_name']
if name == filename:
logger.debug(u"found same filename: {}".format(name))
sha1 = attrs['sha1']
if get_sha1(filepath) == sha1:
logger.debug("same sha1")
return True
logger.debug("diff sha1")
return #attrs['id']
logger.debug(u"file {} not found under the directory {}"
.format(filename, parent))
def _compare_dir(self, localdir, dir_tree, result):
files = dir_tree.find('files')
server_file_nodes = files and files.findall('file') or []
server_file_map = dict((f.attrib['file_name'], f) \
for f in server_file_nodes)
folders = dir_tree.find('folders')
server_folder_nodes = folders and folders.findall('folder') or []
server_folder_map = dict((f.attrib['name'], f) \
for f in server_folder_nodes)
result_item = result.start_add(dir_tree)
dir_subtrees = []
for filename in os.listdir(localdir):
path = os.path.join(localdir, filename)
if os.path.isfile(path):
node = server_file_map.pop(filename, None)
if node is None:
result_item.add_client_unique(True, path)
else:
sha1 = node.attrib['sha1']
result_item.add_compare(get_sha1(path) != sha1, path, node)
elif os.path.isdir(path):
folder_node = server_folder_map.pop(filename, None)
if folder_node is None:
result_item.add_client_unique(False, path)
else:
dir_subtrees.append(folder_node)
result_item.add_server_unique(True, server_file_map)
result_item.add_server_unique(False, server_folder_map)
# compare recursively
for subtree in dir_subtrees:
path = os.path.join(localdir, subtree.attrib['name'])
self._compare_dir(path, subtree, result)
result.end_add()
return result
def compare_file(self, localfile, remotefile, by_name=False):
"""Compare files between server and client(as per SHA1)"""
sha1 = get_sha1(localfile)
info = self.get_file_info(remotefile, True, by_name)
return sha1 == info['sha1']
def compare_file(self, localfile, remotefile, by_name=False):
"""Compare files between server and client"""
sha1 = get_sha1(localfile)
info = self.get_file_info(remotefile, by_name)
return sha1 == info["sha1"]
