def get_root_file_tree(self, root_folder_path=''):
"""
This is a generator function. Each iteration returned will be an instance
of StoreTree - this instance will just show the progress. Just use the last
one returned for a complete tree.
:param root_folder_path: the path to the root folder of the desired store.
:return: StoreTree instance.
"""
# Yandex uses paths only, so these are the ids.
root_standard_path = StoreTree.standardise_path(root_folder_path)
yandex_root_path = _convert_standard_to_yandex_path(root_standard_path)
# Check the root path exists
if self._get_item_metadata(yandex_root_path) is None:
raise ValueError('Couldn\'t find folder with path {}'.format(
root_standard_path))
result_tree = StoreTree(id=yandex_root_path)
# Yandex's resources/files request returns all files, but not directories.
# This means we will miss empty directories...
# So we have to traverse each directory individually using the resources
# request. Slow.
stack = [yandex_root_path]
while len(stack) > 0:
parent_yandex_path = stack.pop()
parent_meta = self._get_item_metadata(parent_yandex_path)
if '_embedded' in parent_meta and len(
parent_meta['_embedded']['items']) > 0:
# Add new dirs and files to parent
for item in parent_meta['_embedded']['items']:
item_id = _yandex_id_from_yandex_path(item['path'])
if item['type'] == 'dir':
result_tree.add_folder(item_id,
name=item['name'],
parent_id=parent_yandex_path)
stack.append(item_id)
else:
result_tree.add_file(
item_id,
item['name'],
parent_id=parent_yandex_path,
modified_datetime=_build_mtime_from_yandex_item(
item),
file_hash=item['md5'])
yield result_tree
def get_root_file_tree(self, root_folder_path=''):
"""
This is a generator function. Each iteration returned will be an instance
of StoreTree - this instance will just show the progress. Just use the last
one returned for a complete tree.
:param root_folder_path: the path to the root folder of the desired store.
:return: StoreTree instance.
"""
root_folder_id = self._get_folder_id_from_path(root_folder_path)
if root_folder_id is None:
raise ValueError(
'Root {} doesn\'t appear to exist.'.format(root_folder_path))
result_tree = StoreTree(root_folder_id)
# Another provider that forces us to traverse every folder...
stack = [_integer_id_from_str_id(root_folder_id)]
while len(stack) > 0:
parent_folder_id = stack.pop()
# Get folders in parent folder, look for current folder
entries = \
self._do_paginated_get(
http_server_utils.join_url_components([self._api_drive_endpoint_prefix,
'folders/{}/items'.format(parent_folder_id)]),
'entries',
params={'fields': _metadata_fields})
for item in entries:
if item['type'] == 'folder':
result_tree.add_folder(
_str_id_from_folder_integer_id(item['id']),
name=item['name'],
parent_id=_str_id_from_folder_integer_id(
parent_folder_id))
stack.append(item['id'])
else:
result_tree.add_file(
_str_id_from_file_integer_id(item['id']),
name=item['name'],
parent_id=_str_id_from_folder_integer_id(
parent_folder_id),
modified_datetime=date_parser.isoparse(
item['content_modified_at']),
file_hash=item['sha1'])
yield result_tree
def get_root_file_tree(self, root_folder_path=''):
"""
This is a generator function. Each iteration returned will be an instance
of StoreTree - this instance will just show the progress. Just use the last
one returned for a complete tree.
:param root_folder_path: the path to the root folder of the desired store.
:return: StoreTree instance.
"""
root_folder_id = self._get_folder_path_metadata(root_folder_path)['id']
result_tree = StoreTree(root_folder_id)
# Recursive traverse of the root
r, rx_dict = self._do_request(
'get',
http_server_utils.join_url_components(
[self._api_drive_endpoint_prefix, 'listfolder']),
params={
'folderid': _integer_id_from_str_id(root_folder_id),
'recursive': 1
})
# DFS the resultant contents lists to build tree
stack = [rx_dict['metadata']]
while len(stack) > 0:
parent_item = stack.pop()
for item in parent_item['contents']:
if item['isfolder']:
result_tree.add_folder(item['id'],
name=item['name'],
parent_id=parent_item['id'])
stack.append(item)
else:
result_tree.add_file(
item['id'],
name=item['name'],
parent_id=parent_item['id'],
modified_datetime=_convert_pcloud_string_to_dt(
item['modified']))
yield result_tree
def testCreateFromId(self):
tree = StoreTree(0)
# - folder1 - folder4 - file2
# - file4
# - folder2
# - folder3 - folder5
# - file1.txt
# - file3.png
tree.add_folder(1, 'folder1')
tree.add_folder(2, 'folder2')
tree.add_folder(3, 'folder3')
tree.add_folder(4, 'folder4', parent_id=1)
tree.add_folder(5, 'folder5', parent_id=3)
tree.add_file(6, 'file1.txt')
tree.add_file(7, 'file2', parent_id=4)
tree.add_file(8, 'file3.png')
tree.add_file(9, 'file4', parent_id=1)
new_tree = tree.create_new_from_id(1)
# folder 1 is now the new root
# - folder4 - file2
# - file4
root_dict = tree.find_item_by_id(1)
self.assertEqual(len(root_dict['files']), 1)
self.assertEqual(len(root_dict['folders']), 1)
self.assertEqual(root_dict['id'], 1)
self.assertEqual(root_dict['name'], 'folder1')
file4_dict = root_dict['files'][0]
self.assertEqual(file4_dict['id'], 9)
self.assertEqual(file4_dict['name'], 'file4')
folder4_dict = tree.find_item_by_id(4)
self.assertEqual(len(folder4_dict['files']), 1)
self.assertEqual(len(folder4_dict['folders']), 0)
self.assertEqual(folder4_dict['id'], 4)
self.assertEqual(folder4_dict['name'], 'folder4')
def testGetfilePathsList(self):
tree = StoreTree(0)
tree.add_folder(1, 'folder1')
tree.add_folder(2, 'folder2')
tree.add_folder(3, 'folder3')
tree.add_folder(4, 'folder4', parent_id=1)
tree.add_folder(5, 'folder5', parent_id=3)
tree.add_file(6, 'file1.txt')
tree.add_file(7, 'file2', parent_id=4)
tree.add_file(8, 'file3.png')
file_paths = tree.get_file_paths_list()
self.assertEqual(len(file_paths), 3)
self.assertIn(StoreTree.concat_paths(['', 'file1.txt']), file_paths)
self.assertIn(
StoreTree.concat_paths(['', 'folder1', 'folder4', 'file2']),
file_paths)
self.assertIn(StoreTree.concat_paths(['', 'file3.png']), file_paths)
def testFindItemByPath(self):
tree = StoreTree(0)
tree.add_folder(1, 'folder1')
tree.add_folder(2, 'folder2')
tree.add_folder(3, 'folder3')
tree.add_folder(4, 'folder4', parent_id=1)
tree.add_folder(5, 'folder5', parent_id=3)
tree.add_file(6, 'file1.txt')
tree.add_file(7, 'file2', parent_id=4)
tree.add_file(8, 'file3.png')
root_dict = tree.find_item_by_path('')
self.assertEqual(len(root_dict['files']), 2)
self.assertEqual(len(root_dict['folders']), 3)
self.assertEqual(root_dict['id'], 0)
self.assertEqual(root_dict['name'], '')
folder1_dict = tree.find_item_by_path('folder1/',
is_path_to_file=False)
self.assertEqual(len(folder1_dict['files']), 0)
self.assertEqual(len(folder1_dict['folders']), 1)
self.assertEqual(folder1_dict['id'], 1)
self.assertEqual(folder1_dict['name'], 'folder1')
folder3_dict = tree.find_item_by_path('folder3', is_path_to_file=False)
self.assertEqual(len(folder3_dict['files']), 0)
self.assertEqual(len(folder3_dict['folders']), 1)
self.assertEqual(folder3_dict['id'], 3)
self.assertEqual(folder3_dict['name'], 'folder3')
folder4_dict = tree.find_item_by_path('folder1/folder4',
is_path_to_file=False)
self.assertEqual(len(folder4_dict['files']), 1)
self.assertEqual(len(folder4_dict['folders']), 0)
self.assertEqual(folder4_dict['id'], 4)
self.assertEqual(folder4_dict['name'], 'folder4')
file1_dict = tree.find_item_by_path('file1.txt', is_path_to_file=True)
self.assertEqual(file1_dict['id'], 6)
self.assertEqual(file1_dict['name'], 'file1.txt')
file1_dict = tree.find_item_by_path('file1.txt', is_path_to_file=True)
self.assertEqual(file1_dict['id'], 6)
self.assertEqual(file1_dict['name'], 'file1.txt')
file2_dict = tree.find_item_by_path('folder1/folder4/file2',
is_path_to_file=True)
self.assertEqual(file2_dict['id'], 7)
self.assertEqual(file2_dict['name'], 'file2')
file3_dict = tree.find_item_by_path('file3.png', is_path_to_file=True)
self.assertEqual(file3_dict['id'], 8)
self.assertEqual(file3_dict['name'], 'file3.png')
self.assertIs(tree.find_item_by_path('folder4'), None)
self.assertIs(
tree.find_item_by_path('folder1/folder4/file3',
is_path_to_file=True), None)
self.assertIs(
tree.find_item_by_path('folder1/folder4/file3',
is_path_to_file=False), None)
self.assertIs(
tree.find_item_by_path('folder1/folder4', is_path_to_file=True),
None)
def testFindItemById(self):
tree = StoreTree(0)
tree.add_folder(1, 'folder1')
tree.add_folder(2, 'folder2')
tree.add_folder(3, 'folder3')
tree.add_folder(4, 'folder4', parent_id=1)
tree.add_folder(5, 'folder5', parent_id=3)
tree.add_file(6, 'file1.txt')
tree.add_file(7, 'file2', parent_id=4)
tree.add_file(8, 'file3.png')
root_dict = tree.find_item_by_id(0)
self.assertEqual(len(root_dict['files']), 2)
self.assertEqual(len(root_dict['folders']), 3)
self.assertEqual(root_dict['id'], 0)
self.assertEqual(root_dict['name'], '')
folder1_dict = tree.find_item_by_id(1)
self.assertEqual(len(folder1_dict['files']), 0)
self.assertEqual(len(folder1_dict['folders']), 1)
self.assertEqual(folder1_dict['id'], 1)
self.assertEqual(folder1_dict['name'], 'folder1')
folder4_dict = tree.find_item_by_id(4)
self.assertEqual(len(folder4_dict['files']), 1)
self.assertEqual(len(folder4_dict['folders']), 0)
self.assertEqual(folder4_dict['id'], 4)
self.assertEqual(folder4_dict['name'], 'folder4')
file1_dict = tree.find_item_by_id(6)
self.assertEqual(file1_dict['id'], 6)
self.assertEqual(file1_dict['name'], 'file1.txt')
file2_dict = tree.find_item_by_id(7)
self.assertEqual(file2_dict['id'], 7)
self.assertEqual(file2_dict['name'], 'file2')
file3_dict = tree.find_item_by_id(8)
self.assertEqual(file3_dict['id'], 8)
self.assertEqual(file3_dict['name'], 'file3.png')
self.assertIs(tree.find_item_by_id(39), None)
# ================================================
# Test find parent
folder3_dict, _ = tree.find_item_parent_by_id(5)
self.assertEqual(len(folder3_dict['files']), 0)
self.assertEqual(len(folder3_dict['folders']), 1)
self.assertEqual(folder3_dict['id'], 3)
self.assertEqual(folder3_dict['name'], 'folder3')
root_dict, _ = tree.find_item_parent_by_id(2)
self.assertEqual(len(root_dict['files']), 2)
self.assertEqual(len(root_dict['folders']), 3)
self.assertEqual(root_dict['id'], 0)
self.assertEqual(root_dict['name'], '')
self.assertIs(tree.find_item_parent_by_id(29)[0], None)
def testAddTree(self):
# Tree 1
# - folder1 - folder4 - file2
# - file4
# - folder2
# - folder3 - folder5
# - file1.txt
# - file3.png
tree1 = StoreTree(0)
tree1.add_folder(1, 'folder1')
tree1.add_folder(2, 'folder2')
tree1.add_folder(3, 'folder3')
tree1.add_folder(4, 'folder4', parent_id=1)
tree1.add_folder(5, 'folder5', parent_id=3)
tree1.add_file(6, 'file1.txt')
tree1.add_file(7, 'file2', parent_id=4)
tree1.add_file(8, 'file3.png')
tree1.add_file(9, 'file4', parent_id=1)
# Tree 2
# - folder6 - folder7 - file5
# - file6.png
tree2 = StoreTree(100)
tree2.add_folder(10, 'folder6')
tree2.add_folder(11, 'folder7', parent_id=10)
tree2.add_file(12, 'file5.txt', parent_id=11)
tree2.add_file(13, 'file6.png')
# New tree
# - folder1 - folder4 - file2
# - file4
#
# - folder2 - tree2 - folder6 - folder7 - file5.txt
# - file6.png
#
# - folder3 - folder5
# - file1.txt
# - file3.png
tree1.add_tree(tree2, 2)
tree1.update_folder_name(100, 'tree2')
root_dict = tree1.find_item_by_id(0)
self.assertEqual(len(root_dict['files']), 2)
self.assertEqual(len(root_dict['folders']), 3)
self.assertEqual(root_dict['id'], 0)
self.assertEqual(root_dict['name'], '')
tree2_dict = tree1.find_item_by_id(100)
self.assertEqual(len(tree2_dict['files']), 1)
self.assertEqual(len(tree2_dict['folders']), 1)
self.assertEqual(tree2_dict['id'], 100)
self.assertEqual(tree2_dict['name'], 'tree2')
folder6_dict = tree1.find_item_by_id(10)
self.assertEqual(len(folder6_dict['files']), 0)
self.assertEqual(len(folder6_dict['folders']), 1)
self.assertEqual(folder6_dict['id'], 10)
self.assertEqual(folder6_dict['name'], 'folder6')
folder7_dict = tree1.find_item_by_id(11)
self.assertEqual(len(folder7_dict['files']), 1)
self.assertEqual(len(folder7_dict['folders']), 0)
self.assertEqual(folder7_dict['id'], 11)
self.assertEqual(folder7_dict['name'], 'folder7')
file5_dict = folder7_dict['files'][0]
self.assertEqual(file5_dict['id'], 12)
self.assertEqual(file5_dict['name'], 'file5.txt')
file6_dict = tree1.find_item_by_id(13)
self.assertEqual(file6_dict['id'], 13)
self.assertEqual(file6_dict['name'], 'file6.png')
def testAddFolderPath(self):
tree = StoreTree(0)
# - folder1 - folder2 - file1.txt
# - folder3 - folder4
tree.add_folder_path([{
'name': 'folder1',
'id': 1
}, {
'name': 'folder2',
'id': 2
}])
tree.add_file(3, 'file1.txt', 2)
tree.add_folder(4, 'folder3', 0)
tree.add_folder_path([
{
'name': 'folder3',
'id': 4
},
{
'name': 'folder4',
'id': 5
},
])
#Check exception raised when adding existing folder with different id
with self.assertRaises(ValueError):
tree.add_folder_path([{
'name': 'folder1',
'id': 1
}, {
'name': 'folder2',
'id': 86
}])
root_dict = tree.find_item_by_id(tree.root_id)
self.assertEqual(len(root_dict['files']), 0)
self.assertEqual(len(root_dict['folders']), 2)
self.assertEqual(root_dict['id'], 0)
folder1_dict = tree.find_item_by_id(1)
self.assertEqual(len(folder1_dict['files']), 0)
self.assertEqual(len(folder1_dict['folders']), 1)
self.assertEqual(folder1_dict['id'], 1)
self.assertEqual(folder1_dict['name'], 'folder1')
folder2_dict = tree.find_item_by_id(2)
self.assertEqual(len(folder2_dict['files']), 1)
self.assertEqual(len(folder2_dict['folders']), 0)
self.assertEqual(folder2_dict['id'], 2)
self.assertEqual(folder2_dict['name'], 'folder2')
folder3_dict = tree.find_item_by_id(4)
self.assertEqual(len(folder3_dict['files']), 0)
self.assertEqual(len(folder3_dict['folders']), 1)
self.assertEqual(folder3_dict['id'], 4)
self.assertEqual(folder3_dict['name'], 'folder3')
folder4_dict = tree.find_item_by_id(5)
self.assertEqual(len(folder4_dict['files']), 0)
self.assertEqual(len(folder4_dict['folders']), 0)
self.assertEqual(folder4_dict['id'], 5)
self.assertEqual(folder4_dict['name'], 'folder4')
def get_root_file_tree(self, root_folder_path=''):
"""
This is a generator function. Each iteration returned will be an instance
of StoreTree - this instance will just show the progress. Just use the last
one returned for a complete tree.
:param root_folder_path: the path to the root folder of the desired store.
:return: StoreTree instance.
"""
# Get root id
root_folder_path = StoreTree.standardise_path(root_folder_path)
if root_folder_path == '':
url = http_server_utils.join_url_components(
[self._api_drive_endpoint_prefix, 'root'])
else:
url = http_server_utils.join_url_components([
self._api_drive_endpoint_prefix,
'root:/{}'.format(StoreTree.standardise_path(root_folder_path))
])
r = self._do_request('get',
url,
params={'select': 'id'},
raise_for_status=False)
if r.status_code == 404:
error_dict = r.json()
if error_dict['error']['code'] == 'itemNotFound':
raise ValueError('Couldn\'t find folder with path {}'.format(
root_folder_path))
r.raise_for_status()
root_id = r.json()['id']
result_tree = StoreTree(id=root_id)
# Have to traverse the whole thing per directory, but can use
# batching to help a little with latency...
# We use a stack for returned folder ids and one for nextLink
# urls and service the next link urls first.
stack = [root_id]
while len(stack) > 0:
batch = {}
# For each folder id on the stack, build a request and put in the batch
while len(stack) > 0 and self._batch_is_full(batch) == False:
self._add_request_to_batch(
batch,
'GET',
'/me/drive/items/{}/children'.format(stack.pop()),
params={
'top':
1000,
'select':
'id,name,folder,file,parentReference,fileSystemInfo'
})
# Do batch request
r = self._do_request('post', self._api_drive_batch_url, json=batch)
rx_dict = r.json()
# Maintain a list for response bodies because reach response body could have
# a nextLink that needs to be accessed.
body_list = [
response['body'] for response in rx_dict['responses']
if 'body' in response
]
for body in body_list:
for item in body['value']:
if 'folder' in item:
result_tree.add_folder(
item['id'],
name=item['name'],
parent_id=item['parentReference']['id'])
stack.append(item['id'])
else:
if 'sha1Hash' in item['file']['hashes']:
h = item['file']['hashes']['sha1Hash'].lower()
else:
h = None
result_tree.add_file(
item['id'],
name=item['name'],
parent_id=item['parentReference']['id'],
modified_datetime=date_parser.isoparse(
item['fileSystemInfo']
['lastModifiedDateTime']),
file_hash=h)
if '@odata.nextLink' in body:
r = self._do_request('get', body['@odata.nextLink'])
body_list.append(r.json())
yield result_tree