def _traverse_folder(session_json, folder, output=None, progress=tqdm):
session = Session.from_json(session_json)
folder = core.get_entity(session, folder)
if folder is None:
return
# make folder
if output is None:
path = folder.attributes.name
else:
path = os.path.join(output, folder.attributes.name)
if not os.path.isdir(path):
os.mkdir(path)
# get files
files = [[path, f] for f in session.get(folder.relationships.files.related)]
# for each folders, recurse and return files
folders = session.get(folder.relationships.folders.related)
with Pool() as p:
for subfiles in progress(p.imap_unordered(functools.partial(_traverse_folder,
session.json(),
output=path,
progress=progress),
folders),
desc='Traversing folders',
unit='folder',
total=len(folders)):
files += subfiles
return files
def _download_revision_path(session_json, revision_path, progress=tqdm):
session = Session.from_json(session_json)
try:
return download_revision(session, revision_path[1], output=revision_path[0], progress=progress)
except DownloadException as e:
logging.error("Download error: {}".format(e))