""" Organize file structure and keep the archive clean.

Instances of this class organize where to place what files based on the given folder

structure and keep track of the seen files to simplify cleanup unused files afterwards.

"""

def __init__(self, path: str):

""" Create enw tracker instance.

:param path: Path to the archive.

"""

self.index = []

self.path = path

for dir_path, dir_names, file_names in os.walk(path):

for name in dir_names + file_names:

self.index.append(unicodedata.normalize('NFC', os.path.join(dir_path, name)))

def cleanup(self):

""" Remove all files in archive that where not requested since initialization. """

for path in sorted(self.index, reverse=True, key=len):

if os.path.isfile(path):

try:

os.unlink(str(path))

logging.debug('removed file %r', path)

except OSError:

pass

elif os.path.isdir:

try:

if not os.listdir(str(path)):

os.rmdir(str(path))

logging.debug('removed folder %r', path)

except OSError:

pass

def used(self, path: str):

""" Mark a file as seen by it's absolute path.

:param path: Absolute path to the file to mark as seen.

"""

normalized_path = unicodedata.normalize('NFC', path)

if normalized_path in self.index:

self.index.remove(normalized_path)

return True

else:

return False

@contextmanager

def file_handler(self, folders: list, file_name: str):

""" Get a file handler to store a new file.

:param folders: List of path elements to store the file at.

:param file_name: Name of the file to store

:rtype: Union[None, File]

:return: None if the file already exists or a file handler.

"""

if folders:

file_dir = os.path.join(*folders)

else:

file_dir = ''

if not self.used(file_dir):

try:

absolute_dir = os.path.join(self.path, file_dir)

os.makedirs(absolute_dir)

logging.debug('created folder %r', absolute_dir)

except FileExistsError:

pass

absolute_path = os.path.join(self.path, file_dir, file_name)

if not self.used(absolute_path):

with open(absolute_path, 'wb') as fd:

logging.debug('created file %r', absolute_path)

yield fd

else:

""" Get all dropbox paper document ids in this account.

:param dbx: Dropbox instance with logged in account with sufficient permissions to list documents.

:param page_size: Number of document ids to to get with one request.

:return: Dropbox Paper ids.

"""

listing = dbx.paper_docs_list(limit=page_size)

while True:

for document_id in listing.doc_ids:

yield document_id

if listing.has_more:

listing = dbx.paper_docs_list_continue(listing.cursor.value)

else:

""" Download a file.

:param url: The url to get the file from.

:param fd: The file descriptor to write the file to.

"""

response = requests.get(url, stream=True, verify=False)

for chunk in response.iter_content(chunk_size=2 ** 10):

""" Find all image references, download them and replace the references to the downloaded copies.

:param tracker: Instance of the tracker to create new files with.

:param folders: List of folder names where the document is stored.

:param document_reference: A string unique to the document.

:param body: The documents body to search in.

:return: The document body with replaced image references.

"""

pattern = re.compile(rb'(<img[^<]+src=)(?P<delimiter>[\'"])(?P<url>[^\'"]+)(?P=delimiter)([^>]*>)')

relative_resource_dir = '.%s' % document_reference

cache = {}

for index, match in enumerate(pattern.finditer(body)):

begin, delimiter, url, end = match.groups()

if url in cache:

body = body.replace(match.group(0), begin + delimiter + cache[url].encode('utf-8') + delimiter + end)

else:

file_name = ("%d_" % index) + urllib.parse.urlparse(url).path.split(b'/')[-1].decode('utf-8')

relative_url = os.path.join(relative_resource_dir, file_name)

body = body.replace(match.group(0), begin + delimiter + relative_url.encode('utf-8') + delimiter + end)

cache[url] = relative_url

with tracker.file_handler(folders + [relative_resource_dir], file_name) as fd:

if fd:

download_resource_file(url.decode('utf-8'), fd)

""" Download a document to the local file system.

:param export_format: The format to store the document in. Possible values are "html", "markdown"

or "all". For "html" also referenced images are downloaded.

:param tracker: Instance of the tracker to create new files with.

:param dbx: Dropbox instance with logged in account with sufficient permissions to download documents.

:param document_id: The document id if the paper document to download.

"""

try:

folders = [folder.name.replace('/', '+')

for folder

in dbx.paper_docs_get_folder_info(document_id).folders or []]

except dropbox.exceptions.ApiError:

folders = []

try:

if export_format == 'html' or export_format == 'all':

document_meta, document_body = dbx.paper_docs_download(document_id, dropbox.paper.ExportFormat('html'))

document_reference = "%s-%s" % (document_id, document_meta.revision)

content = replace_images(tracker, folders, document_reference, document_body.content)

file_name = "%s [%s].html" % (document_meta.title.replace('/', '+'), document_reference)

with tracker.file_handler(folders, file_name) as fd:

if fd:

fd.write(content)

if export_format == 'markdown' or export_format == 'all':

document_meta, document_body = dbx.paper_docs_download(document_id, dropbox.paper.ExportFormat('markdown'))

document_reference = "%s-%s" % (document_id, document_meta.revision)

content = document_body.content

file_name = "%s [%s].md" % (document_meta.title.replace('/', '+'), document_reference)

with tracker.file_handler(folders, file_name) as fd:

if fd:

fd.write(content)

except dropbox.exceptions.ApiError as e:

""" Download all accessed documents in the dropbox account.

:param token: Token to login to the dropbox account.

:param target: Relative or absolute path to the download target.

:param export_format: he format to store the document in.

"""

dbx_session = dropbox.dropbox.create_session(max_connections=max_workers * 2)

dbx = dropbox.Dropbox(token, session=dbx_session)

tracker = Tracker(os.path.abspath(target))

pool = Pool(max_workers=max_workers)

for index, document_id in enumerate(paper_documents(dbx)):

pool.submit(store_document, export_format, tracker, dbx, document_id)

pool.shutdown()

""" Request a new token to acces an dropbox account. """

auth_flow = dropbox.DropboxOAuth2FlowNoRedirect(APP_KEY, obf(APP_SECRET).decode('ascii'))

auth_url = auth_flow.start()

webbrowser.open(auth_url)

print("1. Go to: %s" % auth_url)

print("2. Click \"Allow\" (you might have to log in first).")

print("3. Copy the authorization code.")

auth_code = input("Enter the authorization code here: ").strip()

# noinspection PyBroadException

try:

oauth_result = auth_flow.finish(auth_code)

except Exception:

logging.exception("Receiving token failed.")

sys.exit(1)

else:

print("Token received. From now on please start this script with:")

print(" --token=%s" % oauth_result.access_token)