def test_pull_opera(self):
'''
Test fetching Opera profile.
'''
from anytree import Node
from frostmark.common import traverse
from frostmark.models import Folder, Bookmark
from frostmark.importer.opera import OperaImporter
sample = join(
dirname(abspath(__file__)),
'sample_opera.json'
)
# pylint: disable=protected-access
tree = OperaImporter.assemble_import_tree(sample)
flat_tree = traverse(tree)
self.assertIsInstance(tree, Node)
self.maxDiff = None # pylint: disable=invalid-name
self.assertEqual([
(item.folder_name, item.node_type)
if item.node_type == Folder
else (item.title, item.node_type)
for item in flat_tree
], [
('<no title>', Folder),
('Bookmarks bar', Folder),
('PortableApps.com', Bookmark),
('Shared Bookmarks', Folder),
('Speed Dial', Folder),
('V7', Folder),
('', Folder),
('Let\'s connect', Bookmark),
('Booking.com: Cheap Hotels', Bookmark),
('Wikipedia', Bookmark),
('', Folder),
('Spojte sa s nami', Bookmark),
('Zoznam', Bookmark),
('Facebook', Bookmark),
('Amazon.com', Bookmark),
('eBay', Bookmark),
('Yahoo!', Bookmark),
('Twitter', Bookmark),
('YouTube', Bookmark),
('Mall', Bookmark),
('Heureka', Bookmark),
('Topky', Bookmark),
('Amazon.de', Bookmark),
('Azet', Bookmark),
('Trash', Folder),
('Unsorted Bookmarks', Folder),
('My Folders', Folder),
('Other bookmarks', Folder),
('Mobile bookmarks', Folder)
])
def test_tree_folder(self):
'''
Test creating folder tree both nested and flat.
'''
from anytree import Node
from frostmark.common import assemble_folder_tree, traverse
from frostmark.models import Folder
tree = assemble_folder_tree(items=self.FOLDER_DATA,
key='parent_folder_id',
node_type=Folder)
flat_tree = traverse(tree)
self.assertIsInstance(tree, Node)
self.assertEqual(tree.id, 0)
# unfold the same way each branch is build from the root
# to the last child when assembling i.e. sorted and top-down
#
# 0
# |-1
# | |-2
# | |-3
# |
# |-4
# |
# |-5
# |-6
# |-7
self.assertEqual([item.id for item in flat_tree],
list(range(len(self.FOLDER_DATA))))
for item in flat_tree:
self.assertEqual(item.node_type, Folder)
def test_pull_firefox(self):
'''
Test fetching Firefox profiles from places.sqlite.
'''
from anytree import Node
from frostmark.common import traverse
from frostmark.models import Folder, Bookmark
from frostmark.importer import Importer
from frostmark.importer.firefox import FirefoxImporter
sample = join(
dirname(abspath(__file__)),
'sample_firefox.sqlite'
)
# pylint: disable=protected-access
session = Importer._path_session(sample, FirefoxImporter.BASE)
tree = FirefoxImporter.assemble_import_tree(session)
flat_tree = traverse(tree)
self.assertIsInstance(tree, Node)
self.maxDiff = None # pylint: disable=invalid-name
self.assertEqual([
(item.folder_name, item.node_type)
if item.node_type == Folder
else (item.title, item.node_type)
for item in flat_tree
], [
('<no title>', Folder),
('Bookmarks Menu', Folder),
('Mozilla Firefox', Folder),
('Help and Tutorials', Bookmark),
('Customize Firefox', Bookmark),
('Get Involved', Bookmark),
('About Us', Bookmark),
('PortableApps.com', Bookmark),
('Recently Bookmarked', Bookmark),
('Recent Tags', Bookmark),
('Bookmarks Toolbar', Folder),
('Latest Headlines', Folder),
('Getting Started', Bookmark),
('Most Visited', Bookmark),
('Tags', Folder),
('Unsorted Bookmarks', Folder)
])
def test_pull_chrome(self):
'''
Test fetching Chrome profile.
'''
from anytree import Node
from frostmark.common import traverse
from frostmark.models import Folder, Bookmark
from frostmark.importer.chrome import ChromeImporter
sample = join(
dirname(abspath(__file__)),
'sample_chrome.json'
)
# pylint: disable=protected-access
tree = ChromeImporter.assemble_import_tree(sample)
flat_tree = traverse(tree)
self.assertIsInstance(tree, Node)
self.maxDiff = None # pylint: disable=invalid-name
self.assertEqual([
(item.folder_name, item.node_type)
if item.node_type == Folder
else (item.title, item.node_type)
for item in flat_tree
], [
('<no title>', Folder),
('Panel so záložkami', Folder),
('folder', Folder),
('nestedfolder', Folder),
('YouTube', Bookmark),
('emptyfolder', Folder),
('Google', Bookmark),
('PortableApps.com', Bookmark),
('Ostatné', Folder),
('Mapy Google', Bookmark),
('Záložky v mobile', Folder)
])
def test_tree_bookmark(self):
'''
Test creating folder + bookmark tree both nested and flat.
'''
from anytree import Node
from frostmark.common import (assemble_folder_tree,
assemble_bookmark_tree, traverse)
from frostmark.models import Folder, Bookmark
folder_tree = assemble_folder_tree(items=self.FOLDER_DATA,
key='parent_folder_id',
node_type=Folder)
tree = assemble_bookmark_tree(items=self.BOOKMARK_DATA,
key='folder_id',
folder_tree_root=folder_tree,
node_type=Bookmark)
flat_tree = traverse(tree)
self.assertIsInstance(tree, Node)
self.assertEqual(tree.id, 0)
# unfold the same way each as with folders, but make sure bookmarks
# are appended at the end of the Node children
#
# 0
# |-1
# | |-2
# | | |-b7
# | |
# | |-3
# | | |-b8
# | | |-b9
# | | |-b10
# | |
# | |-b4
# | |-b5
# | |-b6
# |
# |-4
# | |-b11
# |
# |-5
# | |-6
# | | |-7
# | | | |-b14
# | | |
# | | |-b13
# | |
# | |-b12
# |
# |-b1
# |-b2
# |-b3
self.assertEqual([item.id for item in flat_tree], [
0, 1, 2, 7, 3, 8, 9, 10, 4, 5, 6, 4, 11, 5, 6, 7, 14, 13, 12, 1, 2,
3
])
for item in flat_tree:
if 'folder_name' in vars(item):
self.assertEqual(item.node_type, Folder)
else:
self.assertEqual(item.node_type, Bookmark)
def import_from(self, path: str):
'''
Import bookmarks from particular path into internal storage.
'''
# pylint: disable=too-many-locals
backend = self.backend
# get the bookmarks tree from file
if isinstance(backend, FirefoxImporter):
source = self._path_session(path=path, base=backend.BASE)
elif isinstance(backend, OperaImporter):
source = path
elif isinstance(backend, ChromeImporter):
source = path
tree = backend.assemble_import_tree(source)
# open internal DB
nodes = traverse(tree)
folders = {node.id: node for node in nodes if node.node_type == Folder}
bookmarks = {
node.id: node
for node in nodes if node.node_type == Bookmark
}
# sqla objects
frost = get_session()
sqla_folders = {}
# add folder structure to the database
sorted_folders = sorted(
folders.values(),
# parent_folder_id is None for root folder
key=lambda item: item.parent_folder_id or 0)
# first sort the tree by parent IDs so that there is each parent
# available, however in case there is a kind-of circular relationship
# between the folders e.g. the ID of a child is smaller than ID of
# a parent which might be caused by browser importing old bookmarks
# from database directly or from a different browser while incorrectly
# setting IDs (or better said re-using already existing IDs when
# possible), therefore sorting by parent ID would work, but when trying
# to access the parent a KeyError would be raised because of parent
# not being available yet due to higher ID than the child has
#
# for that reason try to sort with parent ID first and postpone
# the relationship evaluation by using second/third/etc/... item
# in the sorted list until there is parent ID available (or throw
# IndexError in the end which would pretty much mean that the browser
# DB is just broken due to missing parent / dangling children)
idx = 0
while sorted_folders:
folder = sorted_folders[idx]
kwargs = {'folder_name': folder.folder_name}
if folder.parent_folder_id:
# in case there is a parent, get the Folder object
# and pull its ID after flush() (otherwise it's None)
try:
real_id = sqla_folders[folder.parent_folder_id].id
except KeyError:
idx += 1
continue
kwargs['parent_folder_id'] = real_id
new_folder = Folder(**kwargs)
frost.add(new_folder)
# flush to obtain folder ID,
# especially necessary for nested folders
frost.flush()
# preserve the original ID and point to a SQLA object
sqla_folders[folder.id] = new_folder
# remove current folder
idx = 0
sorted_folders.remove(folder)
# add bookmarks
for key in sorted(bookmarks.keys()):
book = bookmarks[key]
kwargs = {
'title': book.title,
'url': book.url,
'icon': b'',
'folder_id': sqla_folders[book.folder_id].id
}
# no need to flush, nothing required a bookmark
# ID to be present before final commit()
new_book = Bookmark(**kwargs)
frost.add(new_book)
# write data into internal DB
frost.commit()
def prepare_export():
'''
Export bookmarks from internal storage.
'''
xml = dedent('''
<!DOCTYPE NETSCAPE-Bookmark-file-1>
<!-- This is an automatically generated file.
It will be read and overwritten.
DO NOT EDIT!
-->
<META
HTTP-EQUIV="Content-Type"
CONTENT="text/html; charset=UTF-8"
/>
<TITLE>Bookmarks</TITLE>
<H1>Bookmarks</H1>
''')
xml = xml[1:] # strip first \n character
tree = traverse(fetch_bookmark_tree())
tree_len = len(tree)
sep = ' ' * 4
folder_stack = []
for idx, node in enumerate(tree):
if node.node_type == Folder:
# root folder for internal DB
if node.id == 0:
xml += f'{sep * len(folder_stack)}<DL><p>\n'
elif [node.parent_folder_id] == folder_stack[-2:-1]:
# child folder of a second last folder on stack
# stack = [parent, child1]
# F parent
# |-- F child
# +-- F child <- this
# pop the sister folder from stack
# close the previous folder
folder_stack = folder_stack[:-1]
xml += f'{sep * len(folder_stack)}</DL><p>\n'
xml += (f'{sep * len(folder_stack)}'
f'<DT><H3>{node.folder_name}</H3>\n')
xml += f'{sep * len(folder_stack)}<DL><p>\n'
else:
xml += (f'{sep * len(folder_stack)}'
f'<DT><H3>{node.folder_name}</H3>\n')
xml += f'{sep * len(folder_stack)}<DL><p>\n'
# completely new folder, append to stack
folder_stack.append(node.id)
if node.id == 0:
continue
# empty folder, close DL, pop from stack
if not node.children and idx != tree_len - 1:
# pop from stack and close the folder
if node.id in folder_stack:
folder_stack = folder_stack[:-1]
xml += f'{sep * len(folder_stack)}</DL><p>\n'
continue
# last element, in case it's folder close the DL
if idx == tree_len - 1:
folder_stack = folder_stack[:-1]
xml += f'{sep * len(folder_stack)}</DL><p>\n'
continue
elif node.node_type == Bookmark:
# bookmark of a second folder on the stack
# F
# |-- F
# | +-- B
# +-- B <- this
if [node.folder_id] != folder_stack[-1:]:
folder_stack = folder_stack[:-1]
xml += (f'{sep * len(folder_stack)}</DL><p>\n'
f'{sep * len(folder_stack)}<HR>\n')
xml += (
f'{sep * len(folder_stack)}'
f'<DT><A HREF="{node.url}" '
f'ICON="{node.icon.decode("utf-8")}">{node.title}</A>\n')
# if there are remaining folders, those don't have
# any children remaining, so the whole tree can be
# closed with multiple dedented DL closings
while folder_stack:
folder_stack = folder_stack[:-1]
xml += f'{sep * len(folder_stack)}</DL><p>\n'
return xml
def assemble_import_tree(path: str) -> Node:
'''
Assemble a bookmark tree structure from `Bookmarks` file to be able
to either display or correctly import/merge the structure into
internal bookmarks database.
'''
with open(path, 'rb') as fbookmark:
raw = json.loads(fbookmark.read().decode('utf-8'))
trees = []
if 'bookmark_bar' in raw['roots']:
folder_items = OperaImporter.walk_folders(
raw['roots']['bookmark_bar'], 0)
trees.append(
assemble_folder_tree(items=folder_items,
key='parent_folder_id',
node_type=Folder))
if 'custom_root' in raw['roots']:
raw_custom_sorted = sorted(raw['roots']['custom_root'].items(),
key=lambda item: item[0])
for _, value in raw_custom_sorted:
folder_items = OperaImporter.walk_folders(value, 0)
trees.append(
assemble_folder_tree(items=folder_items,
key='parent_folder_id',
node_type=Folder))
if 'other' in raw['roots']:
folder_items = OperaImporter.walk_folders(raw['roots']['other'], 0)
trees.append(
assemble_folder_tree(items=folder_items,
key='parent_folder_id',
node_type=Folder))
if 'synced' in raw['roots']:
folder_items = OperaImporter.walk_folders(raw['roots']['synced'],
0)
trees.append(
assemble_folder_tree(items=folder_items,
key='parent_folder_id',
node_type=Folder))
# printable folder tree
folder_tree = Node(name=0,
node_type=Folder,
id=0,
folder_name='<no title>',
parent_folder_id=None,
item={})
for tree in trees:
tree.parent = folder_tree
tree.parent_folder_id = folder_tree.id # pylint: disable=no-member
bookmarks = []
for folder in traverse(folder_tree):
bookmarks += OperaImporter.walk_bookmarks(folder.item, folder.id)
delattr(folder, 'item')
# printable folder+bookmark tree
bookmark_tree = assemble_bookmark_tree(items=bookmarks,
key='folder_id',
folder_tree_root=folder_tree,
node_type=Bookmark)
return bookmark_tree