class SyncWorker:
"""
Manage sync with mozilla server, will start syncing on init
"""
def check_modules():
"""
True if deps are installed
@return bool
"""
from importlib import util
fxa = util.find_spec("fxa")
crypto = util.find_spec("Crypto")
return fxa is not None and crypto is not None
def __init__(self):
"""
Init worker
"""
self.__syncing = False
self.__sync_cancellable = Gio.Cancellable()
self.__username = ""
self.__password = ""
self.__token = ""
self.__uid = ""
self.__keyB = b""
self.__mtimes = {"bookmarks": 0.1, "history": 0.1}
# We do not create this here because it's slow down Eolie startup
# See __mozilla_sync property
self.__mz = None
self.__state_lock = True
self.__session = None
self.__helper = PasswordsHelper()
self.set_credentials()
def login(self, attributes, password):
"""
Login to service
@param attributes as {}
@param password as str
@raise exceptions
"""
self.__username = ""
self.__password = ""
self.__uid = ""
self.__token = ""
self.__keyB = b""
if attributes is None or not attributes["login"] or not password:
print("SyncWorker::login():", attributes)
return
from base64 import b64encode
session = None
self.__username = attributes["login"]
self.__password = password
# Connect to mozilla sync
try:
session = self.__mozilla_sync.login(self.__username, password)
bid_assertion, key = self.__mozilla_sync.\
get_browserid_assertion(session)
self.__token = session.token
self.__uid = session.uid
self.__keyB = session.keys[1]
keyB_encoded = b64encode(self.__keyB).decode("utf-8")
self.__helper.clear_sync(self.__helper.store_sync,
self.__username,
password,
self.__uid,
self.__token,
keyB_encoded,
self.on_password_stored,
True)
except Exception as e:
self.__helper.clear_sync(self.__helper.store_sync,
attributes["login"],
password,
"", "", "")
raise e
def new_session(self):
"""
Start a new session
"""
# Just reset session, will be set by get_session_bulk_keys()
self.__session = None
def set_credentials(self):
"""
Set credentials using Secret
"""
self.__helper.get_sync(self.__set_credentials)
def sync(self, loop=False, first_sync=False):
"""
Start syncing, you need to check sync_status property
@param loop as bool -> for GLib.timeout_add()
@param first_sync as bool
"""
if Gio.NetworkMonitor.get_default().get_network_available() and\
self.__username and self.__password and not self.syncing:
task_helper = TaskHelper()
task_helper.run(self.__sync, first_sync)
return loop
def push_history(self, history_ids):
"""
Push history ids
@param history_ids as [int]
"""
if Gio.NetworkMonitor.get_default().get_network_available() and\
self.__username and self.__password:
task_helper = TaskHelper()
task_helper.run(self.__push_history, history_ids)
def push_password(self, user_form_name, user_form_value, pass_form_name,
pass_form_value, uri, form_uri, uuid):
"""
Push password
@param user_form_name as str
@param user_form_value as str
@param pass_form_name as str
@param pass_form_value as str
@param uri as str
@param form_uri as str
@param uuid as str
"""
if Gio.NetworkMonitor.get_default().get_network_available() and\
self.__username and self.__password:
task_helper = TaskHelper()
task_helper.run(self.__push_password,
user_form_name, user_form_value, pass_form_name,
pass_form_value, uri, form_uri, uuid)
def remove_from_history(self, guid):
"""
Remove history guid from remote history
@param guid as str
"""
if Gio.NetworkMonitor.get_default().get_network_available() and\
self.__username and self.__password:
task_helper = TaskHelper()
task_helper.run(self.__remove_from_history, guid)
def remove_from_bookmarks(self, guid):
"""
Remove bookmark guid from remote bookmarks
@param guid as str
"""
if Gio.NetworkMonitor.get_default().get_network_available() and\
self.__username and self.__password:
task_helper = TaskHelper()
task_helper.run(self.__remove_from_bookmarks, guid)
def remove_from_passwords(self, uuid):
"""
Remove password from passwords collection
@param uuid as str
"""
if Gio.NetworkMonitor.get_default().get_network_available() and\
self.__username and self.__password:
task_helper = TaskHelper()
task_helper.run(self.__remove_from_passwords, uuid)
def delete_secret(self):
"""
Delete sync secret
"""
self.__username = ""
self.__password = ""
self.__session = None
self.__sync_cancellable.reset()
self.__helper.clear_sync(None)
def stop(self, force=False):
"""
Stop update, if force, kill session too
@param force as bool
"""
self.__sync_cancellable.cancel()
if force:
self.__session = None
def on_password_stored(self, secret, result, sync):
"""
Update credentials
@param secret as Secret
@param result as Gio.AsyncResult
@param sync as bool
"""
self.set_credentials()
if sync:
# Wait for credentials (server side)
GLib.timeout_add(10, self.sync, True)
@property
def mtimes(self):
"""
Sync engine modification times
@return {}
"""
return self.__mtimes
@property
def syncing(self):
"""
True if sync is running
@return bool
"""
return self.__syncing
@property
def status(self):
"""
True if sync is working
@return bool
"""
try:
if self.__username:
self.__get_session_bulk_keys()
self.__mozilla_sync.client.info_collections()
return True
except Exception as e:
print("SyncWorker::status(): %s" % e)
@property
def username(self):
"""
Get username
@return str
"""
return self.__username
#######################
# PRIVATE #
#######################
@property
def __mozilla_sync(self):
"""
Get mozilla sync, create if None
"""
if self.__mz is None:
self.__mz = MozillaSync()
return self.__mz
def __get_session_bulk_keys(self):
"""
Get session decrypt keys
@return keys as (b"", b"")
"""
if self.__session is None:
from fxa.core import Session as FxASession
from fxa.crypto import quick_stretch_password
self.__session = FxASession(self.__mozilla_sync.fxa_client,
self.__username,
quick_stretch_password(
self.__username,
self.__password),
self.__uid,
self.__token)
self.__session.keys = [b"", self.__keyB]
self.__session.check_session_status()
bid_assertion, key = self.__mozilla_sync.get_browserid_assertion(
self.__session)
bulk_keys = self.__mozilla_sync.connect(bid_assertion, key)
return bulk_keys
def __update_state(self):
"""
Update state file
"""
try:
# If syncing, state will be written by self.__sync()
if not self.syncing:
f = open(EOLIE_DATA_PATH + "/mozilla_sync.bin", "wb")
# Lock file
flock(f, LOCK_EX | LOCK_NB)
self.__mtimes = self.__mozilla_sync.client.info_collections()
dump(self.__mtimes, f)
# Unlock file
flock(f, LOCK_UN)
except Exception as e:
debug("SyncWorker::__update_state(): %s" % e)
def __push_history(self, history_ids):
"""
Push history ids if atime is available, else, ask to remove
@param history ids as [int]
"""
try:
bulk_keys = self.__get_session_bulk_keys()
for history_id in history_ids:
self.__check_worker()
sleep(0.01)
record = {}
atimes = El().history.get_atimes(history_id)
guid = El().history.get_guid(history_id)
if atimes:
record["histUri"] = El().history.get_uri(history_id)
record["id"] = guid
record["title"] = El().history.get_title(history_id)
record["visits"] = []
for atime in atimes:
record["visits"].append({"date": atime*1000000,
"type": 1})
debug("pushing %s" % record)
self.__mozilla_sync.add(record, "history", bulk_keys)
else:
record["id"] = guid
record["type"] = "item"
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "history", bulk_keys)
self.__update_state()
except Exception as e:
debug("SyncWorker::__push_history(): %s" % e)
def __push_password(self, user_form_name, user_form_value, pass_form_name,
pass_form_value, uri, form_uri, uuid):
"""
Push password
@param user_form_name as str
@param user_form_value as str
@param pass_form_name as str
@param pass_form_value as str
@param uri as str
@param uuid as str
"""
try:
self.__check_worker()
bulk_keys = self.__get_session_bulk_keys()
record = {}
record["id"] = "{%s}" % uuid
record["hostname"] = uri
record["formSubmitURL"] = form_uri
record["httpRealm"] = None
record["username"] = user_form_value
record["password"] = pass_form_value
record["usernameField"] = user_form_name
record["passwordField"] = pass_form_name
mtime = int(time()*1000)
record["timeCreated"] = mtime
record["timePasswordChanged"] = mtime
debug("pushing %s" % record)
self.__mozilla_sync.add(record, "passwords", bulk_keys)
self.__update_state()
except Exception as e:
print("SyncWorker::__push_password():", e)
def __remove_from_history(self, guid):
"""
Remove from history
@param guid as str
"""
try:
self.__check_worker()
bulk_keys = self.__get_session_bulk_keys()
record = {}
record["id"] = guid
record["type"] = "item"
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "history", bulk_keys)
self.__update_state()
except Exception as e:
debug("SyncWorker::__remove_from_history(): %s" % e)
def __remove_from_bookmarks(self, guid):
"""
Remove from history
@param guid as str
"""
try:
self.__check_worker()
bulk_keys = self.__get_session_bulk_keys()
record = {}
record["id"] = guid
record["type"] = "bookmark"
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "bookmark", bulk_keys)
self.__update_state()
except Exception as e:
debug("SyncWorker::__remove_from_bookmarks(): %s" % e)
def __remove_from_passwords(self, uuid):
"""
Remove password from passwords collection
@param uuid as str
"""
try:
self.__check_worker()
bulk_keys = self.__get_session_bulk_keys()
record = {}
record["id"] = uuid
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "passwords", bulk_keys)
self.__update_state()
except Exception as e:
debug("SyncWorker::__remove_from_passwords(): %s" % e)
def __sync(self, first_sync):
"""
Sync Eolie objects (bookmarks, history, ...) with Mozilla Sync
@param first_sync as bool
"""
debug("Start syncing")
self.__syncing = True
self.__sync_cancellable.reset()
try:
self.__mtimes = load(open(EOLIE_DATA_PATH + "/mozilla_sync.bin",
"rb"))
except:
self.__mtimes = {"bookmarks": 0.1,
"history": 0.1,
"passwords": 0.1}
try:
self.__check_worker()
bulk_keys = self.__get_session_bulk_keys()
new_mtimes = self.__mozilla_sync.client.info_collections()
self.__check_worker()
########################
# Passwords Management #
########################
try:
debug("local passwords: %s, remote passwords: %s" % (
self.__mtimes["passwords"],
new_mtimes["passwords"]))
# Only pull if something new available
if self.__mtimes["passwords"] != new_mtimes["passwords"]:
self.__pull_passwords(bulk_keys)
except:
pass # No passwords in sync
self.__check_worker()
######################
# History Management #
######################
try:
debug("local history: %s, remote history: %s" % (
self.__mtimes["history"],
new_mtimes["history"]))
# Only pull if something new available
if self.__mtimes["history"] != new_mtimes["history"]:
self.__pull_history(bulk_keys)
except:
pass # No history in sync
self.__check_worker()
########################
# Bookmarks Management #
########################
try:
debug("local bookmarks: %s, remote bookmarks: %s" % (
self.__mtimes["bookmarks"],
new_mtimes["bookmarks"]))
# Push new bookmarks
self.__push_bookmarks(bulk_keys)
except:
pass # No bookmarks in sync
self.__check_worker()
# Only pull if something new available
if self.__mtimes["bookmarks"] != new_mtimes["bookmarks"]:
self.__pull_bookmarks(bulk_keys, first_sync)
# Update last sync mtime
self.__syncing = False
self.__update_state()
debug("Stop syncing")
except Exception as e:
debug("SyncWorker::__sync(): %s" % e)
if str(e) == "The authentication token could not be found":
self.set_credentials()
self.__syncing = False
def __push_bookmarks(self, bulk_keys):
"""
Push to bookmarks
@param bulk keys as KeyBundle
@param start time as float
@raise StopIteration
"""
debug("push bookmarks")
parents = []
for bookmark_id in El().bookmarks.get_ids_for_mtime(
self.__mtimes["bookmarks"]):
self.__check_worker()
sleep(0.01)
parent_guid = El().bookmarks.get_parent_guid(bookmark_id)
# No parent, move it to unfiled
if parent_guid is None:
parent_guid = "unfiled"
parent_id = El().bookmarks.get_id_by_guid(parent_guid)
if parent_id not in parents:
parents.append(parent_id)
record = {}
record["bmkUri"] = El().bookmarks.get_uri(bookmark_id)
record["id"] = El().bookmarks.get_guid(bookmark_id)
record["title"] = El().bookmarks.get_title(bookmark_id)
record["tags"] = El().bookmarks.get_tags(bookmark_id)
record["parentid"] = parent_guid
record["parentName"] = El().bookmarks.get_parent_name(bookmark_id)
record["type"] = "bookmark"
debug("pushing %s" % record)
self.__mozilla_sync.add(record, "bookmarks", bulk_keys)
# Del old bookmarks
for bookmark_id in El().bookmarks.get_deleted_ids():
self.__check_worker()
sleep(0.01)
parent_guid = El().bookmarks.get_parent_guid(bookmark_id)
parent_id = El().bookmarks.get_id_by_guid(parent_guid)
if parent_id not in parents:
parents.append(parent_id)
record = {}
record["id"] = El().bookmarks.get_guid(bookmark_id)
record["type"] = "bookmark"
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "bookmarks", bulk_keys)
El().bookmarks.remove(bookmark_id)
# Push parents in this order, parents near root are handled later
# Otherwise, order will be broken by new children updates
while parents:
parent_id = parents.pop(0)
parent_guid = El().bookmarks.get_guid(parent_id)
parent_name = El().bookmarks.get_title(parent_id)
children = El().bookmarks.get_children(parent_guid)
# So search if children in parents
found = False
for child_guid in children:
child_id = El().bookmarks.get_id_by_guid(child_guid)
if child_id in parents:
found = True
break
# Handle children first
if found:
parents.append(parent_id)
debug("later: %s" % parent_name)
continue
record = {}
record["id"] = parent_guid
record["type"] = "folder"
# A parent with parent as unfiled needs to be moved to places
# Firefox internal
grand_parent_guid = El().bookmarks.get_parent_guid(parent_id)
if grand_parent_guid == "unfiled":
grand_parent_guid = "places"
record["parentid"] = grand_parent_guid
record["parentName"] = El().bookmarks.get_parent_name(parent_id)
record["title"] = parent_name
record["children"] = children
debug("pushing parent %s" % record)
self.__mozilla_sync.add(record, "bookmarks", bulk_keys)
El().bookmarks.clean_tags()
def __pull_bookmarks(self, bulk_keys, first_sync):
"""
Pull from bookmarks
@param bulk_keys as KeyBundle
@param first_sync as bool
@raise StopIteration
"""
debug("pull bookmarks")
SqlCursor.add(El().bookmarks)
records = self.__mozilla_sync.get_records("bookmarks", bulk_keys)
children_array = []
for record in records:
self.__check_worker()
if record["modified"] < self.__mtimes["bookmarks"]:
continue
sleep(0.01)
bookmark = record["payload"]
bookmark_id = El().bookmarks.get_id_by_guid(bookmark["id"])
# Nothing to apply, continue
if El().bookmarks.get_mtime(bookmark_id) >= record["modified"]:
continue
debug("pulling %s" % record)
# Deleted bookmark
if "deleted" in bookmark.keys():
El().bookmarks.remove(bookmark_id)
# Keep folder only for firefox compatiblity
elif "type" in bookmark.keys() and bookmark["type"] == "folder"\
and bookmark["id"] is not None\
and bookmark["title"]:
if bookmark_id is None:
bookmark_id = El().bookmarks.add(bookmark["title"],
bookmark["id"],
bookmark["id"],
[],
0,
False)
# Will calculate position later
if "children" in bookmark.keys():
children_array.append(bookmark["children"])
# We have a bookmark, add it
elif "type" in bookmark.keys() and bookmark["type"] == "bookmark"\
and bookmark["id"] is not None\
and bookmark["title"]:
# Add a new bookmark
if bookmark_id is None:
# Use parent name if no bookmarks tags
if "tags" not in bookmark.keys() or\
not bookmark["tags"]:
if "parentName" in bookmark.keys() and\
bookmark["parentName"]:
bookmark["tags"] = [bookmark["parentName"]]
else:
bookmark["tags"] = []
bookmark_id = El().bookmarks.add(bookmark["title"],
bookmark["bmkUri"],
bookmark["id"],
bookmark["tags"],
0,
False)
# Update bookmark
else:
El().bookmarks.set_title(bookmark_id,
bookmark["title"],
False)
El().bookmarks.set_uri(bookmark_id,
bookmark["bmkUri"],
False)
# Update tags
current_tags = El().bookmarks.get_tags(bookmark_id)
for tag in El().bookmarks.get_tags(bookmark_id):
if "tags" in bookmark.keys() and\
tag not in bookmark["tags"]:
tag_id = El().bookmarks.get_tag_id(tag)
current_tags.remove(tag)
El().bookmarks.del_tag_from(tag_id,
bookmark_id,
False)
if "tags" in bookmark.keys():
for tag in bookmark["tags"]:
# Tag already associated
if tag in current_tags:
continue
tag_id = El().bookmarks.get_tag_id(tag)
if tag_id is None:
tag_id = El().bookmarks.add_tag(tag, False)
El().bookmarks.add_tag_to(tag_id,
bookmark_id,
False)
# Update parent name if available
if bookmark_id is not None and "parentName" in bookmark.keys():
El().bookmarks.set_parent(bookmark_id,
bookmark["parentid"],
bookmark["parentName"],
False)
El().bookmarks.set_mtime(bookmark_id,
record["modified"],
False)
# Update bookmark position
for children in children_array:
position = 0
for child in children:
bid = El().bookmarks.get_id_by_guid(child)
El().bookmarks.set_position(bid,
position,
False)
position += 1
El().bookmarks.clean_tags() # Will commit
SqlCursor.remove(El().bookmarks)
def __pull_passwords(self, bulk_keys):
"""
Pull from passwords
@param bulk_keys as KeyBundle
@raise StopIteration
"""
debug("pull passwords")
records = self.__mozilla_sync.get_records("passwords", bulk_keys)
for record in records:
self.__check_worker()
if record["modified"] < self.__mtimes["passwords"]:
continue
sleep(0.01)
debug("pulling %s" % record)
password = record["payload"]
password_id = password["id"].strip("{}")
if "formSubmitURL" in password.keys():
self.__helper.clear(password_id,
self.__helper.store,
password["usernameField"],
password["username"],
password["passwordField"],
password["password"],
password["hostname"],
password["formSubmitURL"],
password_id,
None)
elif "deleted" in password.keys(): # We assume True
self.__helper.clear(password_id)
def __pull_history(self, bulk_keys):
"""
Pull from history
@param bulk_keys as KeyBundle
@raise StopIteration
"""
debug("pull history")
records = self.__mozilla_sync.get_records("history", bulk_keys)
for record in records:
self.__check_worker()
if record["modified"] < self.__mtimes["history"]:
continue
sleep(0.01)
El().history.thread_lock.acquire()
history = record["payload"]
keys = history.keys()
history_id = El().history.get_id_by_guid(history["id"])
# Check we have a valid history item
if "histUri" in keys and\
"title" in keys and\
history["title"] and\
El().history.get_mtime(history_id) < record["modified"]:
# Try to get visit date
atimes = []
try:
for visit in history["visits"]:
atimes.append(round(int(visit["date"]) / 1000000, 2))
except:
El().history.thread_lock.release()
continue
debug("pulling %s" % record)
title = history["title"].rstrip().lstrip()
history_id = El().history.add(title,
history["histUri"],
record["modified"],
history["id"],
atimes,
True)
elif "deleted" in keys:
history_id = El().history.get_id_by_guid(history_id)
El().history.remove(history_id)
El().history.thread_lock.release()
def __set_credentials(self, attributes, password, uri, index, count):
"""
Set credentials
@param attributes as {}
@param password as str
@param uri as None
@param index as int
@param count as int
"""
if attributes is None:
return
from base64 import b64decode
try:
self.__username = attributes["login"]
self.__password = password
self.__token = attributes["token"]
self.__uid = attributes["uid"]
self.__keyB = b64decode(attributes["keyB"])
# Force login if no token
if not self.__token:
self.login(attributes, password)
except Exception as e:
debug("SyncWorker::__set_credentials(): %s" % e)
def __check_worker(self):
"""
Raise an exception if worker should not be syncing: error&cancel
"""
if self.__sync_cancellable.is_cancelled():
raise StopIteration("SyncWorker: cancelled")
elif not self.__username:
raise StopIteration("SyncWorker: missing username")
elif not self.__password:
raise StopIteration("SyncWorker: missing password")
elif not self.__token:
raise StopIteration("SyncWorker: missing token")
class SyncWorker(GObject.GObject):
"""
Manage sync with mozilla server, will start syncing on init
"""
__gsignals__ = {"sync-finished": (GObject.SignalFlags.RUN_FIRST, None, ())}
def __init__(self):
"""
Init worker
"""
GObject.GObject.__init__(self)
self.__stop = True
self.__username = ""
self.__password = ""
self.__mtimes = {"bookmarks": 0.1, "history": 0.1}
self.__mozilla_sync = MozillaSync()
self.__session = None
self.__helper = PasswordsHelper()
def login(self, attributes, password, uri):
"""
Login to service
@param attributes as {}
@param password as str
@param uri as None
@raise exceptions
"""
if attributes is None:
return
keyB = ""
session = None
# Connect to mozilla sync
session = self.__mozilla_sync.login(attributes["login"], password)
bid_assertion, key = self.__mozilla_sync.get_browserid_assertion(
session)
keyB = base64.b64encode(session.keys[1]).decode("utf-8")
# Store credentials
if session is None:
uid = ""
token = ""
else:
uid = session.uid
token = session.token
self.__helper.store_sync(attributes["login"], password, uid, token,
keyB, lambda x, y: self.sync(True))
def sync(self, first_sync=False):
"""
Start syncing, you need to check sync_status property
@param first_sync as bool
"""
if self.syncing or\
not Gio.NetworkMonitor.get_default().get_network_available():
return
self.__username = ""
self.__password = ""
self.__stop = False
# We force session reset to user last stored token
if first_sync:
self.__session = None
self.__helper.get_sync(self.__start_sync, first_sync)
def push_history(self, history_ids):
"""
Push history ids
@param history_ids as [int]
"""
if Gio.NetworkMonitor.get_default().get_network_available():
thread = Thread(target=self.__push_history, args=(history_ids, ))
thread.daemon = True
thread.start()
def push_password(self, username, userform, password, passform, uri, uuid):
"""
Push password
@param username as str
@param userform as str
@param password as str
@param passform as str
@param uri as str
@param uuid as str
"""
if Gio.NetworkMonitor.get_default().get_network_available():
thread = Thread(target=self.__push_password,
args=(username, userform, password, passform, uri,
uuid))
thread.daemon = True
thread.start()
def remove_from_history(self, guid):
"""
Remove history id from remote history
A first call to sync() is needed to populate secrets
@param guid as str
"""
if Gio.NetworkMonitor.get_default().get_network_available():
thread = Thread(target=self.__remove_from_history, args=(guid, ))
thread.daemon = True
thread.start()
def remove_from_passwords(self, uri):
"""
Remove password from passwords collection
@param uri as str
"""
if Gio.NetworkMonitor.get_default().get_network_available():
self.__helper.get(uri, self.__remove_from_passwords_thread)
def delete_secret(self):
"""
Delete sync secret
"""
self.__username = ""
self.__password = ""
self.__session = None
self.__stop = True
self.__helper.clear_sync()
def stop(self):
"""
Stop update
"""
self.__stop = True
@property
def mtimes(self):
"""
Sync engine modification times
@return {}
"""
return self.__mtimes
@property
def syncing(self):
"""
True if sync is running
@return bool
"""
return not self.__stop
@property
def status(self):
"""
True if sync is working
@return bool
"""
try:
self.__mozilla_sync.client.info_collections()
return True
except:
return False
@property
def username(self):
"""
Get username
@return str
"""
return self.__username
#######################
# PRIVATE #
#######################
def __get_session_bulk_keys(self):
"""
Get session decrypt keys
@return keys as (b"", b"")
"""
if self.__session is None:
self.__session = FxASession(
self.__mozilla_sync.fxa_client, self.__username,
quick_stretch_password(self.__username, self.__password),
self.__uid, self.__token)
self.__session.keys = [b"", self.__keyB]
self.__session.check_session_status()
bid_assertion, key = self.__mozilla_sync.get_browserid_assertion(
self.__session)
bulk_keys = self.__mozilla_sync.connect(bid_assertion, key)
return bulk_keys
def __push_history(self, history_ids):
"""
Push history ids if atime is available, else, ask to remove
@param history ids as [int]
"""
if not self.__username or not self.__password:
return
try:
bulk_keys = self.__get_session_bulk_keys()
for history_id in history_ids:
sleep(0.01)
record = {}
atimes = El().history.get_atimes(history_id)
guid = El().history.get_guid(history_id)
if atimes:
record["histUri"] = El().history.get_uri(history_id)
record["id"] = guid
record["title"] = El().history.get_title(history_id)
record["visits"] = []
for atime in atimes:
record["visits"].append({
"date": atime * 1000000,
"type": 1
})
debug("pushing %s" % record)
self.__mozilla_sync.add(record, "history", bulk_keys)
else:
record["id"] = guid
record["type"] = "item"
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "history", bulk_keys)
self.__mtimes = self.__mozilla_sync.client.info_collections()
dump(self.__mtimes, open(LOCAL_PATH + "/mozilla_sync.bin",
"wb"))
except Exception as e:
print("SyncWorker::__push_history():", e)
def __push_password(self, username, userform, password, passform, uri,
uuid):
"""
Push password
@param username as str
@param userform as str
@param password as str
@param passform as str
@param uri as str
@param uuid as str
"""
if not self.__username or not self.__password:
return
try:
bulk_keys = self.__get_session_bulk_keys()
record = {}
record["id"] = "{%s}" % uuid
record["hostname"] = uri
record["formSubmitURL"] = uri
record["httpRealm"] = None
record["username"] = username
record["password"] = password
record["usernameField"] = userform
record["passwordField"] = passform
mtime = int(time() * 1000)
record["timeCreated"] = mtime
record["timePasswordChanged"] = mtime
debug("pushing %s" % record)
self.__mozilla_sync.add(record, "passwords", bulk_keys)
except Exception as e:
print("SyncWorker::__push_password():", e)
def __remove_from_history(self, guid):
"""
Remove from history
@param guid as str
"""
if not self.__username or not self.__password:
return
try:
bulk_keys = self.__get_session_bulk_keys()
record = {}
record["id"] = guid
record["type"] = "item"
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "history", bulk_keys)
except Exception as e:
print("SyncWorker::__remove_from_history():", e)
def __remove_from_passwords(self, attributes, password, uri):
"""
Remove password from passwords collection
@param attributes as {}
@param password as str
@param uri as str
"""
if not self.__username or not self.__password:
return
try:
bulk_keys = self.__get_session_bulk_keys()
record = {}
record["id"] = attributes["uuid"]
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "passwords", bulk_keys)
self.__helper.clear(uri)
except Exception as e:
print("SyncWorker::__remove_from_passwords():", e)
def __remove_from_passwords_thread(self, attributes, password, uri):
"""
Remove password from passwords collection
@param attributes as {}
@param password as str
@param uri as str
"""
thread = Thread(target=self.__remove_from_passwords,
args=(attributes, password, uri))
thread.daemon = True
thread.start()
def __sync(self, first_sync):
"""
Sync Eolie objects (bookmarks, history, ...) with Mozilla Sync
@param first_sync as bool
"""
debug("Start syncing")
if not self.__username or not self.__password:
self.__stop = True
return
try:
self.__mtimes = load(open(LOCAL_PATH + "/mozilla_sync.bin", "rb"))
except:
self.__mtimes = {
"bookmarks": 0.1,
"history": 0.1,
"passwords": 0.1
}
try:
bulk_keys = self.__get_session_bulk_keys()
new_mtimes = self.__mozilla_sync.client.info_collections()
if self.__stop:
return
######################
# Passwords Management #
######################
debug("local passwords: %s, remote passwords: %s" %
(self.__mtimes["passwords"], new_mtimes["passwords"]))
# Only pull if something new available
if self.__mtimes["passwords"] != new_mtimes["passwords"]:
self.__pull_passwords(bulk_keys)
if self.__stop:
return
######################
# History Management #
######################
debug("local history: %s, remote history: %s" %
(self.__mtimes["history"], new_mtimes["history"]))
# Only pull if something new available
if self.__mtimes["history"] != new_mtimes["history"]:
self.__pull_history(bulk_keys)
if self.__stop:
return
########################
# Bookmarks Management #
########################
debug("local bookmarks: %s, remote bookmarks: %s" %
(self.__mtimes["bookmarks"], new_mtimes["bookmarks"]))
# Push new bookmarks
self.__push_bookmarks(bulk_keys)
if self.__stop:
return
# Only pull if something new available
if self.__mtimes["bookmarks"] != new_mtimes["bookmarks"]:
self.__pull_bookmarks(bulk_keys, first_sync)
# Update last sync mtime
self.__mtimes = self.__mozilla_sync.client.info_collections()
dump(self.__mtimes, open(LOCAL_PATH + "/mozilla_sync.bin", "wb"))
debug("Stop syncing")
GLib.idle_add(self.emit, "sync-finished")
except Exception as e:
print("SyncWorker::__sync():", e)
if str(e) == "The authentication token could not be found":
self.__helper.get_sync(self.login)
self.__stop = True
def __push_bookmarks(self, bulk_keys):
"""
Push to bookmarks
@param bulk keys as KeyBundle
@param start time as float
@raise StopIteration
"""
debug("push bookmarks")
parents = []
for bookmark_id in El().bookmarks.get_ids_for_mtime(
self.__mtimes["bookmarks"]):
if self.__stop:
raise StopIteration("Cancelled")
sleep(0.01)
parent_guid = El().bookmarks.get_parent_guid(bookmark_id)
# No parent, move it to unfiled
if parent_guid is None:
parent_guid = "unfiled"
parent_id = El().bookmarks.get_id_by_guid(parent_guid)
if parent_id not in parents:
parents.append(parent_id)
record = {}
record["bmkUri"] = El().bookmarks.get_uri(bookmark_id)
record["id"] = El().bookmarks.get_guid(bookmark_id)
record["title"] = El().bookmarks.get_title(bookmark_id)
record["tags"] = El().bookmarks.get_tags(bookmark_id)
record["parentid"] = parent_guid
record["type"] = "bookmark"
debug("pushing %s" % record)
self.__mozilla_sync.add(record, "bookmarks", bulk_keys)
# Del old bookmarks
for bookmark_id in El().bookmarks.get_deleted_ids():
if self.__stop:
raise StopIteration("Cancelled")
sleep(0.01)
parent_guid = El().bookmarks.get_parent_guid(bookmark_id)
parent_id = El().bookmarks.get_id_by_guid(parent_guid)
if parent_id not in parents:
parents.append(parent_id)
record = {}
record["id"] = El().bookmarks.get_guid(bookmark_id)
record["type"] = "item"
record["deleted"] = True
debug("deleting %s" % record)
self.__mozilla_sync.add(record, "bookmarks", bulk_keys)
El().bookmarks.remove(bookmark_id)
# Push parents in this order, parents near root are handled later
# Otherwise, order will be broken by new children updates
while parents:
parent_id = parents.pop(0)
parent_guid = El().bookmarks.get_guid(parent_id)
parent_name = El().bookmarks.get_title(parent_id)
children = El().bookmarks.get_children(parent_guid)
# So search if children in parents
found = False
for child_guid in children:
child_id = El().bookmarks.get_id_by_guid(child_guid)
if child_id in parents:
found = True
break
# Handle children first
if found:
parents.append(parent_id)
debug("later: %s" % parent_name)
continue
record = {}
record["id"] = parent_guid
record["type"] = "folder"
# A parent with parent as unfiled needs to be moved to places
# Firefox internal
grand_parent_guid = El().bookmarks.get_parent_guid(parent_id)
if grand_parent_guid == "unfiled":
grand_parent_guid = "places"
record["parentid"] = grand_parent_guid
record["parentName"] = El().bookmarks.get_parent_name(parent_id)
record["title"] = parent_name
record["children"] = children
debug("pushing parent %s" % record)
self.__mozilla_sync.add(record, "bookmarks", bulk_keys)
El().bookmarks.clean_tags()
def __pull_bookmarks(self, bulk_keys, first_sync):
"""
Pull from bookmarks
@param bulk_keys as KeyBundle
@param first_sync as bool
@raise StopIteration
"""
debug("pull bookmarks")
SqlCursor.add(El().bookmarks)
records = self.__mozilla_sync.get_records("bookmarks", bulk_keys)
# We get all guids here and remove them while sync
# At the end, we have deleted records
# On fist sync, keep all
if first_sync:
to_delete = []
else:
to_delete = El().bookmarks.get_guids()
for record in records:
if self.__stop:
raise StopIteration("Cancelled")
sleep(0.01)
bookmark = record["payload"]
if "type" not in bookmark.keys() or\
bookmark["type"] not in ["folder", "bookmark"]:
continue
bookmark_id = El().bookmarks.get_id_by_guid(bookmark["id"])
# This bookmark exists, remove from to delete
if bookmark["id"] in to_delete:
to_delete.remove(bookmark["id"])
# Nothing to apply, continue
if El().bookmarks.get_mtime(bookmark_id) >= record["modified"]:
continue
debug("pulling %s" % record)
if bookmark_id is None:
if "bmkUri" in bookmark.keys():
# Use parent name if no bookmarks tags
if "tags" not in bookmark.keys() or\
not bookmark["tags"]:
if "parentName" in bookmark.keys() and\
bookmark["parentName"]:
bookmark["tags"] = [bookmark["parentName"]]
else:
bookmark["tags"] = []
bookmark_id = El().bookmarks.add(bookmark["title"],
bookmark["bmkUri"],
bookmark["id"],
bookmark["tags"], False)
else:
bookmark["tags"] = []
bookmark_id = El().bookmarks.add(bookmark["title"],
bookmark["id"],
bookmark["id"],
bookmark["tags"], False)
else:
El().bookmarks.set_title(bookmark_id, bookmark["title"], False)
if "bmkUri" in bookmark.keys():
El().bookmarks.set_uri(bookmark_id, bookmark["bmkUri"],
False)
elif "children" in bookmark.keys():
position = 0
for child in bookmark["children"]:
bid = El().bookmarks.get_id_by_guid(child)
El().bookmarks.set_position(bid, position, False)
position += 1
# Remove previous tags
current_tags = El().bookmarks.get_tags(bookmark_id)
for tag in El().bookmarks.get_tags(bookmark_id):
if "tags" in bookmark.keys() and\
tag not in bookmark["tags"]:
tag_id = El().bookmarks.get_tag_id(tag)
current_tags.remove(tag)
El().bookmarks.del_tag_from(tag_id, bookmark_id, False)
if "tags" in bookmark.keys():
for tag in bookmark["tags"]:
# Tag already associated
if tag in current_tags:
continue
tag_id = El().bookmarks.get_tag_id(tag)
if tag_id is None:
tag_id = El().bookmarks.add_tag(tag, False)
El().bookmarks.add_tag_to(tag_id, bookmark_id, False)
El().bookmarks.set_mtime(bookmark_id, record["modified"], False)
if "parentName" in bookmark.keys():
El().bookmarks.set_parent(bookmark_id, bookmark["parentid"],
bookmark["parentName"], False)
for guid in to_delete:
if self.__stop:
raise StopIteration("Cancelled")
debug("deleting: %s" % guid)
bookmark_id = El().bookmarks.get_id_by_guid(guid)
if bookmark_id is not None:
El().bookmarks.remove(bookmark_id, False)
El().bookmarks.clean_tags() # Will commit
SqlCursor.remove(El().bookmarks)
def __pull_passwords(self, bulk_keys):
"""
Pull from passwords
@param bulk_keys as KeyBundle
@raise StopIteration
"""
debug("pull passwords")
records = self.__mozilla_sync.get_records("passwords", bulk_keys)
for record in records:
if self.__stop:
raise StopIteration("Cancelled")
sleep(0.01)
debug("pulling %s" % record)
password = record["payload"]
if "formSubmitURL" in password.keys():
self.__helper.clear(password["formSubmitURL"])
self.__helper.store(password["username"], password["password"],
password["formSubmitURL"], password["id"],
None)
elif "deleted" in password.keys(): # We assume True
self.__helper.clear(password["id"])
def __pull_history(self, bulk_keys):
"""
Pull from history
@param bulk_keys as KeyBundle
@raise StopIteration
"""
debug("pull history")
records = self.__mozilla_sync.get_records("history", bulk_keys)
for record in records:
if self.__stop:
raise StopIteration("Cancelled")
sleep(0.01)
El().history.thread_lock.acquire()
history = record["payload"]
keys = history.keys()
# Ignore pages without a title
if "title" not in keys or not history["title"]:
El().history.thread_lock.release()
continue
# Ignore pages without an uri (deleted)
if "histUri" not in keys:
El().history.thread_lock.release()
continue
history_id = El().history.get_id_by_guid(history["id"])
# Nothing to apply, continue
if El().history.get_mtime(history_id) >= record["modified"]:
El().history.thread_lock.release()
continue
# Try to get visit date
atimes = []
try:
for visit in history["visits"]:
atimes.append(round(int(visit["date"]) / 1000000, 2))
except:
El().history.thread_lock.release()
continue
debug("pulling %s" % record)
title = history["title"].rstrip().lstrip()
history_id = El().history.add(title, history["histUri"],
record["modified"], history["id"],
atimes, True)
El().history.thread_lock.release()
def __start_sync(self, attributes, password, uri, first_sync):
"""
Set params and start sync
@param attributes as {}
@param password as str
@param uri as None
@param first_sync as bool
"""
if attributes is None:
return
try:
self.__username = attributes["login"]
self.__password = password
self.__token = attributes["token"]
self.__uid = attributes["uid"]
self.__keyB = base64.b64decode(attributes["keyB"])
if Gio.NetworkMonitor.get_default().get_network_available():
thread = Thread(target=self.__sync, args=(first_sync, ))
thread.daemon = True
thread.start()
except Exception as e:
print("SyncWorker::__start_sync()", e)