def __install_engine(self, uri, window):
"""
Install engine from uri
@param uri as str
@param window as Window
"""
task_helper = TaskHelper(self.__user_agent)
task_helper.load_uri_content(uri, None, self.__on_engine_loaded,
window)
def search_suggestions(self, value, cancellable, callback):
"""
Search suggestions for value
@param value as str
@param cancellable as Gio.Cancellable
@param callback as str
"""
try:
if not value.strip(" "):
return
uri = self.__keyword % value
task_helper = TaskHelper()
task_helper.load_uri_content(uri, cancellable, callback,
self.__encoding, value)
except Exception as e:
print("Search::search_suggestions():", e)
def search_suggestions(self, value, cancellable, callback):
"""
Search suggestions for value
@param value as str
@param cancellable as Gio.Cancellable
@param callback as str
"""
try:
if not value.strip(" "):
return
uri = self.__suggest % GLib.uri_escape_string(value, None, True)
task_helper = TaskHelper(self.__user_agent)
task_helper.load_uri_content(uri, cancellable, callback,
self.__encoding, value)
except Exception as e:
Logger.error("Search::search_suggestions(): %s", e)
class ImagesPopover(Gtk.Popover):
"""
Show images for page id
"""
def __init__(self, uri, page_id, window):
"""
Init popover
@param uri as str
@param page_id as int
@param window as Window
"""
Gtk.Popover.__init__(self)
self.set_modal(False)
window.register(self)
self.__cache_uris = []
self.__uri = uri
self.__page_id = page_id
self.__cancellable = Gio.Cancellable()
self.__filter = ""
builder = Gtk.Builder()
builder.add_from_resource("/org/gnome/Eolie/PopoverImages.ui")
builder.connect_signals(self)
widget = builder.get_object("widget")
self.__spinner = builder.get_object("spinner")
self.__flowbox = builder.get_object("flowbox")
self.__flowbox.set_filter_func(self.__filter_func)
self.__entry = builder.get_object("entry")
self.__button = builder.get_object("button")
self.add(widget)
if Gio.NetworkMonitor.get_default().get_network_available():
El().helper.call("GetImages", page_id, None, self.__on_get_images)
(width, height) = El().active_window.get_size()
self.set_size_request(width / 2, height / 1.5)
self.connect("closed", self.__on_closed)
self.__task_helper = TaskHelper()
#######################
# PROTECTED #
#######################
def _on_search_changed(self, entry):
"""
Filter flowbox
@param entry as Gtk.Entry
"""
self.__filter = entry.get_text()
self.__flowbox.invalidate_filter()
def _on_button_clicked(self, button):
"""
Save visible images
@param button as Gtk.Button
"""
task_helper = TaskHelper()
task_helper.run(self.__move_images)
self.__spinner.start()
def _on_button_toggled(self, button):
"""
Cancel previous download
"""
self.__cancellable.cancel()
self.__spinner.start()
self.__button.set_sensitive(False)
for child in self.__flowbox.get_children():
child.destroy()
self.__links = button.get_active()
if Gio.NetworkMonitor.get_default().get_network_available():
if button.get_active():
El().helper.call("GetImageLinks", self.__page_id, None,
self.__on_get_images)
else:
El().helper.call("GetImages", self.__page_id, None,
self.__on_get_images)
#######################
# PRIVATE #
#######################
def __filter_func(self, child):
"""
Filter child
@param child as image
"""
if child.uri.find(self.__filter) != -1:
return True
def __add_image(self, uri):
"""
Add a child to flowbox
@param uri as str
"""
image = Image(uri)
image.show()
self.__flowbox.add(image)
def __move_images(self):
"""
Move image to download directory
"""
parsed = urlparse(self.__uri)
directory_uri = El().settings.get_value('download-uri').get_string()
if not directory_uri:
directory = GLib.get_user_special_dir(
GLib.UserDirectory.DIRECTORY_DOWNLOAD)
directory_uri = GLib.filename_to_uri(directory, None)
destination_uri = "%s/%s" % (directory_uri, parsed.netloc)
directory = Gio.File.new_for_uri(destination_uri)
if not directory.query_exists():
directory.make_directory_with_parents()
for child in self.__flowbox.get_children():
if child.uri.find(self.__filter) != -1:
encoded = sha256(child.uri.encode("utf-8")).hexdigest()
child_basename = child.uri.split("/")[-1]
filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded)
s = Gio.File.new_for_path(filepath)
if not s.query_exists():
continue
d = Gio.File.new_for_uri("%s/%s" %
(destination_uri, child_basename))
try:
s.move(d, Gio.FileCopyFlags.OVERWRITE, None, None, None)
except Exception as e:
print("ImagesPopover::__move_images()", e)
GLib.idle_add(self.hide)
def __clean_cache(self):
"""
Clean the cache
"""
for uri in self.__cache_uris:
encoded = sha256(uri.encode("utf-8")).hexdigest()
filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded)
f = Gio.File.new_for_path(filepath)
try:
if f.query_exists():
f.delete()
except Exception as e:
print("ImagesPopover::__clean_cache():", e)
def __on_write_all_async(self, stream, result, uri):
"""
Add image
@param stream as Gio.OutputStream
@param result as Gio.AsyncResult
@param uri as str
"""
try:
stream.write_all_finish(result)
self.__add_image(uri)
self.__cache_uris.append(uri)
except Exception as e:
print("ImagesPopover::__on_write_all_async()", e)
def __on_load_uri_content(self, uri, status, content, uris):
"""
Load pending uris
@param uri as str
@param status as bool
@param content as bytes
@param uris as [str]
"""
if status:
encoded = sha256(uri.encode("utf-8")).hexdigest()
filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded)
f = Gio.File.new_for_path(filepath)
stream = f.append_to(Gio.FileCreateFlags.REPLACE_DESTINATION,
self.__cancellable)
stream.write_all_async(content, GLib.PRIORITY_DEFAULT,
self.__cancellable,
self.__on_write_all_async, uri)
if uris and not self.__cancellable.is_cancelled():
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri, self.__cancellable,
self.__on_load_uri_content,
uris)
else:
self.__spinner.stop()
self.__button.set_sensitive(True)
def __on_get_images(self, source, result):
"""
Get result and load pending uris
@param source as GObject.Object
@param result as Gio.AsyncResult
"""
uris = []
try:
uris = source.call_finish(result)[0]
except Exception as e:
print("ImagesPopover::__on_get_images()", e)
self.__cancellable.reset()
self.__on_load_uri_content(None, False, b"", uris)
def __on_closed(self, popover):
"""
Clean cache
"""
self.__spinner.stop()
self.__cancellable.cancel()
self.__task_helper.run(self.__clean_cache)
class DatabasePhishing:
"""
Phishing database
"""
__DB_PATH = "%s/phishing.db" % EOLIE_DATA_PATH
__URI = "http://data.phishtank.com/data/online-valid.json"
__SCHEMA_VERSION = 0
__UPDATE = 172800
# SQLite documentation:
# In SQLite, a column with type INTEGER PRIMARY KEY
# is an alias for the ROWID.
# Here, we define an id INT PRIMARY KEY but never feed it,
# this make VACUUM not destroy rowids...
__create_phishing = '''CREATE TABLE phishing (
id INTEGER PRIMARY KEY,
uri TEXT NOT NULL,
mtime INT NOT NULL
)'''
__create_phishing_idx = """CREATE UNIQUE INDEX idx_phishing ON phishing(
uri)"""
def __init__(self):
"""
Create database tables or manage update if needed
"""
self.thread_lock = Lock()
self.__cancellable = Gio.Cancellable.new()
self.__task_helper = TaskHelper()
self.__phishing_mtime = int(time())
self.__regex = None
def create_db(self):
"""
Create databse
"""
if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR):
GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750)
# If DB schema changed, remove it
if GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR):
with SqlCursor(self) as sql:
result = sql.execute("PRAGMA user_version")
v = result.fetchone()
if v is None or v[0] != self.__SCHEMA_VERSION:
f = Gio.File.new_for_path(self.__DB_PATH)
f.delete()
if not GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR):
try:
# Create db schema
with SqlCursor(self) as sql:
sql.execute(self.__create_phishing)
sql.execute(self.__create_phishing_idx)
sql.execute("PRAGMA user_version=%s" %
self.__SCHEMA_VERSION)
except Exception as e:
Logger.error("DatabasePhishing::__init__(): %s", e)
def update(self):
"""
Update database
"""
if not Gio.NetworkMonitor.get_default().get_network_available():
return
# DB version is last successful sync mtime
try:
version = load(open(EOLIE_DATA_PATH + "/phishing.bin", "rb"))
except:
version = 0
self.__cancellable.reset()
if self.__phishing_mtime - version > self.__UPDATE:
self.__on_load_uri_content(None, False, b"", [self.__URI])
def is_phishing(self, uri):
"""
True if uri is phishing
@param uri as str
@return bool
"""
uri = uri.rstrip("/")
try:
with SqlCursor(self) as sql:
result = sql.execute(
"SELECT uri FROM phishing\
WHERE uri=?", (uri, ))
v = result.fetchone()
return v is not None
except Exception as e:
Logger.error("DatabasePhishing::is_phishing(): %s", e)
return False
def stop(self):
"""
Stop update
"""
self.__cancellable.cancel()
self.__stop = True
def get_cursor(self):
"""
Return a new sqlite cursor
"""
try:
c = sqlite3.connect(self.__DB_PATH, 600.0)
return c
except Exception as e:
Logger.error("DatabasePhishing::get_cursor(): %s", e)
exit(-1)
#######################
# PRIVATE #
#######################
def __save_rules(self, rules, uris):
"""
Save rules to db
@param rules as bytes
@param uris as [str]
"""
SqlCursor.add(self)
try:
result = rules.decode('utf-8')
j = json.loads(result)
with SqlCursor(self) as sql:
count = 0
for item in j:
if self.__cancellable.is_cancelled():
raise IOError("Cancelled")
uri = item["url"].rstrip("/")
try:
sql.execute(
"INSERT INTO phishing\
(uri, mtime) VALUES (?, ?)",
(uri, self.__phishing_mtime))
except:
sql.execute(
"UPDATE phishing set mtime=?\
WHERE uri=?",
(self.__phishing_mtime, uri))
count += 1
if count == 1000:
SqlCursor.commit(self)
# Do not flood sqlite
# this allow webkit extension to run
sleep(0.1)
count = 0
# We are the last call to save_rules()?
# Delete removed entries and commit
if not uris:
with SqlCursor(self) as sql:
sql.execute(
"DELETE FROM phishing\
WHERE mtime!=?", (self.__phishing_mtime, ))
try:
dump(self.__phishing_mtime,
open(EOLIE_DATA_PATH + "/phishing.bin", "wb"))
except Exception as e:
Logger.error("DatabasePhishing::__save_rules(): %s", e)
except Exception as e:
Logger.error("DatabasePhishing::__save_rules():%s -> %s", e, rules)
SqlCursor.remove(self)
def __on_load_uri_content(self, uri, status, content, uris):
"""
Load pending uris
@param uri as str
@param status as bool
@param content as bytes
@param uris as [str]
"""
if status:
self.__task_helper.run(self.__save_rules, content, uris)
if uris:
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri, self.__cancellable,
self.__on_load_uri_content,
uris)
class DatabaseAdblock:
"""
Eolie adblock db
"""
DB_PATH = "%s/adblock2.db" % EOLIE_DATA_PATH
__URIS = [
"https://adaway.org/hosts.txt",
"https://pgl.yoyo.org/adservers/serverlist.php?" +
"hostformat=hosts&showintro=0&startdate%5Bday%5D=" +
"&startdate%5Bmonth%5D=&startdate%5Byear%5D=",
"http://winhelp2002.mvps.org/hosts.txt",
"http://hosts-file.net/ad_servers.txt",
"https://pgl.yoyo.org/adservers/serverlist.php?"
"hostformat=hosts&showintro=0&mimetype=plaintext"
]
__CSS_URIS = ["https://easylist-downloads.adblockplus.org/easylist.txt"]
__CSS_LOCALIZED_URIS = {
"bg":
"http://stanev.org/abp/adblock_bg.txt",
"zh":
"https://easylist-downloads.adblockplus.org/easylistchina.txt",
"sk":
"https://raw.github.com/tomasko126/" +
"easylistczechandslovak/master/filters.txt",
"cs":
"https://raw.github.com/tomasko126/" +
"easylistczechandslovak/master/filters.txt",
"nl":
"https://easylist-downloads.adblockplus.org/easylistdutch.txt",
"de":
"https://easylist-downloads.adblockplus.org/easylistgermany.txt",
"he":
"https://raw.githubusercontent.com/easylist/" +
"EasyListHebrew/master/EasyListHebrew.txt",
"it":
"https://easylist-downloads.adblockplus.org/easylistitaly.txt",
"lt":
"http://margevicius.lt/easylistlithuania.txt",
"es":
"https://easylist-downloads.adblockplus.org/easylistspanish.txt",
"lv":
"https://notabug.org/latvian-list/" +
"adblock-latvian/raw/master/lists/latvian-list.txt",
"ar":
"https://easylist-downloads.adblockplus.org/Liste_AR.txt",
"fr":
"https://easylist-downloads.adblockplus.org/liste_fr.txt",
"ro":
"http://www.zoso.ro/pages/rolist.txt",
"ru":
"https://easylist-downloads.adblockplus.org/advblock.txt",
"ja":
"http://bit.ly/11QrCfx",
"fi":
"https://adb.juvander.net/Finland_adb.txt",
"cz":
"http://adblock.dajbych.net/adblock.txt",
"et":
"http://gurud.ee/ab.txt",
"hu":
"https://raw.githubusercontent.com/szpeter80/" +
"hufilter/master/hufilter.txt"
}
__UPDATE = 172800
# SQLite documentation:
# In SQLite, a column with type INTEGER PRIMARY KEY
# is an alias for the ROWID.
# Here, we define an id INT PRIMARY KEY but never feed it,
# this make VACUUM not destroy rowids...
__create_adblock = '''CREATE TABLE adblock (
id INTEGER PRIMARY KEY,
dns TEXT NOT NULL,
mtime INT NOT NULL
)'''
__create_adblock_css = '''CREATE TABLE adblock_css (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
whitelist TEXT DEFAULT "",
blacklist TEXT DEFAULT "",
mtime INT NOT NULL
)'''
def __init__(self):
"""
Create database tables or manage update if needed
"""
self.__cancellable = Gio.Cancellable.new()
self.__task_helper = TaskHelper()
self.__adblock_mtime = int(time())
# Lazy loading if not empty
if not GLib.file_test(self.DB_PATH, GLib.FileTest.IS_REGULAR):
try:
if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR):
GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750)
# Create db schema
with SqlCursor(self) as sql:
sql.execute(self.__create_adblock)
sql.execute(self.__create_adblock_css)
sql.commit()
except Exception as e:
print("DatabaseAdblock::__init__(): %s" % e)
def update(self):
"""
Update database
"""
if not Gio.NetworkMonitor.get_default().get_network_available():
return
# Update adblock_js repo
git = GLib.find_program_in_path("git")
if git is None:
print(_("For stronger ad blocking, install git command"))
else:
if GLib.file_test(ADBLOCK_JS, GLib.FileTest.IS_DIR):
argv = [git, "-C", ADBLOCK_JS, "pull"]
else:
argv = [
git, "clone",
"https://gitlab.gnome.org/gnumdk/eolie-adblock.git",
ADBLOCK_JS
]
(pid, a1, a2,
a3) = GLib.spawn_async(argv,
flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL)
GLib.spawn_close_pid(pid)
# Check entries in DB, do we need to update?
mtime = 0
with SqlCursor(self) as sql:
result = sql.execute("SELECT mtime FROM adblock\
ORDER BY mtime LIMIT 1")
v = result.fetchone()
if v is not None:
mtime = v[0]
self.__cancellable.reset()
if self.__adblock_mtime - mtime > self.__UPDATE:
# Update host rules
uris = list(self.__URIS)
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri, self.__cancellable,
self.__on_load_uri_content,
uris)
else:
self.__on_save_rules()
def stop(self):
"""
Stop update
"""
self.__cancellable.cancel()
def get_default_css_rules(self):
"""
Return default css rules
"""
rules = ""
with SqlCursor(self) as sql:
request = "SELECT name FROM adblock_css WHERE\
blacklist='' AND whitelist=''"
result = sql.execute(request)
for name in list(itertools.chain(*result)):
rules += "%s,\n" % name
return rules[:-2] + "{display: none !important;}"
def get_css_rules(self, uri):
"""
Return css rules for uri
@return str
"""
rules = ""
parsed = urlparse(uri)
if parsed.scheme not in ["http", "https"]:
return ""
netloc = remove_www(parsed.netloc)
with SqlCursor(self) as sql:
request = "SELECT name FROM adblock_css WHERE\
(blacklist!='' AND blacklist!=?) OR whitelist=?"
result = sql.execute(request, (netloc, netloc))
for name in list(itertools.chain(*result)):
rules += "%s,\n" % name
return rules[:-2] + "{display: none !important;}"
def is_blocked(self, uri):
"""
Return True if uri is blocked
@param uri as str
@return bool
"""
try:
parsed = urlparse(uri)
if parsed.scheme not in ["http", "https"] or\
El().adblock_exceptions.find_parsed(parsed):
return False
with SqlCursor(self) as sql:
result = sql.execute(
"SELECT mtime FROM adblock\
WHERE dns=?", (parsed.netloc, ))
v = result.fetchone()
return v is not None
except Exception as e:
print("DatabaseAdblock::is_blocked():", e)
return False
def get_cursor(self):
"""
Return a new sqlite cursor
"""
try:
c = sqlite3.connect(self.DB_PATH, 600.0)
return c
except Exception as e:
print(e)
exit(-1)
#######################
# PRIVATE #
#######################
def __save_rules(self, rules, uris):
"""
Save rules to db
@param rules bytes
@param uris as [str]
"""
SqlCursor.add(self)
result = rules.decode('utf-8')
count = 0
for line in result.split('\n'):
if self.__cancellable.is_cancelled():
raise IOError("Cancelled")
if line.startswith('#'):
continue
array = line.replace(' ', '\t', 1).replace('\t', '@', 1).split('@')
if len(array) <= 1:
continue
dns = array[1].replace(' ', '').replace('\r', '').split('#')[0]
# Update entry if exists, create else
with SqlCursor(self) as sql:
debug("Add filter: %s" % dns)
sql.execute(
"INSERT INTO adblock\
(dns, mtime) VALUES (?, ?)",
(dns, self.__adblock_mtime))
count += 1
if count == 1000:
sql.commit()
count = 0
# We are the last call to save_rules()?
# Delete removed entries and commit
if not uris:
with SqlCursor(self) as sql:
sql.execute(
"DELETE FROM adblock\
WHERE mtime!=?", (self.__adblock_mtime, ))
sql.commit()
SqlCursor.remove(self)
def __save_css_default_rule(self, line):
"""
Save default (without blacklist, whitelist) rule to db
@param line as str
"""
name = line[2:]
# Update entry if exists, create else
with SqlCursor(self) as sql:
debug("Add filter: %s" % name)
sql.execute(
"INSERT INTO adblock_css\
(name, mtime) VALUES (?, ?)",
(name, self.__adblock_mtime))
def __save_css_domain_rule(self, line):
"""
Save domain rule to db
@param line as str
"""
whitelist = ""
blacklist = ""
(domains, name) = line.split("##")
for domain in domains.split(","):
if domain.startswith("~"):
blacklist += "@%s@" % domain[1:]
else:
whitelist += domain
with SqlCursor(self) as sql:
debug("Add filter: %s" % name)
sql.execute(
"INSERT INTO adblock_css\
(name, whitelist, blacklist, mtime)\
VALUES (?, ?, ?, ?)",
(name, whitelist, blacklist, self.__adblock_mtime))
def __save_css_rules(self, rules, uris):
"""
Save rules to db
@param rules as bytes
@param uris as [str]
"""
SqlCursor.add(self)
result = rules.decode("utf-8")
count = 0
for line in result.split('\n'):
if self.__cancellable.is_cancelled():
raise IOError("Cancelled")
if line.find("-abp-") != -1:
continue
elif line.startswith("##"):
self.__save_css_default_rule(line)
elif line.find("##") != -1:
self.__save_css_domain_rule(line)
count += 1
if count == 1000:
with SqlCursor(self) as sql:
sql.commit()
count = 0
# We are the last rule
# Delete old entries
if not uris:
with SqlCursor(self) as sql:
sql.execute(
"DELETE FROM adblock_css\
WHERE mtime!=?", (self.__adblock_mtime, ))
sql.commit()
SqlCursor.remove(self)
def __on_save_css_rules(self, result, uris):
"""
Load next uri
@param result as ??
@param uris as [str]
"""
if uris:
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri, self.__cancellable,
self.__on_load_uri_css_content,
uris)
def __on_load_uri_css_content(self, uri, status, content, uris):
"""
Load pending uris
@param uri as str
@param status as bool
@param content as bytes
@param uris as [str]
"""
if status:
self.__task_helper.run(self.__save_css_rules,
content,
uris,
callback=(self.__on_save_css_rules, uris))
def __on_save_rules(self, result=None, uris=[]):
"""
Load next uri, if finished, load CSS rules
@param result as None
@param uris as [str]
"""
if uris:
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri, self.__cancellable,
self.__on_load_uri_content,
uris)
else:
# Check entries in DB, do we need to update?
mtime = 0
with SqlCursor(self) as sql:
result = sql.execute("SELECT mtime FROM adblock_css\
ORDER BY mtime LIMIT 1")
v = result.fetchone()
if v is not None:
mtime = v[0]
# We ignore update value from rules file
if self.__adblock_mtime - mtime < self.__UPDATE:
return
locales = GLib.get_language_names()
user_locale = locales[0].split("_")[0]
try:
uris = [self.__CSS_LOCALIZED_URIS[user_locale]]
except:
uris = []
uris += list(self.__CSS_URIS)
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri, self.__cancellable,
self.__on_load_uri_css_content,
uris)
def __on_load_uri_content(self, uri, status, content, uris):
"""
Save loaded values
@param uri as str
@param status as bool
@param content as bytes
@param uris as [str]
"""
if status:
self.__task_helper.run(self.__save_rules,
content,
uris,
callback=(self.__on_save_rules, uris))
class DatabaseAdblock:
"""
Eolie adblock db
"""
DB_PATH = "%s/adblock.db" % EOLIE_LOCAL_PATH
__URIS = [
"https://adaway.org/hosts.txt",
"http://winhelp2002.mvps.org/hosts.txt",
"http://hosts-file.net/ad_servers.txt",
"https://pgl.yoyo.org/adservers/serverlist.php?"
"hostformat=hosts&showintro=0&mimetype=plaintext"
]
# SQLite documentation:
# In SQLite, a column with type INTEGER PRIMARY KEY
# is an alias for the ROWID.
# Here, we define an id INT PRIMARY KEY but never feed it,
# this make VACUUM not destroy rowids...
__create_adblock = '''CREATE TABLE adblock (
id INTEGER PRIMARY KEY,
dns TEXT NOT NULL,
mtime INT NOT NULL
)'''
def __init__(self):
"""
Create database tables or manage update if needed
"""
self.__cancellable = Gio.Cancellable.new()
self.__task_helper = TaskHelper()
f = Gio.File.new_for_path(self.DB_PATH)
# Lazy loading if not empty
if not f.query_exists():
try:
d = Gio.File.new_for_path(EOLIE_LOCAL_PATH)
if not d.query_exists():
d.make_directory_with_parents()
# Create db schema
with SqlCursor(self) as sql:
sql.execute(self.__create_adblock)
sql.commit()
except Exception as e:
print("DatabaseAdblock::__init__(): %s" % e)
def update(self):
"""
Update database
"""
if not Gio.NetworkMonitor.get_default().get_network_available():
return
# Update adblock_js repo
git = GLib.find_program_in_path("git")
if git is None:
print(_("For stronger ad blocking, install git command"))
else:
d = Gio.File.new_for_path(ADBLOCK_JS)
if d.query_exists():
argv = [git, "-C", ADBLOCK_JS, "pull"]
else:
argv = [
git, "clone",
"https://github.com/gnumdk/eolie-adblock.git", ADBLOCK_JS
]
(pid, a1, a2,
a3) = GLib.spawn_async(argv,
flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL)
GLib.spawn_close_pid(pid)
# Get in db mtime
# Only update if filters older than one week
mtime = 0
with SqlCursor(self) as sql:
result = sql.execute("SELECT mtime FROM adblock LIMIT 1")
v = result.fetchone()
if v is not None:
mtime = v[0]
self.__mtime = int(time())
if self.__mtime - mtime < 604800:
return
# Update adblock db
self.__cancellable.reset()
self.__on_load_uri_content(None, False, b"", self.__URIS)
def stop(self):
"""
Stop update
"""
self.__cancellable.cancel()
def is_blocked(self, uri):
"""
Return True if uri is blocked
@param uri as str
@return bool
"""
try:
parse = urlparse(uri)
with SqlCursor(self) as sql:
result = sql.execute(
"SELECT mtime FROM adblock\
WHERE dns=?", (parse.netloc, ))
v = result.fetchone()
return v is not None
except Exception as e:
print("DatabaseAdblock::is_blocked():", e)
return False
def get_cursor(self):
"""
Return a new sqlite cursor
"""
try:
c = sqlite3.connect(self.DB_PATH, 600.0)
return c
except Exception as e:
print(e)
exit(-1)
#######################
# PRIVATE #
#######################
def __save_rules(self, rules, uris):
"""
Save rules to db
@param rules as bytes
@param uris as [str]
"""
SqlCursor.add(self)
result = rules.decode('utf-8')
count = 0
for line in result.split('\n'):
if self.__cancellable.is_cancelled():
raise IOError("Cancelled")
if line.startswith('#'):
continue
array = line.replace(' ', '\t', 1).replace('\t', '@', 1).split('@')
if len(array) <= 1:
continue
dns = array[1].replace(' ', '').replace('\r', '').split('#')[0]
# Update entry if exists, create else
with SqlCursor(self) as sql:
sql.execute(
"INSERT INTO adblock\
(dns, mtime)\
VALUES (?, ?)", (dns, self.__mtime))
count += 1
if count == 1000:
sql.commit()
count = 0
# We are the last call to save_rules()?
# Delete removed entries and commit
if not uris:
with SqlCursor(self) as sql:
sql.execute(
"DELETE FROM adblock\
WHERE mtime!=?", (self.__mtime, ))
sql.commit()
SqlCursor.remove(self)
def __on_load_uri_content(self, uri, status, content, uris):
"""
Load pending uris
@param uri as str
@param status as bool
@param content as bytes
@param uris as [str]
"""
if status:
self.__task_helper.run(self.__save_rules, (content, uris))
if uris:
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri, self.__cancellable,
self.__on_load_uri_content,
uris)
def __on_shortcut_action(self, action, param):
"""
Global shortcuts handler
@param action as Gio.SimpleAction
@param param as GLib.Variant
"""
string = param.get_string()
if string == "uri":
if self.is_fullscreen:
self.__fullscreen_revealer.set_reveal_child(True)
self.toolbar.title.focus_entry()
elif string == "fullscreen":
if self.is_fullscreen:
self.unfullscreen()
else:
self.fullscreen()
elif string == "quit":
El().quit(True)
elif string == "new_page":
self.container.add_webview(El().start_page, LoadingType.FOREGROUND)
elif string == "close_page":
if self.is_fullscreen:
self.container.current.webview.emit("leave-fullscreen")
Gtk.ApplicationWindow.unfullscreen(self)
self.__container.try_close_view(self.container.current)
elif string == "reload":
self.container.current.webview.reload()
elif string == "home":
self.container.current.webview.load_uri(El().start_page)
elif string == "source":
uri = self.container.current.webview.uri
task_helper = TaskHelper()
task_helper.load_uri_content(uri, None, self.__on_source_loaded)
elif string == "find":
find_widget = self.container.current.find_widget
find_widget.set_search_mode(True)
find_widget.search()
elif string == "backward":
self.toolbar.actions.backward()
elif string == "forward":
self.toolbar.actions.forward()
elif string == "previous":
self.__container.previous()
elif string == "next":
self.__container.next()
elif string == "previous_site":
self.__container.sites_manager.previous()
elif string == "next_site":
self.__container.sites_manager.next()
elif string == "print":
self.container.current.webview.print()
elif string == "private":
self.container.add_webview(El().start_page, LoadingType.FOREGROUND,
True)
elif string == "last_page":
El().pages_menu.activate_last_action()
elif string == "zoom_in":
self.container.current.webview.zoom_in()
elif string == "zoom_out":
self.container.current.webview.zoom_out()
elif string == "zoom_default":
self.container.current.webview.zoom_default()
elif string == "history":
self.toolbar.title.focus_entry("history")
elif string == "search":
self.toolbar.title.focus_entry("search")
elif string == "save":
self.toolbar.end.save_page()
elif string == "expose":
active = self.toolbar.actions.view_button.get_active()
self.toolbar.actions.view_button.set_active(not active)
elif string == "show_left_panel":
value = El().settings.get_value("show-sidebar")
El().settings.set_value("show-sidebar",
GLib.Variant("b", not value))
class DatabaseAdblock:
"""
Eolie adblock db
"""
__DB_PATH = "%s/adblock.db" % EOLIE_DATA_PATH
__URIS = ["https://adaway.org/hosts.txt",
"https://pgl.yoyo.org/adservers/serverlist.php?" +
"hostformat=hosts&showintro=0&startdate%5Bday%5D=" +
"&startdate%5Bmonth%5D=&startdate%5Byear%5D=",
"http://winhelp2002.mvps.org/hosts.txt",
"http://hosts-file.net/ad_servers.txt",
"https://pgl.yoyo.org/adservers/serverlist.php?"
"hostformat=hosts&showintro=0&mimetype=plaintext"]
__CSS_URIS = ["https://easylist-downloads.adblockplus.org/easylist.txt"]
__CSS_LOCALIZED_URIS = {
"bg": "http://stanev.org/abp/adblock_bg.txt",
"zh": "https://easylist-downloads.adblockplus.org/easylistchina.txt",
"sk": "https://raw.github.com/tomasko126/" +
"easylistczechandslovak/master/filters.txt",
"cs": "https://raw.github.com/tomasko126/" +
"easylistczechandslovak/master/filters.txt",
"nl": "https://easylist-downloads.adblockplus.org/easylistdutch.txt",
"de": "https://easylist-downloads.adblockplus.org/easylistgermany.txt",
"he": "https://raw.githubusercontent.com/easylist/" +
"EasyListHebrew/master/EasyListHebrew.txt",
"it": "https://easylist-downloads.adblockplus.org/easylistitaly.txt",
"lt": "http://margevicius.lt/easylistlithuania.txt",
"es": "https://easylist-downloads.adblockplus.org/easylistspanish.txt",
"lv": "https://notabug.org/latvian-list/" +
"adblock-latvian/raw/master/lists/latvian-list.txt",
"ar": "https://easylist-downloads.adblockplus.org/Liste_AR.txt",
"fr": "https://easylist-downloads.adblockplus.org/liste_fr.txt",
"ro": "http://www.zoso.ro/pages/rolist.txt",
"ru": "https://easylist-downloads.adblockplus.org/advblock.txt",
"ja": "http://bit.ly/11QrCfx",
"fi": "https://adb.juvander.net/Finland_adb.txt",
"cz": "http://adblock.dajbych.net/adblock.txt",
"et": "http://gurud.ee/ab.txt",
"hu": "https://raw.githubusercontent.com/szpeter80/" +
"hufilter/master/hufilter.txt"}
__SCHEMA_VERSION = 0
__UPDATE = 172800
__SPECIAL_CHARS = r"([.$+?{}()\[\]\\])"
__REPLACE_CHARS = {"^": "(?:[^\w\d_\-.%]|$)",
"*": ".*"}
# SQLite documentation:
# In SQLite, a column with type INTEGER PRIMARY KEY
# is an alias for the ROWID.
# Here, we define an id INT PRIMARY KEY but never feed it,
# this make VACUUM not destroy rowids...
__create_adblock = """CREATE TABLE adblock (
id INTEGER PRIMARY KEY,
netloc TEXT TEXT NOT NULL,
mtime INT NOT NULL
)"""
__create_adblock_re = """CREATE TABLE adblock_re (
id INTEGER PRIMARY KEY,
regex TEXT NOT NULL,
mtime INT NOT NULL
)"""
__create_adblock_re_domain = """CREATE TABLE adblock_re_domain (
id INTEGER PRIMARY KEY,
domain TEXT NOT NULL,
regex TEXT NOT NULL,
mtime INT NOT NULL
)"""
__create_adblock_re_domain_ex = """CREATE TABLE adblock_re_domain_ex (
id INTEGER PRIMARY KEY,
domain TEXT NOT NULL,
regex TEXT NOT NULL,
mtime INT NOT NULL
)"""
__create_adblock_css = """CREATE TABLE adblock_css (
id INTEGER PRIMARY KEY,
rule TEXT NOT NULL,
whitelist TEXT DEFAULT '',
blacklist TEXT DEFAULT '',
mtime INT NOT NULL
)"""
__create_adblock_cache = """CREATE TABLE adblock_cache (
id INTEGER PRIMARY KEY,
allowed_uri TEXT NOT NULL
)"""
__create_adblock_idx = """CREATE UNIQUE INDEX idx_adblock ON adblock(
netloc)"""
__create_adblock_re_idx = """CREATE UNIQUE INDEX idx_adblock_re
ON adblock_re(regex)"""
__create_adblock_re_domain_idx = """CREATE INDEX
idx_adblock_re_domain
ON adblock_re_domain(domain)"""
__create_adblock_re_domain_regex_idx = """CREATE INDEX
idx_adblock_re_domain_regex
ON adblock_re_domain(regex)"""
__create_adblock_re_domain_ex_idx = """CREATE INDEX
idx_adblock_re_domain_ex
ON adblock_re_domain(domain)"""
__create_adblock_re_domain_regex_ex_idx = """CREATE INDEX
idx_adblock_re_domain_regex_ex
ON adblock_re_domain(regex)"""
__create_adblock_css_black_idx = """CREATE INDEX
idx_adblock_css_black
ON adblock_css(
blacklist)"""
__create_adblock_css_white_idx = """CREATE INDEX
idx_adblock_css_white
ON adblock_css(
whitelist)"""
__create_adblock_cache_idx = """CREATE UNIQUE INDEX idx_adblock_cache
ON adblock_cache(
allowed_uri)"""
def __init__(self):
"""
Create database tables or manage update if needed
"""
self.thread_lock = Lock()
self.__cancellable = Gio.Cancellable.new()
self.__task_helper = TaskHelper()
self.__adblock_mtime = int(time())
self.__regex = None
def create_db(self):
"""
Create databse
"""
if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR):
GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750)
# If DB schema changed, remove it
if GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR):
with SqlCursor(self) as sql:
result = sql.execute("PRAGMA user_version")
v = result.fetchone()
if v is None or v[0] != self.__SCHEMA_VERSION:
f = Gio.File.new_for_path(self.__DB_PATH)
f.delete()
if not GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR):
try:
# Create db schema
with SqlCursor(self) as sql:
sql.execute(self.__create_adblock)
sql.execute(self.__create_adblock_re)
sql.execute(self.__create_adblock_re_domain)
sql.execute(self.__create_adblock_re_domain_ex)
sql.execute(self.__create_adblock_css)
sql.execute(self.__create_adblock_cache)
sql.execute(self.__create_adblock_idx)
sql.execute(self.__create_adblock_re_idx)
sql.execute(self.__create_adblock_re_domain_idx)
sql.execute(self.__create_adblock_re_domain_regex_idx)
sql.execute(self.__create_adblock_re_domain_ex_idx)
sql.execute(self.__create_adblock_re_domain_regex_ex_idx)
sql.execute(self.__create_adblock_css_black_idx)
sql.execute(self.__create_adblock_css_white_idx)
sql.execute(self.__create_adblock_cache_idx)
sql.execute("PRAGMA user_version=%s" %
self.__SCHEMA_VERSION)
except Exception as e:
Logger.error("DatabaseAdblock::__init__(): %s", e)
def update(self):
"""
Update database
"""
if not Gio.NetworkMonitor.get_default().get_network_available():
return
# Update adblock_js repo
git = GLib.find_program_in_path("git")
if git is None:
Logger.info(_("For stronger ad blocking, install git command"))
else:
if GLib.file_test(ADBLOCK_JS, GLib.FileTest.IS_DIR):
argv = [git,
"-C",
ADBLOCK_JS,
"pull",
"https://gitlab.gnome.org/gnumdk/eolie-adblock.git"]
else:
argv = [git,
"clone",
"https://gitlab.gnome.org/gnumdk/eolie-adblock.git",
ADBLOCK_JS]
(pid, a1, a2, a3) = GLib.spawn_async(
argv,
flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL)
GLib.spawn_close_pid(pid)
# DB version is last successful sync mtime
try:
version = load(open(EOLIE_DATA_PATH + "/adblock.bin", "rb"))
except:
version = 0
self.__cancellable.reset()
if self.__adblock_mtime - version > self.__UPDATE:
# Update host rules
uris = list(self.__URIS)
locales = GLib.get_language_names()
user_locale = locales[0].split("_")[0]
try:
uris += self.__CSS_URIS +\
[self.__CSS_LOCALIZED_URIS[user_locale]]
except:
uris += self.__CSS_URIS
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri,
self.__cancellable,
self.__on_load_uri_content,
uris)
def stop(self):
"""
Stop update
"""
self.__cancellable.cancel()
def get_default_css_rules(self):
"""
Return default css rules
"""
rules = ""
with SqlCursor(self) as sql:
request = "SELECT rule FROM adblock_css WHERE\
blacklist='' AND whitelist=''"
result = sql.execute(request)
for rule in list(itertools.chain(*result)):
rules += "%s,\n" % rule
return rules[:-2] + "{display: none !important;}"
def get_css_rules(self, uri):
"""
Return css rules for uri
@return str
"""
rules = ""
parsed = urlparse(uri)
if parsed.scheme not in ["http", "https"]:
return ""
netloc = remove_www(parsed.netloc)
with SqlCursor(self) as sql:
request = "SELECT rule FROM adblock_css WHERE\
(blacklist!='' AND blacklist NOT LIKE ?) OR\
whitelist LIKE ?"
result = sql.execute(request, ("%" + netloc + "%",
"%" + netloc + "%"))
for rule in list(itertools.chain(*result)):
rules += "%s,\n" % rule
return rules[:-2] + "{display: none !important;}"
def is_netloc_blocked(self, netloc):
"""
Return True if netloc is blocked
@param netloc as str
@return bool
"""
try:
with SqlCursor(self) as sql:
result = sql.execute("SELECT mtime FROM adblock\
WHERE netloc=?", (netloc,))
v = result.fetchone()
return v is not None
except Exception as e:
Logger.error("DatabaseAdblock::is_netloc_blocked(): %s", e)
return False
def is_uri_blocked(self, uri, netloc):
"""
Return True if uri is blocked
@param uri as str
@param netloc as str
@return bool
"""
# We cache result for allowed uris
# because regex are quite slow in python
with SqlCursor(self) as sql:
result = sql.execute("SELECT allowed_uri\
FROM adblock_cache\
WHERE allowed_uri=?", (uri,))
v = result.fetchone()
if v is None:
# Search in main regexes
if self.__regex is None:
request = "SELECT regex FROM adblock_re"
result = sql.execute(request)
rules = list(itertools.chain(*result))
if rules:
regexes = "|".join(regex for regex in rules)
self.__regex = re.compile(regexes)
if self.__regex is not None:
blocked_re = bool(self.__regex.search(uri))
else:
blocked_re = False
# Find in domain regexes
request = "SELECT regex FROM adblock_re_domain\
WHERE domain=?"
result = sql.execute(request, (netloc,))
rules = list(itertools.chain(*result))
if rules:
regexes = "|".join(regex for regex in rules)
blocked_re_domain = bool(re.search(regexes, uri))
else:
blocked_re_domain = False
# If previous regexes blocked uri, check for an exception
if blocked_re_domain:
request = "SELECT regex FROM adblock_re_domain_ex\
WHERE domain=?"
result = sql.execute(request, (netloc,))
rules = list(itertools.chain(*result))
if rules:
regexes = "|".join(regex for regex in rules)
if bool(re.search(regexes, uri)):
blocked_re_domain = False
if not blocked_re and not blocked_re_domain:
sql.execute("INSERT INTO adblock_cache\
(allowed_uri) VALUES (?)",
(uri,))
return False
else:
return True
else:
return False
def get_cursor(self):
"""
Return a new sqlite cursor
"""
try:
c = sqlite3.connect(self.__DB_PATH, 600.0)
return c
except Exception as e:
Logger.error("DatabaseAdblock::get_cursor(): %s", e)
exit(-1)
#######################
# PRIVATE #
#######################
def __add_netloc(self, netloc):
"""
Add a new netloc
@param netloc as str
"""
with SqlCursor(self) as sql:
try:
sql.execute("INSERT INTO adblock\
(netloc, mtime) VALUES (?, ?)",
(netloc, self.__adblock_mtime))
except:
sql.execute("UPDATE adblock SET mtime=?\
WHERE netloc=?",
(self.__adblock_mtime, netloc))
def __add_regex(self, regex):
"""
Add a new regex
@param regex as str
"""
with SqlCursor(self) as sql:
try:
sql.execute("INSERT INTO adblock_re\
(regex, mtime) VALUES (?, ?)",
(regex, self.__adblock_mtime))
except:
sql.execute("UPDATE adblock_re SET mtime=?\
WHERE regex=?",
(self.__adblock_mtime, regex))
def __add_regex_domain(self, regex, domain):
"""
Add a new regex
@param regex as str
@param domain
"""
with SqlCursor(self) as sql:
result = sql.execute("SELECT mtime FROM adblock_re_domain\
WHERE regex=? AND domain=?",
(regex, domain))
v = result.fetchone()
if v is None:
sql.execute("INSERT INTO adblock_re_domain\
(regex, domain, mtime) VALUES (?, ?, ?)",
(regex, domain, self.__adblock_mtime))
else:
sql.execute("UPDATE adblock_re_domain SET mtime=?\
WHERE regex=? AND domain=?",
(self.__adblock_mtime, regex, domain))
def __add_regex_domain_ex(self, regex, domain):
"""
Add a new exception regex
@param regex as str
@param domain
"""
with SqlCursor(self) as sql:
result = sql.execute("SELECT mtime FROM adblock_re_domain_ex\
WHERE regex=? AND domain=?",
(regex, domain))
v = result.fetchone()
if v is None:
sql.execute("INSERT INTO adblock_re_domain_ex\
(regex, domain, mtime) VALUES (?, ?, ?)",
(regex, domain, self.__adblock_mtime))
else:
sql.execute("UPDATE adblock_re_domain_ex SET mtime=?\
WHERE regex=? AND domain=?",
(self.__adblock_mtime, regex, domain))
def __rule_to_regex(self, rule):
"""
Convert rule to regex
@param rule as str
@return regex as str
"""
try:
# Do nothing if rule is already a regex
if rule[0] == rule[-1] == "/":
return rule[1:-1]
rule = re.sub(self.__SPECIAL_CHARS, r"\\\1", rule)
# Handle ^ separator character, *, etc...
for key in self.__REPLACE_CHARS.keys():
rule = rule.replace(key, self.__REPLACE_CHARS[key])
# End of the address
if rule[-1] == "|":
rule = rule[:-1] + "$"
# Start of the address
if rule[0] == "|":
rule = "^" + rule[1:]
# Escape remaining | but not |$ => see self.__REPLACE_CHARS
rule = re.sub("(\|)[^$]", r"\|", rule)
return rule
except Exception as e:
Logger.error("DatabaseAdblock::__rule_to_regex(): %s", e)
return None
def __save_rules(self, rules):
"""
Save rules to db
@param rules bytes
"""
SqlCursor.add(self)
result = rules.decode('utf-8')
count = 0
for line in result.split('\n'):
SqlCursor.allow_thread_execution(self)
if self.__cancellable.is_cancelled():
SqlCursor.remove(self)
raise Exception("Cancelled")
if line.startswith('#'):
continue
array = line.replace(
' ', '\t', 1).replace('\t', '@', 1).split('@')
if len(array) <= 1:
continue
netloc = array[1].replace(
' ', '').replace('\r', '').split('#')[0]
# Update entry if exists, create else
if netloc != "localhost":
Logger.debug("Add filter: %s", netloc)
self.__add_netloc(netloc)
count += 1
if count == 1000:
SqlCursor.commit(self)
# Do not flood sqlite, this allow webkit extension to run
sleep(0.1)
count = 0
SqlCursor.remove(self)
def __save_css_default_rule(self, line):
"""
Save default (without blacklist, whitelist) rule to db
@param line as str
"""
rule = line[2:]
# Update entry if exists, create else
with SqlCursor(self) as sql:
try:
sql.execute("INSERT INTO adblock_css\
(rule, mtime) VALUES (?, ?)",
(rule, self.__adblock_mtime))
except:
sql.execute("UPDATE adblock_css SET mtime=?\
WHERE rule=?",
(self.__adblock_mtime, rule))
def __save_css_domain_rule(self, line):
"""
Save domain rule to db
@param line as str
"""
whitelist = []
blacklist = []
(domains, rule) = line.split("##")
for domain in domains.split(","):
if domain.startswith("~"):
blacklist.append(domain[1:])
else:
whitelist.append(domain)
str_whitelist = ",".join(whitelist)
str_blacklist = ",".join(blacklist)
with SqlCursor(self) as sql:
result = sql.execute("SELECT mtime FROM adblock_css\
WHERE blacklist=? AND whitelist=?\
AND rule=?",
(str_blacklist, str_whitelist, rule))
v = result.fetchone()
if v is None:
sql.execute("INSERT INTO adblock_css\
(rule, whitelist, blacklist, mtime)\
VALUES (?, ?, ?, ?)",
(rule, str_whitelist, str_blacklist,
self.__adblock_mtime))
else:
sql.execute("UPDATE adblock_css SET mtime=?\
WHERE rule=? and blacklist=? and whitelist=?",
(self.__adblock_mtime, rule,
str_blacklist, str_whitelist))
def __save_css_exception(self, line):
"""
Add a new exception
@param line as str
"""
(domain, rule) = line.split("#@#")
with SqlCursor(self) as sql:
result = sql.execute("SELECT rowid, blacklist FROM adblock_css\
WHERE rule=?", (rule,))
v = result.fetchone()
if v is None:
sql.execute("INSERT INTO adblock_css\
(rule, whitelist, blacklist, mtime)\
VALUES (?, ?, ?, ?)",
(rule, "", domain, self.__adblock_mtime))
else:
(rowid, blacklist) = v
blacklist += ",%s" % domain
sql.execute("UPDATE adblock_css SET blacklist=?\
WHERE rowid=?", (blacklist, rowid))
def __save_abp_rule(self, rule, exception):
"""
Save abp rule
@param rule as str
@param exception as bool
"""
# Simple host rule
if rule[:2] == "||":
if rule[-1] == "^" and not exception:
self.__add_netloc(rule[2:-1])
else:
regex = self.__rule_to_regex(rule[2:])
split = re.split("/|\^", rule[2:])
uri = "http://%s" % split[0]
parsed = urlparse(uri)
if parsed.netloc:
if exception:
self.__add_regex_domain_ex(regex, parsed.netloc)
else:
self.__add_regex_domain(regex, parsed.netloc)
elif not exception:
self.__add_regex(regex)
elif not exception:
regex = self.__rule_to_regex(rule)
if regex is not None:
self.__add_regex(regex)
def __save_abp_rules(self, rules):
"""
Save rules to db
@param rules as bytes
"""
SqlCursor.add(self)
result = rules.decode("utf-8")
count = 0
for line in result.split('\n'):
SqlCursor.allow_thread_execution(self)
if self.__cancellable.is_cancelled():
SqlCursor.remove(self)
raise Exception("Cancelled")
if "-abp-" in line or "$" in line or "!" in line or "[" in line:
continue
elif line.startswith("##"):
self.__save_css_default_rule(line)
elif "#@#" in line:
self.__save_css_exception(line)
elif "##" in line:
self.__save_css_domain_rule(line)
elif line.startswith("@@"):
self.__save_abp_rule(line[2:], True)
else:
self.__save_abp_rule(line, False)
Logger.debug("Add abp filter: %s", line)
count += 1
if count == 1000:
SqlCursor.commit(self)
# Do not flood sqlite, this allow webkit extension to run
sleep(0.1)
count = 0
SqlCursor.remove(self)
def __on_save_rules(self, result, uris=[]):
"""
Load next uri, if finished, load CSS rules
@param result (unused)
@param uris as [str]
"""
if self.__cancellable.is_cancelled():
return
if uris:
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri,
self.__cancellable,
self.__on_load_uri_content,
uris)
else:
with SqlCursor(self) as sql:
sql.execute("DELETE FROM adblock\
WHERE mtime!=?", (self.__adblock_mtime,))
sql.execute("DELETE FROM adblock_re\
WHERE mtime!=?", (self.__adblock_mtime,))
sql.execute("DELETE FROM adblock_re_domain\
WHERE mtime!=?", (self.__adblock_mtime,))
sql.execute("DELETE FROM adblock_re_domain_ex\
WHERE mtime!=?", (self.__adblock_mtime,))
sql.execute("DELETE FROM adblock_css\
WHERE mtime!=?", (self.__adblock_mtime,))
sql.execute("DELETE FROM adblock_cache")
try:
dump(self.__adblock_mtime,
open(EOLIE_DATA_PATH + "/adblock.bin", "wb"))
except Exception as e:
Logger.error("DatabaseAdblock::__on_save_rules(): %s", e)
def __on_load_uri_content(self, uri, status, content, uris):
"""
Save loaded values
@param uri as str
@param status as bool
@param content as bytes
@param uris as [str]
"""
Logger.debug("DatabaseAdblock::__on_load_uri_content(): %s", uri)
if status:
if uri in self.__URIS:
self.__task_helper.run(self.__save_rules, content,
callback=(self.__on_save_rules, uris))
else:
self.__task_helper.run(self.__save_abp_rules, content,
callback=(self.__on_save_rules, uris))
else:
self.__on_save_rules(None, uris)
class DatabasePhishing:
"""
Phishing database
"""
DB_PATH = "%s/phishing.db" % EOLIE_DATA_PATH
__URI = "http://data.phishtank.com/data/online-valid.json"
# SQLite documentation:
# In SQLite, a column with type INTEGER PRIMARY KEY
# is an alias for the ROWID.
# Here, we define an id INT PRIMARY KEY but never feed it,
# this make VACUUM not destroy rowids...
__create_phishing = '''CREATE TABLE phishing (
id INTEGER PRIMARY KEY,
uri TEXT NOT NULL,
mtime INT NOT NULL
)'''
def __init__(self):
"""
Create database tables or manage update if needed
"""
self.__cancellable = Gio.Cancellable.new()
self.__task_helper = TaskHelper()
# Lazy loading if not empty
if not GLib.file_test(self.DB_PATH, GLib.FileTest.IS_REGULAR):
try:
if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR):
GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750)
# Create db schema
with SqlCursor(self) as sql:
sql.execute(self.__create_phishing)
sql.commit()
except Exception as e:
print("DatabasePhishing::__init__(): %s" % e)
def update(self):
"""
Update database
"""
if not Gio.NetworkMonitor.get_default().get_network_available():
return
# Get in db mtime
# Only update if filters older than one day
mtime = 0
with SqlCursor(self) as sql:
result = sql.execute("SELECT mtime FROM phishing LIMIT 1")
v = result.fetchone()
if v is not None:
mtime = v[0]
self.__mtime = int(time())
if self.__mtime - mtime < 86400:
return
# Update phishing db
self.__cancellable.reset()
self.__on_load_uri_content(None, False, b"", [self.__URI])
def is_phishing(self, uri):
"""
True if uri is phishing
@param uri as str
@return bool
"""
uri = uri.rstrip("/")
try:
with SqlCursor(self) as sql:
result = sql.execute(
"SELECT uri FROM phishing\
WHERE uri=?", (uri, ))
v = result.fetchone()
return v is not None
except Exception as e:
print("DatabasePhishing::is_phishing():", e)
return False
def stop(self):
"""
Stop update
"""
self.__cancellable.cancel()
self.__stop = True
def get_cursor(self):
"""
Return a new sqlite cursor
"""
try:
c = sqlite3.connect(self.DB_PATH, 600.0)
return c
except Exception as e:
print(e)
exit(-1)
#######################
# PRIVATE #
#######################
def __save_rules(self, rules, uris):
"""
Save rules to db
@param rules as bytes
@param uris as [str]
"""
SqlCursor.add(self)
result = rules.decode('utf-8')
j = json.loads(result)
with SqlCursor(self) as sql:
count = 0
for item in j:
if self.__cancellable.is_cancelled():
raise IOError("Cancelled")
sql.execute(
"INSERT INTO phishing\
(uri, mtime) VALUES (?, ?)",
(item["url"].rstrip("/"), self.__mtime))
count += 1
if count == 1000:
sql.commit()
count = 0
sql.commit()
# We are the last call to save_rules()?
# Delete removed entries and commit
if not uris:
with SqlCursor(self) as sql:
sql.execute(
"DELETE FROM phishing\
WHERE mtime!=?", (self.__mtime, ))
sql.commit()
SqlCursor.remove(self)
def __on_load_uri_content(self, uri, status, content, uris):
"""
Load pending uris
@param uri as str
@param status as bool
@param content as bytes
@param uris as [str]
"""
if status:
self.__task_helper.run(self.__save_rules, content, uris)
if uris:
uri = uris.pop(0)
self.__task_helper.load_uri_content(uri, self.__cancellable,
self.__on_load_uri_content,
uris)