def __install_engine(self, uri, window): """ Install engine from uri @param uri as str @param window as Window """ task_helper = TaskHelper(self.__user_agent) task_helper.load_uri_content(uri, None, self.__on_engine_loaded, window)
def search_suggestions(self, value, cancellable, callback): """ Search suggestions for value @param value as str @param cancellable as Gio.Cancellable @param callback as str """ try: if not value.strip(" "): return uri = self.__keyword % value task_helper = TaskHelper() task_helper.load_uri_content(uri, cancellable, callback, self.__encoding, value) except Exception as e: print("Search::search_suggestions():", e)
def search_suggestions(self, value, cancellable, callback): """ Search suggestions for value @param value as str @param cancellable as Gio.Cancellable @param callback as str """ try: if not value.strip(" "): return uri = self.__suggest % GLib.uri_escape_string(value, None, True) task_helper = TaskHelper(self.__user_agent) task_helper.load_uri_content(uri, cancellable, callback, self.__encoding, value) except Exception as e: Logger.error("Search::search_suggestions(): %s", e)
class ImagesPopover(Gtk.Popover): """ Show images for page id """ def __init__(self, uri, page_id, window): """ Init popover @param uri as str @param page_id as int @param window as Window """ Gtk.Popover.__init__(self) self.set_modal(False) window.register(self) self.__cache_uris = [] self.__uri = uri self.__page_id = page_id self.__cancellable = Gio.Cancellable() self.__filter = "" builder = Gtk.Builder() builder.add_from_resource("/org/gnome/Eolie/PopoverImages.ui") builder.connect_signals(self) widget = builder.get_object("widget") self.__spinner = builder.get_object("spinner") self.__flowbox = builder.get_object("flowbox") self.__flowbox.set_filter_func(self.__filter_func) self.__entry = builder.get_object("entry") self.__button = builder.get_object("button") self.add(widget) if Gio.NetworkMonitor.get_default().get_network_available(): El().helper.call("GetImages", page_id, None, self.__on_get_images) (width, height) = El().active_window.get_size() self.set_size_request(width / 2, height / 1.5) self.connect("closed", self.__on_closed) self.__task_helper = TaskHelper() ####################### # PROTECTED # ####################### def _on_search_changed(self, entry): """ Filter flowbox @param entry as Gtk.Entry """ self.__filter = entry.get_text() self.__flowbox.invalidate_filter() def _on_button_clicked(self, button): """ Save visible images @param button as Gtk.Button """ task_helper = TaskHelper() task_helper.run(self.__move_images) self.__spinner.start() def _on_button_toggled(self, button): """ Cancel previous download """ self.__cancellable.cancel() self.__spinner.start() self.__button.set_sensitive(False) for child in self.__flowbox.get_children(): child.destroy() self.__links = button.get_active() if Gio.NetworkMonitor.get_default().get_network_available(): if button.get_active(): El().helper.call("GetImageLinks", self.__page_id, None, self.__on_get_images) else: El().helper.call("GetImages", self.__page_id, None, self.__on_get_images) ####################### # PRIVATE # ####################### def __filter_func(self, child): """ Filter child @param child as image """ if child.uri.find(self.__filter) != -1: return True def __add_image(self, uri): """ Add a child to flowbox @param uri as str """ image = Image(uri) image.show() self.__flowbox.add(image) def __move_images(self): """ Move image to download directory """ parsed = urlparse(self.__uri) directory_uri = El().settings.get_value('download-uri').get_string() if not directory_uri: directory = GLib.get_user_special_dir( GLib.UserDirectory.DIRECTORY_DOWNLOAD) directory_uri = GLib.filename_to_uri(directory, None) destination_uri = "%s/%s" % (directory_uri, parsed.netloc) directory = Gio.File.new_for_uri(destination_uri) if not directory.query_exists(): directory.make_directory_with_parents() for child in self.__flowbox.get_children(): if child.uri.find(self.__filter) != -1: encoded = sha256(child.uri.encode("utf-8")).hexdigest() child_basename = child.uri.split("/")[-1] filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded) s = Gio.File.new_for_path(filepath) if not s.query_exists(): continue d = Gio.File.new_for_uri("%s/%s" % (destination_uri, child_basename)) try: s.move(d, Gio.FileCopyFlags.OVERWRITE, None, None, None) except Exception as e: print("ImagesPopover::__move_images()", e) GLib.idle_add(self.hide) def __clean_cache(self): """ Clean the cache """ for uri in self.__cache_uris: encoded = sha256(uri.encode("utf-8")).hexdigest() filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded) f = Gio.File.new_for_path(filepath) try: if f.query_exists(): f.delete() except Exception as e: print("ImagesPopover::__clean_cache():", e) def __on_write_all_async(self, stream, result, uri): """ Add image @param stream as Gio.OutputStream @param result as Gio.AsyncResult @param uri as str """ try: stream.write_all_finish(result) self.__add_image(uri) self.__cache_uris.append(uri) except Exception as e: print("ImagesPopover::__on_write_all_async()", e) def __on_load_uri_content(self, uri, status, content, uris): """ Load pending uris @param uri as str @param status as bool @param content as bytes @param uris as [str] """ if status: encoded = sha256(uri.encode("utf-8")).hexdigest() filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded) f = Gio.File.new_for_path(filepath) stream = f.append_to(Gio.FileCreateFlags.REPLACE_DESTINATION, self.__cancellable) stream.write_all_async(content, GLib.PRIORITY_DEFAULT, self.__cancellable, self.__on_write_all_async, uri) if uris and not self.__cancellable.is_cancelled(): uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_content, uris) else: self.__spinner.stop() self.__button.set_sensitive(True) def __on_get_images(self, source, result): """ Get result and load pending uris @param source as GObject.Object @param result as Gio.AsyncResult """ uris = [] try: uris = source.call_finish(result)[0] except Exception as e: print("ImagesPopover::__on_get_images()", e) self.__cancellable.reset() self.__on_load_uri_content(None, False, b"", uris) def __on_closed(self, popover): """ Clean cache """ self.__spinner.stop() self.__cancellable.cancel() self.__task_helper.run(self.__clean_cache)
class DatabasePhishing: """ Phishing database """ __DB_PATH = "%s/phishing.db" % EOLIE_DATA_PATH __URI = "http://data.phishtank.com/data/online-valid.json" __SCHEMA_VERSION = 0 __UPDATE = 172800 # SQLite documentation: # In SQLite, a column with type INTEGER PRIMARY KEY # is an alias for the ROWID. # Here, we define an id INT PRIMARY KEY but never feed it, # this make VACUUM not destroy rowids... __create_phishing = '''CREATE TABLE phishing ( id INTEGER PRIMARY KEY, uri TEXT NOT NULL, mtime INT NOT NULL )''' __create_phishing_idx = """CREATE UNIQUE INDEX idx_phishing ON phishing( uri)""" def __init__(self): """ Create database tables or manage update if needed """ self.thread_lock = Lock() self.__cancellable = Gio.Cancellable.new() self.__task_helper = TaskHelper() self.__phishing_mtime = int(time()) self.__regex = None def create_db(self): """ Create databse """ if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR): GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750) # If DB schema changed, remove it if GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR): with SqlCursor(self) as sql: result = sql.execute("PRAGMA user_version") v = result.fetchone() if v is None or v[0] != self.__SCHEMA_VERSION: f = Gio.File.new_for_path(self.__DB_PATH) f.delete() if not GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR): try: # Create db schema with SqlCursor(self) as sql: sql.execute(self.__create_phishing) sql.execute(self.__create_phishing_idx) sql.execute("PRAGMA user_version=%s" % self.__SCHEMA_VERSION) except Exception as e: Logger.error("DatabasePhishing::__init__(): %s", e) def update(self): """ Update database """ if not Gio.NetworkMonitor.get_default().get_network_available(): return # DB version is last successful sync mtime try: version = load(open(EOLIE_DATA_PATH + "/phishing.bin", "rb")) except: version = 0 self.__cancellable.reset() if self.__phishing_mtime - version > self.__UPDATE: self.__on_load_uri_content(None, False, b"", [self.__URI]) def is_phishing(self, uri): """ True if uri is phishing @param uri as str @return bool """ uri = uri.rstrip("/") try: with SqlCursor(self) as sql: result = sql.execute( "SELECT uri FROM phishing\ WHERE uri=?", (uri, )) v = result.fetchone() return v is not None except Exception as e: Logger.error("DatabasePhishing::is_phishing(): %s", e) return False def stop(self): """ Stop update """ self.__cancellable.cancel() self.__stop = True def get_cursor(self): """ Return a new sqlite cursor """ try: c = sqlite3.connect(self.__DB_PATH, 600.0) return c except Exception as e: Logger.error("DatabasePhishing::get_cursor(): %s", e) exit(-1) ####################### # PRIVATE # ####################### def __save_rules(self, rules, uris): """ Save rules to db @param rules as bytes @param uris as [str] """ SqlCursor.add(self) try: result = rules.decode('utf-8') j = json.loads(result) with SqlCursor(self) as sql: count = 0 for item in j: if self.__cancellable.is_cancelled(): raise IOError("Cancelled") uri = item["url"].rstrip("/") try: sql.execute( "INSERT INTO phishing\ (uri, mtime) VALUES (?, ?)", (uri, self.__phishing_mtime)) except: sql.execute( "UPDATE phishing set mtime=?\ WHERE uri=?", (self.__phishing_mtime, uri)) count += 1 if count == 1000: SqlCursor.commit(self) # Do not flood sqlite # this allow webkit extension to run sleep(0.1) count = 0 # We are the last call to save_rules()? # Delete removed entries and commit if not uris: with SqlCursor(self) as sql: sql.execute( "DELETE FROM phishing\ WHERE mtime!=?", (self.__phishing_mtime, )) try: dump(self.__phishing_mtime, open(EOLIE_DATA_PATH + "/phishing.bin", "wb")) except Exception as e: Logger.error("DatabasePhishing::__save_rules(): %s", e) except Exception as e: Logger.error("DatabasePhishing::__save_rules():%s -> %s", e, rules) SqlCursor.remove(self) def __on_load_uri_content(self, uri, status, content, uris): """ Load pending uris @param uri as str @param status as bool @param content as bytes @param uris as [str] """ if status: self.__task_helper.run(self.__save_rules, content, uris) if uris: uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_content, uris)
class DatabaseAdblock: """ Eolie adblock db """ DB_PATH = "%s/adblock2.db" % EOLIE_DATA_PATH __URIS = [ "https://adaway.org/hosts.txt", "https://pgl.yoyo.org/adservers/serverlist.php?" + "hostformat=hosts&showintro=0&startdate%5Bday%5D=" + "&startdate%5Bmonth%5D=&startdate%5Byear%5D=", "http://winhelp2002.mvps.org/hosts.txt", "http://hosts-file.net/ad_servers.txt", "https://pgl.yoyo.org/adservers/serverlist.php?" "hostformat=hosts&showintro=0&mimetype=plaintext" ] __CSS_URIS = ["https://easylist-downloads.adblockplus.org/easylist.txt"] __CSS_LOCALIZED_URIS = { "bg": "http://stanev.org/abp/adblock_bg.txt", "zh": "https://easylist-downloads.adblockplus.org/easylistchina.txt", "sk": "https://raw.github.com/tomasko126/" + "easylistczechandslovak/master/filters.txt", "cs": "https://raw.github.com/tomasko126/" + "easylistczechandslovak/master/filters.txt", "nl": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", "de": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", "he": "https://raw.githubusercontent.com/easylist/" + "EasyListHebrew/master/EasyListHebrew.txt", "it": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", "lt": "http://margevicius.lt/easylistlithuania.txt", "es": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", "lv": "https://notabug.org/latvian-list/" + "adblock-latvian/raw/master/lists/latvian-list.txt", "ar": "https://easylist-downloads.adblockplus.org/Liste_AR.txt", "fr": "https://easylist-downloads.adblockplus.org/liste_fr.txt", "ro": "http://www.zoso.ro/pages/rolist.txt", "ru": "https://easylist-downloads.adblockplus.org/advblock.txt", "ja": "http://bit.ly/11QrCfx", "fi": "https://adb.juvander.net/Finland_adb.txt", "cz": "http://adblock.dajbych.net/adblock.txt", "et": "http://gurud.ee/ab.txt", "hu": "https://raw.githubusercontent.com/szpeter80/" + "hufilter/master/hufilter.txt" } __UPDATE = 172800 # SQLite documentation: # In SQLite, a column with type INTEGER PRIMARY KEY # is an alias for the ROWID. # Here, we define an id INT PRIMARY KEY but never feed it, # this make VACUUM not destroy rowids... __create_adblock = '''CREATE TABLE adblock ( id INTEGER PRIMARY KEY, dns TEXT NOT NULL, mtime INT NOT NULL )''' __create_adblock_css = '''CREATE TABLE adblock_css ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, whitelist TEXT DEFAULT "", blacklist TEXT DEFAULT "", mtime INT NOT NULL )''' def __init__(self): """ Create database tables or manage update if needed """ self.__cancellable = Gio.Cancellable.new() self.__task_helper = TaskHelper() self.__adblock_mtime = int(time()) # Lazy loading if not empty if not GLib.file_test(self.DB_PATH, GLib.FileTest.IS_REGULAR): try: if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR): GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750) # Create db schema with SqlCursor(self) as sql: sql.execute(self.__create_adblock) sql.execute(self.__create_adblock_css) sql.commit() except Exception as e: print("DatabaseAdblock::__init__(): %s" % e) def update(self): """ Update database """ if not Gio.NetworkMonitor.get_default().get_network_available(): return # Update adblock_js repo git = GLib.find_program_in_path("git") if git is None: print(_("For stronger ad blocking, install git command")) else: if GLib.file_test(ADBLOCK_JS, GLib.FileTest.IS_DIR): argv = [git, "-C", ADBLOCK_JS, "pull"] else: argv = [ git, "clone", "https://gitlab.gnome.org/gnumdk/eolie-adblock.git", ADBLOCK_JS ] (pid, a1, a2, a3) = GLib.spawn_async(argv, flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL) GLib.spawn_close_pid(pid) # Check entries in DB, do we need to update? mtime = 0 with SqlCursor(self) as sql: result = sql.execute("SELECT mtime FROM adblock\ ORDER BY mtime LIMIT 1") v = result.fetchone() if v is not None: mtime = v[0] self.__cancellable.reset() if self.__adblock_mtime - mtime > self.__UPDATE: # Update host rules uris = list(self.__URIS) uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_content, uris) else: self.__on_save_rules() def stop(self): """ Stop update """ self.__cancellable.cancel() def get_default_css_rules(self): """ Return default css rules """ rules = "" with SqlCursor(self) as sql: request = "SELECT name FROM adblock_css WHERE\ blacklist='' AND whitelist=''" result = sql.execute(request) for name in list(itertools.chain(*result)): rules += "%s,\n" % name return rules[:-2] + "{display: none !important;}" def get_css_rules(self, uri): """ Return css rules for uri @return str """ rules = "" parsed = urlparse(uri) if parsed.scheme not in ["http", "https"]: return "" netloc = remove_www(parsed.netloc) with SqlCursor(self) as sql: request = "SELECT name FROM adblock_css WHERE\ (blacklist!='' AND blacklist!=?) OR whitelist=?" result = sql.execute(request, (netloc, netloc)) for name in list(itertools.chain(*result)): rules += "%s,\n" % name return rules[:-2] + "{display: none !important;}" def is_blocked(self, uri): """ Return True if uri is blocked @param uri as str @return bool """ try: parsed = urlparse(uri) if parsed.scheme not in ["http", "https"] or\ El().adblock_exceptions.find_parsed(parsed): return False with SqlCursor(self) as sql: result = sql.execute( "SELECT mtime FROM adblock\ WHERE dns=?", (parsed.netloc, )) v = result.fetchone() return v is not None except Exception as e: print("DatabaseAdblock::is_blocked():", e) return False def get_cursor(self): """ Return a new sqlite cursor """ try: c = sqlite3.connect(self.DB_PATH, 600.0) return c except Exception as e: print(e) exit(-1) ####################### # PRIVATE # ####################### def __save_rules(self, rules, uris): """ Save rules to db @param rules bytes @param uris as [str] """ SqlCursor.add(self) result = rules.decode('utf-8') count = 0 for line in result.split('\n'): if self.__cancellable.is_cancelled(): raise IOError("Cancelled") if line.startswith('#'): continue array = line.replace(' ', '\t', 1).replace('\t', '@', 1).split('@') if len(array) <= 1: continue dns = array[1].replace(' ', '').replace('\r', '').split('#')[0] # Update entry if exists, create else with SqlCursor(self) as sql: debug("Add filter: %s" % dns) sql.execute( "INSERT INTO adblock\ (dns, mtime) VALUES (?, ?)", (dns, self.__adblock_mtime)) count += 1 if count == 1000: sql.commit() count = 0 # We are the last call to save_rules()? # Delete removed entries and commit if not uris: with SqlCursor(self) as sql: sql.execute( "DELETE FROM adblock\ WHERE mtime!=?", (self.__adblock_mtime, )) sql.commit() SqlCursor.remove(self) def __save_css_default_rule(self, line): """ Save default (without blacklist, whitelist) rule to db @param line as str """ name = line[2:] # Update entry if exists, create else with SqlCursor(self) as sql: debug("Add filter: %s" % name) sql.execute( "INSERT INTO adblock_css\ (name, mtime) VALUES (?, ?)", (name, self.__adblock_mtime)) def __save_css_domain_rule(self, line): """ Save domain rule to db @param line as str """ whitelist = "" blacklist = "" (domains, name) = line.split("##") for domain in domains.split(","): if domain.startswith("~"): blacklist += "@%s@" % domain[1:] else: whitelist += domain with SqlCursor(self) as sql: debug("Add filter: %s" % name) sql.execute( "INSERT INTO adblock_css\ (name, whitelist, blacklist, mtime)\ VALUES (?, ?, ?, ?)", (name, whitelist, blacklist, self.__adblock_mtime)) def __save_css_rules(self, rules, uris): """ Save rules to db @param rules as bytes @param uris as [str] """ SqlCursor.add(self) result = rules.decode("utf-8") count = 0 for line in result.split('\n'): if self.__cancellable.is_cancelled(): raise IOError("Cancelled") if line.find("-abp-") != -1: continue elif line.startswith("##"): self.__save_css_default_rule(line) elif line.find("##") != -1: self.__save_css_domain_rule(line) count += 1 if count == 1000: with SqlCursor(self) as sql: sql.commit() count = 0 # We are the last rule # Delete old entries if not uris: with SqlCursor(self) as sql: sql.execute( "DELETE FROM adblock_css\ WHERE mtime!=?", (self.__adblock_mtime, )) sql.commit() SqlCursor.remove(self) def __on_save_css_rules(self, result, uris): """ Load next uri @param result as ?? @param uris as [str] """ if uris: uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_css_content, uris) def __on_load_uri_css_content(self, uri, status, content, uris): """ Load pending uris @param uri as str @param status as bool @param content as bytes @param uris as [str] """ if status: self.__task_helper.run(self.__save_css_rules, content, uris, callback=(self.__on_save_css_rules, uris)) def __on_save_rules(self, result=None, uris=[]): """ Load next uri, if finished, load CSS rules @param result as None @param uris as [str] """ if uris: uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_content, uris) else: # Check entries in DB, do we need to update? mtime = 0 with SqlCursor(self) as sql: result = sql.execute("SELECT mtime FROM adblock_css\ ORDER BY mtime LIMIT 1") v = result.fetchone() if v is not None: mtime = v[0] # We ignore update value from rules file if self.__adblock_mtime - mtime < self.__UPDATE: return locales = GLib.get_language_names() user_locale = locales[0].split("_")[0] try: uris = [self.__CSS_LOCALIZED_URIS[user_locale]] except: uris = [] uris += list(self.__CSS_URIS) uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_css_content, uris) def __on_load_uri_content(self, uri, status, content, uris): """ Save loaded values @param uri as str @param status as bool @param content as bytes @param uris as [str] """ if status: self.__task_helper.run(self.__save_rules, content, uris, callback=(self.__on_save_rules, uris))
class DatabaseAdblock: """ Eolie adblock db """ DB_PATH = "%s/adblock.db" % EOLIE_LOCAL_PATH __URIS = [ "https://adaway.org/hosts.txt", "http://winhelp2002.mvps.org/hosts.txt", "http://hosts-file.net/ad_servers.txt", "https://pgl.yoyo.org/adservers/serverlist.php?" "hostformat=hosts&showintro=0&mimetype=plaintext" ] # SQLite documentation: # In SQLite, a column with type INTEGER PRIMARY KEY # is an alias for the ROWID. # Here, we define an id INT PRIMARY KEY but never feed it, # this make VACUUM not destroy rowids... __create_adblock = '''CREATE TABLE adblock ( id INTEGER PRIMARY KEY, dns TEXT NOT NULL, mtime INT NOT NULL )''' def __init__(self): """ Create database tables or manage update if needed """ self.__cancellable = Gio.Cancellable.new() self.__task_helper = TaskHelper() f = Gio.File.new_for_path(self.DB_PATH) # Lazy loading if not empty if not f.query_exists(): try: d = Gio.File.new_for_path(EOLIE_LOCAL_PATH) if not d.query_exists(): d.make_directory_with_parents() # Create db schema with SqlCursor(self) as sql: sql.execute(self.__create_adblock) sql.commit() except Exception as e: print("DatabaseAdblock::__init__(): %s" % e) def update(self): """ Update database """ if not Gio.NetworkMonitor.get_default().get_network_available(): return # Update adblock_js repo git = GLib.find_program_in_path("git") if git is None: print(_("For stronger ad blocking, install git command")) else: d = Gio.File.new_for_path(ADBLOCK_JS) if d.query_exists(): argv = [git, "-C", ADBLOCK_JS, "pull"] else: argv = [ git, "clone", "https://github.com/gnumdk/eolie-adblock.git", ADBLOCK_JS ] (pid, a1, a2, a3) = GLib.spawn_async(argv, flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL) GLib.spawn_close_pid(pid) # Get in db mtime # Only update if filters older than one week mtime = 0 with SqlCursor(self) as sql: result = sql.execute("SELECT mtime FROM adblock LIMIT 1") v = result.fetchone() if v is not None: mtime = v[0] self.__mtime = int(time()) if self.__mtime - mtime < 604800: return # Update adblock db self.__cancellable.reset() self.__on_load_uri_content(None, False, b"", self.__URIS) def stop(self): """ Stop update """ self.__cancellable.cancel() def is_blocked(self, uri): """ Return True if uri is blocked @param uri as str @return bool """ try: parse = urlparse(uri) with SqlCursor(self) as sql: result = sql.execute( "SELECT mtime FROM adblock\ WHERE dns=?", (parse.netloc, )) v = result.fetchone() return v is not None except Exception as e: print("DatabaseAdblock::is_blocked():", e) return False def get_cursor(self): """ Return a new sqlite cursor """ try: c = sqlite3.connect(self.DB_PATH, 600.0) return c except Exception as e: print(e) exit(-1) ####################### # PRIVATE # ####################### def __save_rules(self, rules, uris): """ Save rules to db @param rules as bytes @param uris as [str] """ SqlCursor.add(self) result = rules.decode('utf-8') count = 0 for line in result.split('\n'): if self.__cancellable.is_cancelled(): raise IOError("Cancelled") if line.startswith('#'): continue array = line.replace(' ', '\t', 1).replace('\t', '@', 1).split('@') if len(array) <= 1: continue dns = array[1].replace(' ', '').replace('\r', '').split('#')[0] # Update entry if exists, create else with SqlCursor(self) as sql: sql.execute( "INSERT INTO adblock\ (dns, mtime)\ VALUES (?, ?)", (dns, self.__mtime)) count += 1 if count == 1000: sql.commit() count = 0 # We are the last call to save_rules()? # Delete removed entries and commit if not uris: with SqlCursor(self) as sql: sql.execute( "DELETE FROM adblock\ WHERE mtime!=?", (self.__mtime, )) sql.commit() SqlCursor.remove(self) def __on_load_uri_content(self, uri, status, content, uris): """ Load pending uris @param uri as str @param status as bool @param content as bytes @param uris as [str] """ if status: self.__task_helper.run(self.__save_rules, (content, uris)) if uris: uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_content, uris)
def __on_shortcut_action(self, action, param): """ Global shortcuts handler @param action as Gio.SimpleAction @param param as GLib.Variant """ string = param.get_string() if string == "uri": if self.is_fullscreen: self.__fullscreen_revealer.set_reveal_child(True) self.toolbar.title.focus_entry() elif string == "fullscreen": if self.is_fullscreen: self.unfullscreen() else: self.fullscreen() elif string == "quit": El().quit(True) elif string == "new_page": self.container.add_webview(El().start_page, LoadingType.FOREGROUND) elif string == "close_page": if self.is_fullscreen: self.container.current.webview.emit("leave-fullscreen") Gtk.ApplicationWindow.unfullscreen(self) self.__container.try_close_view(self.container.current) elif string == "reload": self.container.current.webview.reload() elif string == "home": self.container.current.webview.load_uri(El().start_page) elif string == "source": uri = self.container.current.webview.uri task_helper = TaskHelper() task_helper.load_uri_content(uri, None, self.__on_source_loaded) elif string == "find": find_widget = self.container.current.find_widget find_widget.set_search_mode(True) find_widget.search() elif string == "backward": self.toolbar.actions.backward() elif string == "forward": self.toolbar.actions.forward() elif string == "previous": self.__container.previous() elif string == "next": self.__container.next() elif string == "previous_site": self.__container.sites_manager.previous() elif string == "next_site": self.__container.sites_manager.next() elif string == "print": self.container.current.webview.print() elif string == "private": self.container.add_webview(El().start_page, LoadingType.FOREGROUND, True) elif string == "last_page": El().pages_menu.activate_last_action() elif string == "zoom_in": self.container.current.webview.zoom_in() elif string == "zoom_out": self.container.current.webview.zoom_out() elif string == "zoom_default": self.container.current.webview.zoom_default() elif string == "history": self.toolbar.title.focus_entry("history") elif string == "search": self.toolbar.title.focus_entry("search") elif string == "save": self.toolbar.end.save_page() elif string == "expose": active = self.toolbar.actions.view_button.get_active() self.toolbar.actions.view_button.set_active(not active) elif string == "show_left_panel": value = El().settings.get_value("show-sidebar") El().settings.set_value("show-sidebar", GLib.Variant("b", not value))
class DatabaseAdblock: """ Eolie adblock db """ __DB_PATH = "%s/adblock.db" % EOLIE_DATA_PATH __URIS = ["https://adaway.org/hosts.txt", "https://pgl.yoyo.org/adservers/serverlist.php?" + "hostformat=hosts&showintro=0&startdate%5Bday%5D=" + "&startdate%5Bmonth%5D=&startdate%5Byear%5D=", "http://winhelp2002.mvps.org/hosts.txt", "http://hosts-file.net/ad_servers.txt", "https://pgl.yoyo.org/adservers/serverlist.php?" "hostformat=hosts&showintro=0&mimetype=plaintext"] __CSS_URIS = ["https://easylist-downloads.adblockplus.org/easylist.txt"] __CSS_LOCALIZED_URIS = { "bg": "http://stanev.org/abp/adblock_bg.txt", "zh": "https://easylist-downloads.adblockplus.org/easylistchina.txt", "sk": "https://raw.github.com/tomasko126/" + "easylistczechandslovak/master/filters.txt", "cs": "https://raw.github.com/tomasko126/" + "easylistczechandslovak/master/filters.txt", "nl": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", "de": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", "he": "https://raw.githubusercontent.com/easylist/" + "EasyListHebrew/master/EasyListHebrew.txt", "it": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", "lt": "http://margevicius.lt/easylistlithuania.txt", "es": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", "lv": "https://notabug.org/latvian-list/" + "adblock-latvian/raw/master/lists/latvian-list.txt", "ar": "https://easylist-downloads.adblockplus.org/Liste_AR.txt", "fr": "https://easylist-downloads.adblockplus.org/liste_fr.txt", "ro": "http://www.zoso.ro/pages/rolist.txt", "ru": "https://easylist-downloads.adblockplus.org/advblock.txt", "ja": "http://bit.ly/11QrCfx", "fi": "https://adb.juvander.net/Finland_adb.txt", "cz": "http://adblock.dajbych.net/adblock.txt", "et": "http://gurud.ee/ab.txt", "hu": "https://raw.githubusercontent.com/szpeter80/" + "hufilter/master/hufilter.txt"} __SCHEMA_VERSION = 0 __UPDATE = 172800 __SPECIAL_CHARS = r"([.$+?{}()\[\]\\])" __REPLACE_CHARS = {"^": "(?:[^\w\d_\-.%]|$)", "*": ".*"} # SQLite documentation: # In SQLite, a column with type INTEGER PRIMARY KEY # is an alias for the ROWID. # Here, we define an id INT PRIMARY KEY but never feed it, # this make VACUUM not destroy rowids... __create_adblock = """CREATE TABLE adblock ( id INTEGER PRIMARY KEY, netloc TEXT TEXT NOT NULL, mtime INT NOT NULL )""" __create_adblock_re = """CREATE TABLE adblock_re ( id INTEGER PRIMARY KEY, regex TEXT NOT NULL, mtime INT NOT NULL )""" __create_adblock_re_domain = """CREATE TABLE adblock_re_domain ( id INTEGER PRIMARY KEY, domain TEXT NOT NULL, regex TEXT NOT NULL, mtime INT NOT NULL )""" __create_adblock_re_domain_ex = """CREATE TABLE adblock_re_domain_ex ( id INTEGER PRIMARY KEY, domain TEXT NOT NULL, regex TEXT NOT NULL, mtime INT NOT NULL )""" __create_adblock_css = """CREATE TABLE adblock_css ( id INTEGER PRIMARY KEY, rule TEXT NOT NULL, whitelist TEXT DEFAULT '', blacklist TEXT DEFAULT '', mtime INT NOT NULL )""" __create_adblock_cache = """CREATE TABLE adblock_cache ( id INTEGER PRIMARY KEY, allowed_uri TEXT NOT NULL )""" __create_adblock_idx = """CREATE UNIQUE INDEX idx_adblock ON adblock( netloc)""" __create_adblock_re_idx = """CREATE UNIQUE INDEX idx_adblock_re ON adblock_re(regex)""" __create_adblock_re_domain_idx = """CREATE INDEX idx_adblock_re_domain ON adblock_re_domain(domain)""" __create_adblock_re_domain_regex_idx = """CREATE INDEX idx_adblock_re_domain_regex ON adblock_re_domain(regex)""" __create_adblock_re_domain_ex_idx = """CREATE INDEX idx_adblock_re_domain_ex ON adblock_re_domain(domain)""" __create_adblock_re_domain_regex_ex_idx = """CREATE INDEX idx_adblock_re_domain_regex_ex ON adblock_re_domain(regex)""" __create_adblock_css_black_idx = """CREATE INDEX idx_adblock_css_black ON adblock_css( blacklist)""" __create_adblock_css_white_idx = """CREATE INDEX idx_adblock_css_white ON adblock_css( whitelist)""" __create_adblock_cache_idx = """CREATE UNIQUE INDEX idx_adblock_cache ON adblock_cache( allowed_uri)""" def __init__(self): """ Create database tables or manage update if needed """ self.thread_lock = Lock() self.__cancellable = Gio.Cancellable.new() self.__task_helper = TaskHelper() self.__adblock_mtime = int(time()) self.__regex = None def create_db(self): """ Create databse """ if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR): GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750) # If DB schema changed, remove it if GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR): with SqlCursor(self) as sql: result = sql.execute("PRAGMA user_version") v = result.fetchone() if v is None or v[0] != self.__SCHEMA_VERSION: f = Gio.File.new_for_path(self.__DB_PATH) f.delete() if not GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR): try: # Create db schema with SqlCursor(self) as sql: sql.execute(self.__create_adblock) sql.execute(self.__create_adblock_re) sql.execute(self.__create_adblock_re_domain) sql.execute(self.__create_adblock_re_domain_ex) sql.execute(self.__create_adblock_css) sql.execute(self.__create_adblock_cache) sql.execute(self.__create_adblock_idx) sql.execute(self.__create_adblock_re_idx) sql.execute(self.__create_adblock_re_domain_idx) sql.execute(self.__create_adblock_re_domain_regex_idx) sql.execute(self.__create_adblock_re_domain_ex_idx) sql.execute(self.__create_adblock_re_domain_regex_ex_idx) sql.execute(self.__create_adblock_css_black_idx) sql.execute(self.__create_adblock_css_white_idx) sql.execute(self.__create_adblock_cache_idx) sql.execute("PRAGMA user_version=%s" % self.__SCHEMA_VERSION) except Exception as e: Logger.error("DatabaseAdblock::__init__(): %s", e) def update(self): """ Update database """ if not Gio.NetworkMonitor.get_default().get_network_available(): return # Update adblock_js repo git = GLib.find_program_in_path("git") if git is None: Logger.info(_("For stronger ad blocking, install git command")) else: if GLib.file_test(ADBLOCK_JS, GLib.FileTest.IS_DIR): argv = [git, "-C", ADBLOCK_JS, "pull", "https://gitlab.gnome.org/gnumdk/eolie-adblock.git"] else: argv = [git, "clone", "https://gitlab.gnome.org/gnumdk/eolie-adblock.git", ADBLOCK_JS] (pid, a1, a2, a3) = GLib.spawn_async( argv, flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL) GLib.spawn_close_pid(pid) # DB version is last successful sync mtime try: version = load(open(EOLIE_DATA_PATH + "/adblock.bin", "rb")) except: version = 0 self.__cancellable.reset() if self.__adblock_mtime - version > self.__UPDATE: # Update host rules uris = list(self.__URIS) locales = GLib.get_language_names() user_locale = locales[0].split("_")[0] try: uris += self.__CSS_URIS +\ [self.__CSS_LOCALIZED_URIS[user_locale]] except: uris += self.__CSS_URIS uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_content, uris) def stop(self): """ Stop update """ self.__cancellable.cancel() def get_default_css_rules(self): """ Return default css rules """ rules = "" with SqlCursor(self) as sql: request = "SELECT rule FROM adblock_css WHERE\ blacklist='' AND whitelist=''" result = sql.execute(request) for rule in list(itertools.chain(*result)): rules += "%s,\n" % rule return rules[:-2] + "{display: none !important;}" def get_css_rules(self, uri): """ Return css rules for uri @return str """ rules = "" parsed = urlparse(uri) if parsed.scheme not in ["http", "https"]: return "" netloc = remove_www(parsed.netloc) with SqlCursor(self) as sql: request = "SELECT rule FROM adblock_css WHERE\ (blacklist!='' AND blacklist NOT LIKE ?) OR\ whitelist LIKE ?" result = sql.execute(request, ("%" + netloc + "%", "%" + netloc + "%")) for rule in list(itertools.chain(*result)): rules += "%s,\n" % rule return rules[:-2] + "{display: none !important;}" def is_netloc_blocked(self, netloc): """ Return True if netloc is blocked @param netloc as str @return bool """ try: with SqlCursor(self) as sql: result = sql.execute("SELECT mtime FROM adblock\ WHERE netloc=?", (netloc,)) v = result.fetchone() return v is not None except Exception as e: Logger.error("DatabaseAdblock::is_netloc_blocked(): %s", e) return False def is_uri_blocked(self, uri, netloc): """ Return True if uri is blocked @param uri as str @param netloc as str @return bool """ # We cache result for allowed uris # because regex are quite slow in python with SqlCursor(self) as sql: result = sql.execute("SELECT allowed_uri\ FROM adblock_cache\ WHERE allowed_uri=?", (uri,)) v = result.fetchone() if v is None: # Search in main regexes if self.__regex is None: request = "SELECT regex FROM adblock_re" result = sql.execute(request) rules = list(itertools.chain(*result)) if rules: regexes = "|".join(regex for regex in rules) self.__regex = re.compile(regexes) if self.__regex is not None: blocked_re = bool(self.__regex.search(uri)) else: blocked_re = False # Find in domain regexes request = "SELECT regex FROM adblock_re_domain\ WHERE domain=?" result = sql.execute(request, (netloc,)) rules = list(itertools.chain(*result)) if rules: regexes = "|".join(regex for regex in rules) blocked_re_domain = bool(re.search(regexes, uri)) else: blocked_re_domain = False # If previous regexes blocked uri, check for an exception if blocked_re_domain: request = "SELECT regex FROM adblock_re_domain_ex\ WHERE domain=?" result = sql.execute(request, (netloc,)) rules = list(itertools.chain(*result)) if rules: regexes = "|".join(regex for regex in rules) if bool(re.search(regexes, uri)): blocked_re_domain = False if not blocked_re and not blocked_re_domain: sql.execute("INSERT INTO adblock_cache\ (allowed_uri) VALUES (?)", (uri,)) return False else: return True else: return False def get_cursor(self): """ Return a new sqlite cursor """ try: c = sqlite3.connect(self.__DB_PATH, 600.0) return c except Exception as e: Logger.error("DatabaseAdblock::get_cursor(): %s", e) exit(-1) ####################### # PRIVATE # ####################### def __add_netloc(self, netloc): """ Add a new netloc @param netloc as str """ with SqlCursor(self) as sql: try: sql.execute("INSERT INTO adblock\ (netloc, mtime) VALUES (?, ?)", (netloc, self.__adblock_mtime)) except: sql.execute("UPDATE adblock SET mtime=?\ WHERE netloc=?", (self.__adblock_mtime, netloc)) def __add_regex(self, regex): """ Add a new regex @param regex as str """ with SqlCursor(self) as sql: try: sql.execute("INSERT INTO adblock_re\ (regex, mtime) VALUES (?, ?)", (regex, self.__adblock_mtime)) except: sql.execute("UPDATE adblock_re SET mtime=?\ WHERE regex=?", (self.__adblock_mtime, regex)) def __add_regex_domain(self, regex, domain): """ Add a new regex @param regex as str @param domain """ with SqlCursor(self) as sql: result = sql.execute("SELECT mtime FROM adblock_re_domain\ WHERE regex=? AND domain=?", (regex, domain)) v = result.fetchone() if v is None: sql.execute("INSERT INTO adblock_re_domain\ (regex, domain, mtime) VALUES (?, ?, ?)", (regex, domain, self.__adblock_mtime)) else: sql.execute("UPDATE adblock_re_domain SET mtime=?\ WHERE regex=? AND domain=?", (self.__adblock_mtime, regex, domain)) def __add_regex_domain_ex(self, regex, domain): """ Add a new exception regex @param regex as str @param domain """ with SqlCursor(self) as sql: result = sql.execute("SELECT mtime FROM adblock_re_domain_ex\ WHERE regex=? AND domain=?", (regex, domain)) v = result.fetchone() if v is None: sql.execute("INSERT INTO adblock_re_domain_ex\ (regex, domain, mtime) VALUES (?, ?, ?)", (regex, domain, self.__adblock_mtime)) else: sql.execute("UPDATE adblock_re_domain_ex SET mtime=?\ WHERE regex=? AND domain=?", (self.__adblock_mtime, regex, domain)) def __rule_to_regex(self, rule): """ Convert rule to regex @param rule as str @return regex as str """ try: # Do nothing if rule is already a regex if rule[0] == rule[-1] == "/": return rule[1:-1] rule = re.sub(self.__SPECIAL_CHARS, r"\\\1", rule) # Handle ^ separator character, *, etc... for key in self.__REPLACE_CHARS.keys(): rule = rule.replace(key, self.__REPLACE_CHARS[key]) # End of the address if rule[-1] == "|": rule = rule[:-1] + "$" # Start of the address if rule[0] == "|": rule = "^" + rule[1:] # Escape remaining | but not |$ => see self.__REPLACE_CHARS rule = re.sub("(\|)[^$]", r"\|", rule) return rule except Exception as e: Logger.error("DatabaseAdblock::__rule_to_regex(): %s", e) return None def __save_rules(self, rules): """ Save rules to db @param rules bytes """ SqlCursor.add(self) result = rules.decode('utf-8') count = 0 for line in result.split('\n'): SqlCursor.allow_thread_execution(self) if self.__cancellable.is_cancelled(): SqlCursor.remove(self) raise Exception("Cancelled") if line.startswith('#'): continue array = line.replace( ' ', '\t', 1).replace('\t', '@', 1).split('@') if len(array) <= 1: continue netloc = array[1].replace( ' ', '').replace('\r', '').split('#')[0] # Update entry if exists, create else if netloc != "localhost": Logger.debug("Add filter: %s", netloc) self.__add_netloc(netloc) count += 1 if count == 1000: SqlCursor.commit(self) # Do not flood sqlite, this allow webkit extension to run sleep(0.1) count = 0 SqlCursor.remove(self) def __save_css_default_rule(self, line): """ Save default (without blacklist, whitelist) rule to db @param line as str """ rule = line[2:] # Update entry if exists, create else with SqlCursor(self) as sql: try: sql.execute("INSERT INTO adblock_css\ (rule, mtime) VALUES (?, ?)", (rule, self.__adblock_mtime)) except: sql.execute("UPDATE adblock_css SET mtime=?\ WHERE rule=?", (self.__adblock_mtime, rule)) def __save_css_domain_rule(self, line): """ Save domain rule to db @param line as str """ whitelist = [] blacklist = [] (domains, rule) = line.split("##") for domain in domains.split(","): if domain.startswith("~"): blacklist.append(domain[1:]) else: whitelist.append(domain) str_whitelist = ",".join(whitelist) str_blacklist = ",".join(blacklist) with SqlCursor(self) as sql: result = sql.execute("SELECT mtime FROM adblock_css\ WHERE blacklist=? AND whitelist=?\ AND rule=?", (str_blacklist, str_whitelist, rule)) v = result.fetchone() if v is None: sql.execute("INSERT INTO adblock_css\ (rule, whitelist, blacklist, mtime)\ VALUES (?, ?, ?, ?)", (rule, str_whitelist, str_blacklist, self.__adblock_mtime)) else: sql.execute("UPDATE adblock_css SET mtime=?\ WHERE rule=? and blacklist=? and whitelist=?", (self.__adblock_mtime, rule, str_blacklist, str_whitelist)) def __save_css_exception(self, line): """ Add a new exception @param line as str """ (domain, rule) = line.split("#@#") with SqlCursor(self) as sql: result = sql.execute("SELECT rowid, blacklist FROM adblock_css\ WHERE rule=?", (rule,)) v = result.fetchone() if v is None: sql.execute("INSERT INTO adblock_css\ (rule, whitelist, blacklist, mtime)\ VALUES (?, ?, ?, ?)", (rule, "", domain, self.__adblock_mtime)) else: (rowid, blacklist) = v blacklist += ",%s" % domain sql.execute("UPDATE adblock_css SET blacklist=?\ WHERE rowid=?", (blacklist, rowid)) def __save_abp_rule(self, rule, exception): """ Save abp rule @param rule as str @param exception as bool """ # Simple host rule if rule[:2] == "||": if rule[-1] == "^" and not exception: self.__add_netloc(rule[2:-1]) else: regex = self.__rule_to_regex(rule[2:]) split = re.split("/|\^", rule[2:]) uri = "http://%s" % split[0] parsed = urlparse(uri) if parsed.netloc: if exception: self.__add_regex_domain_ex(regex, parsed.netloc) else: self.__add_regex_domain(regex, parsed.netloc) elif not exception: self.__add_regex(regex) elif not exception: regex = self.__rule_to_regex(rule) if regex is not None: self.__add_regex(regex) def __save_abp_rules(self, rules): """ Save rules to db @param rules as bytes """ SqlCursor.add(self) result = rules.decode("utf-8") count = 0 for line in result.split('\n'): SqlCursor.allow_thread_execution(self) if self.__cancellable.is_cancelled(): SqlCursor.remove(self) raise Exception("Cancelled") if "-abp-" in line or "$" in line or "!" in line or "[" in line: continue elif line.startswith("##"): self.__save_css_default_rule(line) elif "#@#" in line: self.__save_css_exception(line) elif "##" in line: self.__save_css_domain_rule(line) elif line.startswith("@@"): self.__save_abp_rule(line[2:], True) else: self.__save_abp_rule(line, False) Logger.debug("Add abp filter: %s", line) count += 1 if count == 1000: SqlCursor.commit(self) # Do not flood sqlite, this allow webkit extension to run sleep(0.1) count = 0 SqlCursor.remove(self) def __on_save_rules(self, result, uris=[]): """ Load next uri, if finished, load CSS rules @param result (unused) @param uris as [str] """ if self.__cancellable.is_cancelled(): return if uris: uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_content, uris) else: with SqlCursor(self) as sql: sql.execute("DELETE FROM adblock\ WHERE mtime!=?", (self.__adblock_mtime,)) sql.execute("DELETE FROM adblock_re\ WHERE mtime!=?", (self.__adblock_mtime,)) sql.execute("DELETE FROM adblock_re_domain\ WHERE mtime!=?", (self.__adblock_mtime,)) sql.execute("DELETE FROM adblock_re_domain_ex\ WHERE mtime!=?", (self.__adblock_mtime,)) sql.execute("DELETE FROM adblock_css\ WHERE mtime!=?", (self.__adblock_mtime,)) sql.execute("DELETE FROM adblock_cache") try: dump(self.__adblock_mtime, open(EOLIE_DATA_PATH + "/adblock.bin", "wb")) except Exception as e: Logger.error("DatabaseAdblock::__on_save_rules(): %s", e) def __on_load_uri_content(self, uri, status, content, uris): """ Save loaded values @param uri as str @param status as bool @param content as bytes @param uris as [str] """ Logger.debug("DatabaseAdblock::__on_load_uri_content(): %s", uri) if status: if uri in self.__URIS: self.__task_helper.run(self.__save_rules, content, callback=(self.__on_save_rules, uris)) else: self.__task_helper.run(self.__save_abp_rules, content, callback=(self.__on_save_rules, uris)) else: self.__on_save_rules(None, uris)
class DatabasePhishing: """ Phishing database """ DB_PATH = "%s/phishing.db" % EOLIE_DATA_PATH __URI = "http://data.phishtank.com/data/online-valid.json" # SQLite documentation: # In SQLite, a column with type INTEGER PRIMARY KEY # is an alias for the ROWID. # Here, we define an id INT PRIMARY KEY but never feed it, # this make VACUUM not destroy rowids... __create_phishing = '''CREATE TABLE phishing ( id INTEGER PRIMARY KEY, uri TEXT NOT NULL, mtime INT NOT NULL )''' def __init__(self): """ Create database tables or manage update if needed """ self.__cancellable = Gio.Cancellable.new() self.__task_helper = TaskHelper() # Lazy loading if not empty if not GLib.file_test(self.DB_PATH, GLib.FileTest.IS_REGULAR): try: if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR): GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750) # Create db schema with SqlCursor(self) as sql: sql.execute(self.__create_phishing) sql.commit() except Exception as e: print("DatabasePhishing::__init__(): %s" % e) def update(self): """ Update database """ if not Gio.NetworkMonitor.get_default().get_network_available(): return # Get in db mtime # Only update if filters older than one day mtime = 0 with SqlCursor(self) as sql: result = sql.execute("SELECT mtime FROM phishing LIMIT 1") v = result.fetchone() if v is not None: mtime = v[0] self.__mtime = int(time()) if self.__mtime - mtime < 86400: return # Update phishing db self.__cancellable.reset() self.__on_load_uri_content(None, False, b"", [self.__URI]) def is_phishing(self, uri): """ True if uri is phishing @param uri as str @return bool """ uri = uri.rstrip("/") try: with SqlCursor(self) as sql: result = sql.execute( "SELECT uri FROM phishing\ WHERE uri=?", (uri, )) v = result.fetchone() return v is not None except Exception as e: print("DatabasePhishing::is_phishing():", e) return False def stop(self): """ Stop update """ self.__cancellable.cancel() self.__stop = True def get_cursor(self): """ Return a new sqlite cursor """ try: c = sqlite3.connect(self.DB_PATH, 600.0) return c except Exception as e: print(e) exit(-1) ####################### # PRIVATE # ####################### def __save_rules(self, rules, uris): """ Save rules to db @param rules as bytes @param uris as [str] """ SqlCursor.add(self) result = rules.decode('utf-8') j = json.loads(result) with SqlCursor(self) as sql: count = 0 for item in j: if self.__cancellable.is_cancelled(): raise IOError("Cancelled") sql.execute( "INSERT INTO phishing\ (uri, mtime) VALUES (?, ?)", (item["url"].rstrip("/"), self.__mtime)) count += 1 if count == 1000: sql.commit() count = 0 sql.commit() # We are the last call to save_rules()? # Delete removed entries and commit if not uris: with SqlCursor(self) as sql: sql.execute( "DELETE FROM phishing\ WHERE mtime!=?", (self.__mtime, )) sql.commit() SqlCursor.remove(self) def __on_load_uri_content(self, uri, status, content, uris): """ Load pending uris @param uri as str @param status as bool @param content as bytes @param uris as [str] """ if status: self.__task_helper.run(self.__save_rules, content, uris) if uris: uri = uris.pop(0) self.__task_helper.load_uri_content(uri, self.__cancellable, self.__on_load_uri_content, uris)