예제 #1
0
    def __on_entry_changed_timeout(self, entry, value):
        """
            Update popover search if needed
            @param entry as Gtk.Entry
            @param value as str
        """
        task_helper = TaskHelper()
        self.__entry_changed_timeout = None

        self.__window.container.current.webview.add_text_entry(value)

        # Populate completion model
        task_helper.run(self.__populate_completion, (value,))

        self.__cancellable.cancel()
        self.__cancellable.reset()
        parsed = urlparse(value)
        self.__show_related_view(value)
        network = Gio.NetworkMonitor.get_default().get_network_available()
        is_uri = parsed.scheme in ["about, http", "file", "https", "populars"]
        if is_uri:
            self.__popover.set_search_text(parsed.netloc + parsed.path)
        else:
            self.__popover.set_search_text(value)

        parsed = urlparse(self.__uri)
        if value and not is_uri and network:
            El().search.search_suggestions(value,
                                           self.__cancellable,
                                           self.__search_suggestion)

        self.__entry.set_icon_from_icon_name(Gtk.EntryIconPosition.PRIMARY,
                                             "system-search-symbolic")
        self.__entry.set_icon_tooltip_text(Gtk.EntryIconPosition.PRIMARY,
                                           "")
예제 #2
0
    def quit(self, vacuum=False):
        """
            Quit application
            @param vacuum as bool
        """
        self.__save_state()
        for window in self.windows:
            window.hide()
        # Stop pending tasks
        self.download_manager.cancel()
        self.adblock.stop()
        # Clear history
        active_id = str(self.settings.get_enum("history-storage"))
        if active_id != TimeSpan.FOREVER:
            atime = time()
            if active_id != TimeSpan.NEVER:
                atime -= TimeSpanValues[active_id] / 1000000
            self.history.clear_to(int(atime))

        if self.sync_worker is not None:
            if self.sync_worker.syncing:
                self.sync_worker.stop()
            self.sync_worker.save_pendings()
        if vacuum:
            task_helper = TaskHelper()
            task_helper.run(self.__vacuum,
                            callback=(lambda x: Gio.Application.quit(self), ))
        else:
            Gio.Application.quit(self)
예제 #3
0
 def quit(self, vacuum=False):
     """
         Quit application
         @param vacuum as bool
     """
     # Save webpage state
     self.__save_state()
     # Stop pending tasks
     self.download_manager.cancel()
     self.adblock.stop()
     # Clear history
     active_id = str(self.settings.get_enum("history-storage"))
     if active_id != TimeSpan.FOREVER:
         atime = time()
         if active_id != TimeSpan.NEVER:
             atime -= TimeSpanValues[active_id]/1000000
         self.history.clear_to(int(atime))
     # If sync is running, to avoid db lock, we do not vacuum
     if self.sync_worker is not None and self.sync_worker.syncing:
         self.sync_worker.stop()
         Gio.Application.quit(self)
     elif vacuum:
         task_helper = TaskHelper()
         task_helper.run(self.__vacuum,
                         (),
                         lambda x: Gio.Application.quit(self))
     else:
         Gio.Application.quit(self)
예제 #4
0
 def remove_from_passwords(self, uuid):
     """
         Remove password from passwords collection
         @param uuid as str
     """
     if self.__username and self.__password:
         task_helper = TaskHelper()
         task_helper.run(self.__remove_from_passwords, uuid)
예제 #5
0
 def remove_from_bookmarks(self, guid):
     """
         Remove bookmark guid from remote bookmarks
         @param guid as str
     """
     if self.__username and self.__password:
         task_helper = TaskHelper()
         task_helper.run(self.__remove_from_bookmarks, guid)
예제 #6
0
 def remove_from_history(self, guid):
     """
         Remove history guid from remote history
         @param guid as str
     """
     if self.__username and self.__password:
         task_helper = TaskHelper()
         task_helper.run(self.__remove_from_history, guid)
예제 #7
0
 def push_bookmark(self, bookmark_id):
     """
         Push bookmark id
         @param bookmark_id as int
     """
     if self.__username and self.__password:
         task_helper = TaskHelper()
         task_helper.run(self.__push_bookmark, bookmark_id)
예제 #8
0
 def sync(self):
     """
         Start syncing
     """
     if Gio.NetworkMonitor.get_default().get_network_available() and\
             self.__username and self.__password and not self.syncing:
         task_helper = TaskHelper()
         task_helper.run(self.__sync)
예제 #9
0
 def push_history(self, history_id):
     """
         Push history id
         @param history_id as int
     """
     if self.__username and self.__password:
         task_helper = TaskHelper()
         task_helper.run(self.__push_history, history_id)
예제 #10
0
 def remove_from_history(self, guid):
     """
         Remove history guid from remote history
         @param guid as str
     """
     if Gio.NetworkMonitor.get_default().get_network_available():
         task_helper = TaskHelper()
         task_helper.run(self.__remove_from_history, (guid, ))
예제 #11
0
 def push_history(self, history_ids):
     """
         Push history ids
         @param history_ids as [int]
     """
     if Gio.NetworkMonitor.get_default().get_network_available():
         task_helper = TaskHelper()
         task_helper.run(self.__push_history, (history_ids, ))
예제 #12
0
 def remove_from_passwords(self, uuid):
     """
         Remove password from passwords collection
         @param uuid as str
     """
     if Gio.NetworkMonitor.get_default().get_network_available():
         task_helper = TaskHelper()
         task_helper.run(self.__remove_from_passwords, (uuid, ))
예제 #13
0
 def _on_button_clicked(self, button):
     """
         Save visible images
         @param button as Gtk.Button
     """
     task_helper = TaskHelper()
     task_helper.run(self.__move_images)
     self.__spinner.start()
예제 #14
0
 def remove_from_bookmarks(self, guid):
     """
         Remove bookmark guid from remote bookmarks
         @param guid as str
     """
     if Gio.NetworkMonitor.get_default().get_network_available():
         task_helper = TaskHelper()
         task_helper.run(self.__remove_from_bookmarks, (guid, ))
예제 #15
0
 def push_history(self, history_ids):
     """
         Push history ids
         @param history_ids as [int]
     """
     if Gio.NetworkMonitor.get_default().get_network_available() and\
             self.__username and self.__password:
         task_helper = TaskHelper()
         task_helper.run(self.__push_history, history_ids)
예제 #16
0
 def sync(self, loop=False, first_sync=False):
     """
         Start syncing, you need to check sync_status property
         @param loop as bool -> for GLib.timeout_add()
         @param first_sync as bool
     """
     if Gio.NetworkMonitor.get_default().get_network_available() and\
             self.__username and self.__password and not self.syncing:
         task_helper = TaskHelper()
         task_helper.run(self.__sync, first_sync)
     return loop
예제 #17
0
 def sync(self, loop=False, first_sync=False):
     """
         Start syncing, you need to check sync_status property
         @param loop as bool -> for GLib.timeout_add()
         @param first_sync as bool
     """
     if self.syncing or\
             not Gio.NetworkMonitor.get_default().get_network_available():
         return
     task_helper = TaskHelper()
     task_helper.run(self.__sync, (first_sync, ))
     return loop
예제 #18
0
 def push_password(self, user_form_name, user_form_value, pass_form_name,
                   pass_form_value, uri, form_uri, uuid):
     """
         Push password
         @param user_form_name as str
         @param user_form_value as str
         @param pass_form_name as str
         @param pass_form_value as str
         @param uri as str
         @param form_uri as str
         @param uuid as str
     """
     if Gio.NetworkMonitor.get_default().get_network_available():
         task_helper = TaskHelper()
         task_helper.run(self.__push_password,
                         (user_form_name, user_form_value, pass_form_name,
                          pass_form_value, uri, form_uri, uuid))
예제 #19
0
 def push_password(self, user_form_name, user_form_value, pass_form_name,
                   pass_form_value, uri, form_uri, uuid):
     """
         Push password
         @param user_form_name as str
         @param user_form_value as str
         @param pass_form_name as str
         @param pass_form_value as str
         @param uri as str
         @param form_uri as str
         @param uuid as str
     """
     if self.__username and self.__password:
         task_helper = TaskHelper()
         task_helper.run(self.__push_password, user_form_name,
                         user_form_value, pass_form_name, pass_form_value,
                         uri, form_uri, uuid)
예제 #20
0
 def _on_infobar_response(self, infobar, response_id):
     """
         Handle user response and remove wanted history ids
         @param infobar as Gtk.InfoBar
         @param response_id as int
     """
     if response_id == 1:
         active_id = self.__infobar_select.get_active_id()
         if active_id == TimeSpan.CUSTOM:
             (year, month, day) = self.__calendar.get_date()
             date = "%02d/%02d/%s" % (day, month + 1, year)
             atime = mktime(datetime.strptime(date, "%d/%m/%Y").timetuple())
         else:
             atime = int(time() - TimeSpanValues[active_id] / 1000000)
         task_helper = TaskHelper()
         task_helper.run(self.__clear_history, (atime, ))
     infobar.hide()
예제 #21
0
 def _on_sync_button_clicked(self, button):
     """
         Connect to Mozilla Sync to get tokens
         @param button as Gtk.Button
     """
     icon_name = self.__result_image.get_icon_name()[0]
     if icon_name == "network-transmit-receive-symbolic":
         El().sync_worker.stop(True)
         El().sync_worker.delete_secret()
         self.__setup_sync_button(False)
     else:
         El().sync_worker.delete_secret()
         self.__result_label.set_text(_("Connecting…"))
         button.set_sensitive(False)
         self.__result_image.set_from_icon_name("content-loading-symbolic",
                                                Gtk.IconSize.MENU)
         task_helper = TaskHelper()
         task_helper.run(self.__connect_mozilla_sync,
                         (self.__login_entry.get_text(),
                          self.__password_entry.get_text()))
예제 #22
0
    def __on_entry_changed_timeout(self, entry, value):
        """
            Update popover search if needed
            @param entry as Gtk.Entry
            @param value as str
        """
        task_helper = TaskHelper()
        self.__entry_changed_id = None

        self.__window.container.current.webview.add_text_entry(value)

        # Populate completion model
        task_helper.run(self.__populate_completion, value)

        self.__cancellable.cancel()
        self.__cancellable.reset()
        parsed = urlparse(value)

        network = Gio.NetworkMonitor.get_default().get_network_available()
        is_uri = parsed.scheme in ["about, http", "file", "https", "populars"]
        if is_uri:
            self.__popover.set_search_text(parsed.netloc + parsed.path)
        else:
            self.__popover.set_search_text(value)

        # Remove any pending suggestion search
        if self.__suggestion_id is not None:
            GLib.source_remove(self.__suggestion_id)
            self.__suggestion_id = None
        # Search for suggestions if needed
        if App().settings.get_value("enable-suggestions") and\
                value and not is_uri and network:
            self.__suggestion_id = GLib.timeout_add(
                50, self.__on_suggestion_timeout, value)
        task_helper.run(self.__search_in_current_views, value)
        self.__entry.set_icon_from_icon_name(Gtk.EntryIconPosition.PRIMARY,
                                             "system-search-symbolic")
        self.__entry.set_icon_tooltip_text(Gtk.EntryIconPosition.PRIMARY, "")
예제 #23
0
class DatabasePhishing:
    """
        Phishing database
    """
    DB_PATH = "%s/phishing.db" % EOLIE_DATA_PATH
    __URI = "http://data.phishtank.com/data/online-valid.json"
    # SQLite documentation:
    # In SQLite, a column with type INTEGER PRIMARY KEY
    # is an alias for the ROWID.
    # Here, we define an id INT PRIMARY KEY but never feed it,
    # this make VACUUM not destroy rowids...
    __create_phishing = '''CREATE TABLE phishing (
                                               id INTEGER PRIMARY KEY,
                                               uri TEXT NOT NULL,
                                               mtime INT NOT NULL
                                               )'''

    def __init__(self):
        """
            Create database tables or manage update if needed
        """
        self.__cancellable = Gio.Cancellable.new()
        self.__task_helper = TaskHelper()
        # Lazy loading if not empty
        if not GLib.file_test(self.DB_PATH, GLib.FileTest.IS_REGULAR):
            try:
                if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR):
                    GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750)
                # Create db schema
                with SqlCursor(self) as sql:
                    sql.execute(self.__create_phishing)
                    sql.commit()
            except Exception as e:
                print("DatabasePhishing::__init__(): %s" % e)

    def update(self):
        """
            Update database
        """
        if not Gio.NetworkMonitor.get_default().get_network_available():
            return
        # Get in db mtime
        # Only update if filters older than one day
        mtime = 0
        with SqlCursor(self) as sql:
            result = sql.execute("SELECT mtime FROM phishing LIMIT 1")
            v = result.fetchone()
            if v is not None:
                mtime = v[0]
        self.__mtime = int(time())
        if self.__mtime - mtime < 86400:
            return
        # Update phishing db
        self.__cancellable.reset()
        self.__on_load_uri_content(None, False, b"", [self.__URI])

    def is_phishing(self, uri):
        """
            True if uri is phishing
            @param uri as str
            @return bool
        """
        uri = uri.rstrip("/")
        try:
            with SqlCursor(self) as sql:
                result = sql.execute(
                    "SELECT uri FROM phishing\
                                      WHERE uri=?", (uri, ))
                v = result.fetchone()
                return v is not None
        except Exception as e:
            print("DatabasePhishing::is_phishing():", e)
            return False

    def stop(self):
        """
            Stop update
        """
        self.__cancellable.cancel()
        self.__stop = True

    def get_cursor(self):
        """
            Return a new sqlite cursor
        """
        try:
            c = sqlite3.connect(self.DB_PATH, 600.0)
            return c
        except Exception as e:
            print(e)
            exit(-1)

#######################
# PRIVATE             #
#######################

    def __save_rules(self, rules, uris):
        """
            Save rules to db
            @param rules as bytes
            @param uris as [str]
        """
        SqlCursor.add(self)
        result = rules.decode('utf-8')
        j = json.loads(result)
        with SqlCursor(self) as sql:
            count = 0
            for item in j:
                if self.__cancellable.is_cancelled():
                    raise IOError("Cancelled")
                sql.execute(
                    "INSERT INTO phishing\
                             (uri, mtime) VALUES (?, ?)",
                    (item["url"].rstrip("/"), self.__mtime))
                count += 1
                if count == 1000:
                    sql.commit()
                    count = 0
            sql.commit()
        # We are the last call to save_rules()?
        # Delete removed entries and commit
        if not uris:
            with SqlCursor(self) as sql:
                sql.execute(
                    "DELETE FROM phishing\
                             WHERE mtime!=?", (self.__mtime, ))
                sql.commit()
        SqlCursor.remove(self)

    def __on_load_uri_content(self, uri, status, content, uris):
        """
            Load pending uris
            @param uri as str
            @param status as bool
            @param content as bytes
            @param uris as [str]
        """
        if status:
            self.__task_helper.run(self.__save_rules, content, uris)
        if uris:
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri, self.__cancellable,
                                                self.__on_load_uri_content,
                                                uris)
예제 #24
0
class DatabaseAdblock:
    """
        Eolie adblock db
    """
    __DB_PATH = "%s/adblock.db" % EOLIE_DATA_PATH

    __URIS = ["https://adaway.org/hosts.txt",
              "https://pgl.yoyo.org/adservers/serverlist.php?" +
              "hostformat=hosts&showintro=0&startdate%5Bday%5D=" +
              "&startdate%5Bmonth%5D=&startdate%5Byear%5D=",
              "http://winhelp2002.mvps.org/hosts.txt",
              "http://hosts-file.net/ad_servers.txt",
              "https://pgl.yoyo.org/adservers/serverlist.php?"
              "hostformat=hosts&showintro=0&mimetype=plaintext"]

    __CSS_URIS = ["https://easylist-downloads.adblockplus.org/easylist.txt"]

    __CSS_LOCALIZED_URIS = {
        "bg": "http://stanev.org/abp/adblock_bg.txt",
        "zh": "https://easylist-downloads.adblockplus.org/easylistchina.txt",
        "sk": "https://raw.github.com/tomasko126/" +
              "easylistczechandslovak/master/filters.txt",
        "cs": "https://raw.github.com/tomasko126/" +
              "easylistczechandslovak/master/filters.txt",
        "nl": "https://easylist-downloads.adblockplus.org/easylistdutch.txt",
        "de": "https://easylist-downloads.adblockplus.org/easylistgermany.txt",
        "he": "https://raw.githubusercontent.com/easylist/" +
              "EasyListHebrew/master/EasyListHebrew.txt",
        "it": "https://easylist-downloads.adblockplus.org/easylistitaly.txt",
        "lt": "http://margevicius.lt/easylistlithuania.txt",
        "es": "https://easylist-downloads.adblockplus.org/easylistspanish.txt",
        "lv": "https://notabug.org/latvian-list/" +
              "adblock-latvian/raw/master/lists/latvian-list.txt",
        "ar": "https://easylist-downloads.adblockplus.org/Liste_AR.txt",
        "fr": "https://easylist-downloads.adblockplus.org/liste_fr.txt",
        "ro": "http://www.zoso.ro/pages/rolist.txt",
        "ru": "https://easylist-downloads.adblockplus.org/advblock.txt",
        "ja": "http://bit.ly/11QrCfx",
        "fi": "https://adb.juvander.net/Finland_adb.txt",
        "cz": "http://adblock.dajbych.net/adblock.txt",
        "et": "http://gurud.ee/ab.txt",
        "hu": "https://raw.githubusercontent.com/szpeter80/" +
              "hufilter/master/hufilter.txt"}

    __SCHEMA_VERSION = 0
    __UPDATE = 172800

    __SPECIAL_CHARS = r"([.$+?{}()\[\]\\])"
    __REPLACE_CHARS = {"^": "(?:[^\w\d_\-.%]|$)",
                       "*": ".*"}

    # SQLite documentation:
    # In SQLite, a column with type INTEGER PRIMARY KEY
    # is an alias for the ROWID.
    # Here, we define an id INT PRIMARY KEY but never feed it,
    # this make VACUUM not destroy rowids...
    __create_adblock = """CREATE TABLE adblock (
                                               id INTEGER PRIMARY KEY,
                                               netloc TEXT TEXT NOT NULL,
                                               mtime INT NOT NULL
                                               )"""
    __create_adblock_re = """CREATE TABLE adblock_re (
                                               id INTEGER PRIMARY KEY,
                                               regex TEXT NOT NULL,
                                               mtime INT NOT NULL
                                               )"""
    __create_adblock_re_domain = """CREATE TABLE adblock_re_domain (
                                               id INTEGER PRIMARY KEY,
                                               domain TEXT NOT NULL,
                                               regex TEXT NOT NULL,
                                               mtime INT NOT NULL
                                               )"""
    __create_adblock_re_domain_ex = """CREATE TABLE adblock_re_domain_ex (
                                               id INTEGER PRIMARY KEY,
                                               domain TEXT NOT NULL,
                                               regex TEXT NOT NULL,
                                               mtime INT NOT NULL
                                               )"""
    __create_adblock_css = """CREATE TABLE adblock_css (
                                               id INTEGER PRIMARY KEY,
                                               rule TEXT NOT NULL,
                                               whitelist TEXT DEFAULT '',
                                               blacklist TEXT DEFAULT '',
                                               mtime INT NOT NULL
                                               )"""
    __create_adblock_cache = """CREATE TABLE adblock_cache (
                                               id INTEGER PRIMARY KEY,
                                               allowed_uri TEXT NOT NULL
                                               )"""
    __create_adblock_idx = """CREATE UNIQUE INDEX idx_adblock ON adblock(
                                               netloc)"""
    __create_adblock_re_idx = """CREATE UNIQUE INDEX idx_adblock_re
                                               ON adblock_re(regex)"""
    __create_adblock_re_domain_idx = """CREATE INDEX
                                               idx_adblock_re_domain
                                               ON adblock_re_domain(domain)"""
    __create_adblock_re_domain_regex_idx = """CREATE INDEX
                                               idx_adblock_re_domain_regex
                                               ON adblock_re_domain(regex)"""
    __create_adblock_re_domain_ex_idx = """CREATE INDEX
                                               idx_adblock_re_domain_ex
                                               ON adblock_re_domain(domain)"""
    __create_adblock_re_domain_regex_ex_idx = """CREATE INDEX
                                               idx_adblock_re_domain_regex_ex
                                               ON adblock_re_domain(regex)"""
    __create_adblock_css_black_idx = """CREATE INDEX
                                               idx_adblock_css_black
                                               ON adblock_css(
                                               blacklist)"""
    __create_adblock_css_white_idx = """CREATE INDEX
                                               idx_adblock_css_white
                                               ON adblock_css(
                                               whitelist)"""
    __create_adblock_cache_idx = """CREATE UNIQUE INDEX idx_adblock_cache
                                               ON adblock_cache(
                                               allowed_uri)"""

    def __init__(self):
        """
            Create database tables or manage update if needed
        """
        self.thread_lock = Lock()
        self.__cancellable = Gio.Cancellable.new()
        self.__task_helper = TaskHelper()
        self.__adblock_mtime = int(time())
        self.__regex = None

    def create_db(self):
        """
            Create databse
        """
        if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR):
            GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750)
        # If DB schema changed, remove it
        if GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR):
            with SqlCursor(self) as sql:
                result = sql.execute("PRAGMA user_version")
                v = result.fetchone()
                if v is None or v[0] != self.__SCHEMA_VERSION:
                    f = Gio.File.new_for_path(self.__DB_PATH)
                    f.delete()
        if not GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR):
            try:
                # Create db schema
                with SqlCursor(self) as sql:
                    sql.execute(self.__create_adblock)
                    sql.execute(self.__create_adblock_re)
                    sql.execute(self.__create_adblock_re_domain)
                    sql.execute(self.__create_adblock_re_domain_ex)
                    sql.execute(self.__create_adblock_css)
                    sql.execute(self.__create_adblock_cache)
                    sql.execute(self.__create_adblock_idx)
                    sql.execute(self.__create_adblock_re_idx)
                    sql.execute(self.__create_adblock_re_domain_idx)
                    sql.execute(self.__create_adblock_re_domain_regex_idx)
                    sql.execute(self.__create_adblock_re_domain_ex_idx)
                    sql.execute(self.__create_adblock_re_domain_regex_ex_idx)
                    sql.execute(self.__create_adblock_css_black_idx)
                    sql.execute(self.__create_adblock_css_white_idx)
                    sql.execute(self.__create_adblock_cache_idx)
                    sql.execute("PRAGMA user_version=%s" %
                                self.__SCHEMA_VERSION)
            except Exception as e:
                Logger.error("DatabaseAdblock::__init__(): %s", e)

    def update(self):
        """
            Update database
        """
        if not Gio.NetworkMonitor.get_default().get_network_available():
            return
        # Update adblock_js repo
        git = GLib.find_program_in_path("git")
        if git is None:
            Logger.info(_("For stronger ad blocking, install git command"))
        else:
            if GLib.file_test(ADBLOCK_JS, GLib.FileTest.IS_DIR):
                argv = [git,
                        "-C",
                        ADBLOCK_JS,
                        "pull",
                        "https://gitlab.gnome.org/gnumdk/eolie-adblock.git"]
            else:
                argv = [git,
                        "clone",
                        "https://gitlab.gnome.org/gnumdk/eolie-adblock.git",
                        ADBLOCK_JS]
            (pid, a1, a2, a3) = GLib.spawn_async(
                argv,
                flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL)
            GLib.spawn_close_pid(pid)

        # DB version is last successful sync mtime
        try:
            version = load(open(EOLIE_DATA_PATH + "/adblock.bin", "rb"))
        except:
            version = 0
        self.__cancellable.reset()
        if self.__adblock_mtime - version > self.__UPDATE:
            # Update host rules
            uris = list(self.__URIS)
            locales = GLib.get_language_names()
            user_locale = locales[0].split("_")[0]
            try:
                uris += self.__CSS_URIS +\
                    [self.__CSS_LOCALIZED_URIS[user_locale]]
            except:
                uris += self.__CSS_URIS
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri,
                                                self.__cancellable,
                                                self.__on_load_uri_content,
                                                uris)

    def stop(self):
        """
            Stop update
        """
        self.__cancellable.cancel()

    def get_default_css_rules(self):
        """
            Return default css rules
        """
        rules = ""
        with SqlCursor(self) as sql:
            request = "SELECT rule FROM adblock_css WHERE\
                       blacklist='' AND whitelist=''"
            result = sql.execute(request)
            for rule in list(itertools.chain(*result)):
                rules += "%s,\n" % rule
        return rules[:-2] + "{display: none !important;}"

    def get_css_rules(self, uri):
        """
            Return css rules for uri
            @return str
        """
        rules = ""
        parsed = urlparse(uri)
        if parsed.scheme not in ["http", "https"]:
            return ""
        netloc = remove_www(parsed.netloc)
        with SqlCursor(self) as sql:
            request = "SELECT rule FROM adblock_css WHERE\
                       (blacklist!='' AND blacklist NOT LIKE ?) OR\
                       whitelist LIKE ?"
            result = sql.execute(request, ("%" + netloc + "%",
                                           "%" + netloc + "%"))
            for rule in list(itertools.chain(*result)):
                rules += "%s,\n" % rule
        return rules[:-2] + "{display: none !important;}"

    def is_netloc_blocked(self, netloc):
        """
            Return True if netloc is blocked
            @param netloc as str
            @return bool
        """
        try:
            with SqlCursor(self) as sql:
                result = sql.execute("SELECT mtime FROM adblock\
                                      WHERE netloc=?", (netloc,))
                v = result.fetchone()
                return v is not None
        except Exception as e:
            Logger.error("DatabaseAdblock::is_netloc_blocked(): %s", e)
            return False

    def is_uri_blocked(self, uri, netloc):
        """
            Return True if uri is blocked
            @param uri as str
            @param netloc as str
            @return bool
        """
        # We cache result for allowed uris
        # because regex are quite slow in python
        with SqlCursor(self) as sql:
            result = sql.execute("SELECT allowed_uri\
                                  FROM adblock_cache\
                                  WHERE allowed_uri=?", (uri,))
            v = result.fetchone()
            if v is None:
                # Search in main regexes
                if self.__regex is None:
                    request = "SELECT regex FROM adblock_re"
                    result = sql.execute(request)
                    rules = list(itertools.chain(*result))
                    if rules:
                        regexes = "|".join(regex for regex in rules)
                        self.__regex = re.compile(regexes)
                if self.__regex is not None:
                    blocked_re = bool(self.__regex.search(uri))
                else:
                    blocked_re = False
                # Find in domain regexes
                request = "SELECT regex FROM adblock_re_domain\
                           WHERE domain=?"
                result = sql.execute(request, (netloc,))
                rules = list(itertools.chain(*result))
                if rules:
                    regexes = "|".join(regex for regex in rules)
                    blocked_re_domain = bool(re.search(regexes, uri))
                else:
                    blocked_re_domain = False
                # If previous regexes blocked uri, check for an exception
                if blocked_re_domain:
                    request = "SELECT regex FROM adblock_re_domain_ex\
                               WHERE domain=?"
                    result = sql.execute(request, (netloc,))
                    rules = list(itertools.chain(*result))
                    if rules:
                        regexes = "|".join(regex for regex in rules)
                        if bool(re.search(regexes, uri)):
                            blocked_re_domain = False
                if not blocked_re and not blocked_re_domain:
                    sql.execute("INSERT INTO adblock_cache\
                                (allowed_uri) VALUES (?)",
                                (uri,))
                    return False
                else:
                    return True
            else:
                return False

    def get_cursor(self):
        """
            Return a new sqlite cursor
        """
        try:
            c = sqlite3.connect(self.__DB_PATH, 600.0)
            return c
        except Exception as e:
            Logger.error("DatabaseAdblock::get_cursor(): %s", e)
            exit(-1)

#######################
# PRIVATE             #
#######################
    def __add_netloc(self, netloc):
        """
            Add a new netloc
            @param netloc as str
        """
        with SqlCursor(self) as sql:
            try:
                sql.execute("INSERT INTO adblock\
                            (netloc, mtime) VALUES (?, ?)",
                            (netloc, self.__adblock_mtime))
            except:
                sql.execute("UPDATE adblock SET mtime=?\
                             WHERE netloc=?",
                            (self.__adblock_mtime, netloc))

    def __add_regex(self, regex):
        """
            Add a new regex
            @param regex as str
        """
        with SqlCursor(self) as sql:
            try:
                sql.execute("INSERT INTO adblock_re\
                            (regex, mtime) VALUES (?, ?)",
                            (regex, self.__adblock_mtime))
            except:
                sql.execute("UPDATE adblock_re SET mtime=?\
                             WHERE regex=?",
                            (self.__adblock_mtime, regex))

    def __add_regex_domain(self, regex, domain):
        """
            Add a new regex
            @param regex as str
            @param domain
        """
        with SqlCursor(self) as sql:
            result = sql.execute("SELECT mtime FROM adblock_re_domain\
                                  WHERE regex=? AND domain=?",
                                 (regex, domain))
            v = result.fetchone()
            if v is None:
                sql.execute("INSERT INTO adblock_re_domain\
                            (regex, domain, mtime) VALUES (?, ?, ?)",
                            (regex, domain, self.__adblock_mtime))
            else:
                sql.execute("UPDATE adblock_re_domain SET mtime=?\
                             WHERE regex=? AND domain=?",
                            (self.__adblock_mtime, regex, domain))

    def __add_regex_domain_ex(self, regex, domain):
        """
            Add a new exception regex
            @param regex as str
            @param domain
        """
        with SqlCursor(self) as sql:
            result = sql.execute("SELECT mtime FROM adblock_re_domain_ex\
                                  WHERE regex=? AND domain=?",
                                 (regex, domain))
            v = result.fetchone()
            if v is None:
                sql.execute("INSERT INTO adblock_re_domain_ex\
                            (regex, domain, mtime) VALUES (?, ?, ?)",
                            (regex, domain, self.__adblock_mtime))
            else:
                sql.execute("UPDATE adblock_re_domain_ex SET mtime=?\
                             WHERE regex=? AND domain=?",
                            (self.__adblock_mtime, regex, domain))

    def __rule_to_regex(self, rule):
        """
            Convert rule to regex
            @param rule as str
            @return regex as str
        """
        try:
            # Do nothing if rule is already a regex
            if rule[0] == rule[-1] == "/":
                return rule[1:-1]

            rule = re.sub(self.__SPECIAL_CHARS, r"\\\1", rule)

            # Handle ^ separator character, *, etc...
            for key in self.__REPLACE_CHARS.keys():
                rule = rule.replace(key, self.__REPLACE_CHARS[key])
            # End of the address
            if rule[-1] == "|":
                rule = rule[:-1] + "$"
            # Start of the address
            if rule[0] == "|":
                rule = "^" + rule[1:]
            # Escape remaining | but not |$ => see self.__REPLACE_CHARS
            rule = re.sub("(\|)[^$]", r"\|", rule)
            return rule
        except Exception as e:
            Logger.error("DatabaseAdblock::__rule_to_regex(): %s", e)
            return None

    def __save_rules(self, rules):
        """
            Save rules to db
            @param rules bytes
        """
        SqlCursor.add(self)
        result = rules.decode('utf-8')
        count = 0
        for line in result.split('\n'):
            SqlCursor.allow_thread_execution(self)
            if self.__cancellable.is_cancelled():
                SqlCursor.remove(self)
                raise Exception("Cancelled")
            if line.startswith('#'):
                continue
            array = line.replace(
                ' ', '\t', 1).replace('\t', '@', 1).split('@')
            if len(array) <= 1:
                continue
            netloc = array[1].replace(
                ' ', '').replace('\r', '').split('#')[0]
            # Update entry if exists, create else
            if netloc != "localhost":
                Logger.debug("Add filter: %s", netloc)
                self.__add_netloc(netloc)
            count += 1
            if count == 1000:
                SqlCursor.commit(self)
                # Do not flood sqlite, this allow webkit extension to run
                sleep(0.1)
                count = 0
        SqlCursor.remove(self)

    def __save_css_default_rule(self, line):
        """
            Save default (without blacklist, whitelist) rule to db
            @param line as str
        """
        rule = line[2:]
        # Update entry if exists, create else
        with SqlCursor(self) as sql:
            try:
                sql.execute("INSERT INTO adblock_css\
                            (rule, mtime) VALUES (?, ?)",
                            (rule, self.__adblock_mtime))
            except:
                sql.execute("UPDATE adblock_css SET mtime=?\
                             WHERE rule=?",
                            (self.__adblock_mtime, rule))

    def __save_css_domain_rule(self, line):
        """
            Save domain rule to db
            @param line as str
        """
        whitelist = []
        blacklist = []
        (domains, rule) = line.split("##")
        for domain in domains.split(","):
            if domain.startswith("~"):
                blacklist.append(domain[1:])
            else:
                whitelist.append(domain)
        str_whitelist = ",".join(whitelist)
        str_blacklist = ",".join(blacklist)
        with SqlCursor(self) as sql:
            result = sql.execute("SELECT mtime FROM adblock_css\
                                  WHERE blacklist=? AND whitelist=?\
                                  AND rule=?",
                                 (str_blacklist, str_whitelist, rule))
            v = result.fetchone()
            if v is None:
                sql.execute("INSERT INTO adblock_css\
                             (rule, whitelist, blacklist, mtime)\
                             VALUES (?, ?, ?, ?)",
                            (rule, str_whitelist, str_blacklist,
                             self.__adblock_mtime))
            else:
                sql.execute("UPDATE adblock_css SET mtime=?\
                             WHERE rule=? and blacklist=? and whitelist=?",
                            (self.__adblock_mtime, rule,
                             str_blacklist, str_whitelist))

    def __save_css_exception(self, line):
        """
            Add a new exception
            @param line as str
        """
        (domain, rule) = line.split("#@#")
        with SqlCursor(self) as sql:
            result = sql.execute("SELECT rowid, blacklist FROM adblock_css\
                                  WHERE rule=?", (rule,))
            v = result.fetchone()
            if v is None:
                sql.execute("INSERT INTO adblock_css\
                             (rule, whitelist, blacklist, mtime)\
                             VALUES (?, ?, ?, ?)",
                            (rule, "", domain, self.__adblock_mtime))
            else:
                (rowid, blacklist) = v
                blacklist += ",%s" % domain
                sql.execute("UPDATE adblock_css SET blacklist=?\
                             WHERE rowid=?", (blacklist, rowid))

    def __save_abp_rule(self, rule, exception):
        """
            Save abp rule
            @param rule as str
            @param exception as bool
        """
        # Simple host rule
        if rule[:2] == "||":
            if rule[-1] == "^" and not exception:
                self.__add_netloc(rule[2:-1])
            else:
                regex = self.__rule_to_regex(rule[2:])
                split = re.split("/|\^", rule[2:])
                uri = "http://%s" % split[0]
                parsed = urlparse(uri)
                if parsed.netloc:
                    if exception:
                        self.__add_regex_domain_ex(regex, parsed.netloc)
                    else:
                        self.__add_regex_domain(regex, parsed.netloc)
                elif not exception:
                    self.__add_regex(regex)
        elif not exception:
            regex = self.__rule_to_regex(rule)
            if regex is not None:
                self.__add_regex(regex)

    def __save_abp_rules(self, rules):
        """
            Save rules to db
            @param rules as bytes
        """
        SqlCursor.add(self)
        result = rules.decode("utf-8")
        count = 0
        for line in result.split('\n'):
            SqlCursor.allow_thread_execution(self)
            if self.__cancellable.is_cancelled():
                SqlCursor.remove(self)
                raise Exception("Cancelled")
            if "-abp-" in line or "$" in line or "!" in line or "[" in line:
                continue
            elif line.startswith("##"):
                self.__save_css_default_rule(line)
            elif "#@#" in line:
                self.__save_css_exception(line)
            elif "##" in line:
                self.__save_css_domain_rule(line)
            elif line.startswith("@@"):
                self.__save_abp_rule(line[2:], True)
            else:
                self.__save_abp_rule(line, False)
            Logger.debug("Add abp filter: %s", line)
            count += 1
            if count == 1000:
                SqlCursor.commit(self)
                # Do not flood sqlite, this allow webkit extension to run
                sleep(0.1)
                count = 0
        SqlCursor.remove(self)

    def __on_save_rules(self, result, uris=[]):
        """
            Load next uri, if finished, load CSS rules
            @param result (unused)
            @param uris as [str]
        """
        if self.__cancellable.is_cancelled():
            return
        if uris:
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri,
                                                self.__cancellable,
                                                self.__on_load_uri_content,
                                                uris)
        else:
            with SqlCursor(self) as sql:
                sql.execute("DELETE FROM adblock\
                             WHERE mtime!=?", (self.__adblock_mtime,))
                sql.execute("DELETE FROM adblock_re\
                             WHERE mtime!=?", (self.__adblock_mtime,))
                sql.execute("DELETE FROM adblock_re_domain\
                             WHERE mtime!=?", (self.__adblock_mtime,))
                sql.execute("DELETE FROM adblock_re_domain_ex\
                             WHERE mtime!=?", (self.__adblock_mtime,))
                sql.execute("DELETE FROM adblock_css\
                             WHERE mtime!=?", (self.__adblock_mtime,))
                sql.execute("DELETE FROM adblock_cache")
                try:
                    dump(self.__adblock_mtime,
                         open(EOLIE_DATA_PATH + "/adblock.bin", "wb"))
                except Exception as e:
                    Logger.error("DatabaseAdblock::__on_save_rules(): %s", e)

    def __on_load_uri_content(self, uri, status, content, uris):
        """
            Save loaded values
            @param uri as str
            @param status as bool
            @param content as bytes
            @param uris as [str]
        """
        Logger.debug("DatabaseAdblock::__on_load_uri_content(): %s", uri)
        if status:
            if uri in self.__URIS:
                self.__task_helper.run(self.__save_rules, content,
                                       callback=(self.__on_save_rules, uris))
            else:
                self.__task_helper.run(self.__save_abp_rules, content,
                                       callback=(self.__on_save_rules, uris))
        else:
            self.__on_save_rules(None, uris)
예제 #25
0
class WebViewArtwork:
    """
        Handle webview artwork: snapshot and favicon
    """
    def __init__(self):
        """
            Init class
        """
        self.__helper = TaskHelper()
        self.__snapshot_id = None
        self.__save_favicon_timeout_id = None
        self.__cancellable = Gio.Cancellable()
        self.__initial_uri = None
        self.__favicon_width = {}
        self.__current_netloc = None
        self.connect("notify::uri", self.__on_uri_changed)

    def set_favicon(self):
        """
            Set favicon based on current webview favicon
        """
        if self.ephemeral or\
                self.error or\
                self.current_event != WebKit2.LoadEvent.FINISHED:
            return

        self.__set_favicon_from_surface(self.get_favicon(), self.uri,
                                        self.__initial_uri)

    def set_current_favicon(self):
        """
            Set favicon based on current webview favicon
            Use this for JS update (do not update initial uri)
        """
        self.__set_favicon_from_surface(self.get_favicon(), self.uri, None)

#######################
# PROTECTED           #
#######################

    def _on_load_changed(self, webview, event):
        """
            Update sidebar/urlbar
            @param webview as WebView
            @param event as WebKit2.LoadEvent
        """
        parsed = urlparse(self.uri)
        if event == WebKit2.LoadEvent.STARTED:
            self.__cancellable.cancel()
            self.__cancellable.reset()
            if parsed.scheme in ["http", "https"]:
                self.__initial_uri = self.uri.rstrip('/')
            else:
                self.__initial_uri = None
        elif event == WebKit2.LoadEvent.FINISHED:
            is_http = parsed.scheme in ["http", "https"]
            if self.__snapshot_id is not None:
                GLib.source_remove(self.__snapshot_id)
            self.__snapshot_id = GLib.timeout_add(2500, self.__set_snapshot,
                                                  is_http)
            self.set_favicon()
            self.__current_netloc = parsed.netloc or None

#######################
# PRIVATE             #
#######################

    def __set_snapshot(self, save):
        """
            Set webpage preview
            @param save as bool
        """
        self.__snapshot_id = None
        self.get_snapshot(WebKit2.SnapshotRegion.FULL_DOCUMENT,
                          WebKit2.SnapshotOptions.NONE, self.__cancellable,
                          get_snapshot, self.__on_snapshot, save, True)

    def __set_favicon_from_surface(self, surface, uri, initial_uri):
        """
            Set favicon for surface
            @param favicon_db as WebKit2.FaviconDatabase
            @param result as Gio.AsyncResult
            @param uri as str
            @param initial_uri as str
        """
        resized = None
        # Save webview favicon
        if surface is not None:
            favicon_width = surface.get_width()
            if uri not in self.__favicon_width.keys() or\
                    favicon_width >= self.__favicon_width[uri]:
                if self.__save_favicon_timeout_id is not None:
                    GLib.source_remove(self.__save_favicon_timeout_id)
                    self.__save_favicon_timeout_id = None
                self.__favicon_width[uri] = favicon_width
                resized = resize_favicon(surface)
                favicon_type = "favicon"
        else:
            netloc = remove_www(urlparse(uri).netloc)
            if netloc:
                resized = App().art.get_favicon(uri, self.get_scale_factor())
                if resized is None:
                    resized = get_char_surface(netloc[0])
                favicon_type = "favicon_alt"

        # We wait for a better favicon
        if resized is not None and uri == self.uri:
            self.__save_favicon_timeout_id = GLib.timeout_add(
                500, self.__save_favicon_to_cache, resized, uri, initial_uri,
                favicon_type)

    def __save_favicon_to_cache(self, surface, uri, initial_uri, favicon_type):
        """
            Save favicon to cache
            @param surface as cairo.Surface
            @param uri as str
            @param initial_uri as str
            @param favicon_type as str
        """
        self.__save_favicon_timeout_id = None
        self.emit("favicon-changed", surface)
        # Save favicon for URI
        if not App().art.exists(uri, favicon_type):
            self.__helper.run(App().art.save_artwork, uri, surface,
                              favicon_type)
        # Save favicon for initial URI
        if initial_uri is not None and\
                not App().art.exists(initial_uri, favicon_type):
            striped_uri = uri.rstrip("/")
            if initial_uri != striped_uri:
                self.__helper.run(App().art.save_artwork, initial_uri, surface,
                                  favicon_type)

    def __on_uri_changed(self, webview, param):
        """
            Handle JS updates
            @param webview as WebKit2.WebView
            @param param as GObject.ParamSpec
        """
        if not webview.is_loading() and not webview.ephemeral:
            self.__initial_uri = None
            if self.__snapshot_id is not None:
                GLib.source_remove(self.__snapshot_id)
            self.__snapshot_id = GLib.timeout_add(2500, self.__set_snapshot,
                                                  True)

    def __on_snapshot(self, surface, save, first_pass):
        """
            Cache snapshot
            @param surface as cairo.Surface
            @param uri as str
            @param save as bool
            @param first_pass as bool
        """
        # The 32767 limit on the width/height dimensions
        # of an image surface is new in cairo 1.10,
        # try with WebKit2.SnapshotRegion.VISIBLE
        if surface is None:
            if first_pass:
                self.get_snapshot(WebKit2.SnapshotRegion.VISIBLE,
                                  WebKit2.SnapshotOptions.NONE,
                                  self.__cancellable, get_snapshot,
                                  self.__on_snapshot, save, False)
            return
        self.emit("snapshot-changed", surface)
        if not save or self.error:
            return
        uri = self.uri
        # We also cache initial URI
        uris = [uri.rstrip("/")]
        if self.__initial_uri is not None and\
                self.__initial_uri not in uris:
            uris.append(self.__initial_uri)
        for uri in uris:
            exists = App().art.exists(uri, "start")
            if not exists:
                App().art.save_artwork(uri, surface, "start")
예제 #26
0
class ImagesPopover(Gtk.Popover):
    """
        Show images for page id
    """
    def __init__(self, uri, page_id, window):
        """
            Init popover
            @param uri as str
            @param page_id as int
            @param window as Window
        """
        Gtk.Popover.__init__(self)
        self.set_modal(False)
        window.register(self)
        self.__cache_uris = []
        self.__uri = uri
        self.__page_id = page_id
        self.__cancellable = Gio.Cancellable()
        self.__filter = ""
        builder = Gtk.Builder()
        builder.add_from_resource("/org/gnome/Eolie/PopoverImages.ui")
        builder.connect_signals(self)
        widget = builder.get_object("widget")
        self.__spinner = builder.get_object("spinner")
        self.__flowbox = builder.get_object("flowbox")
        self.__flowbox.set_filter_func(self.__filter_func)
        self.__entry = builder.get_object("entry")
        self.__button = builder.get_object("button")
        self.add(widget)
        if Gio.NetworkMonitor.get_default().get_network_available():
            El().helper.call("GetImages", page_id, None, self.__on_get_images)
        (width, height) = El().active_window.get_size()
        self.set_size_request(width / 2, height / 1.5)
        self.connect("closed", self.__on_closed)
        self.__task_helper = TaskHelper()

#######################
# PROTECTED           #
#######################

    def _on_search_changed(self, entry):
        """
            Filter flowbox
            @param entry as Gtk.Entry
        """
        self.__filter = entry.get_text()
        self.__flowbox.invalidate_filter()

    def _on_button_clicked(self, button):
        """
            Save visible images
            @param button as Gtk.Button
        """
        task_helper = TaskHelper()
        task_helper.run(self.__move_images)
        self.__spinner.start()

    def _on_button_toggled(self, button):
        """
            Cancel previous download
        """
        self.__cancellable.cancel()
        self.__spinner.start()
        self.__button.set_sensitive(False)
        for child in self.__flowbox.get_children():
            child.destroy()
        self.__links = button.get_active()
        if Gio.NetworkMonitor.get_default().get_network_available():
            if button.get_active():
                El().helper.call("GetImageLinks", self.__page_id, None,
                                 self.__on_get_images)
            else:
                El().helper.call("GetImages", self.__page_id, None,
                                 self.__on_get_images)

#######################
# PRIVATE             #
#######################

    def __filter_func(self, child):
        """
            Filter child
            @param child as image
        """
        if child.uri.find(self.__filter) != -1:
            return True

    def __add_image(self, uri):
        """
            Add a child to flowbox
            @param uri as str
        """
        image = Image(uri)
        image.show()
        self.__flowbox.add(image)

    def __move_images(self):
        """
            Move image to download directory
        """
        parsed = urlparse(self.__uri)
        directory_uri = El().settings.get_value('download-uri').get_string()
        if not directory_uri:
            directory = GLib.get_user_special_dir(
                GLib.UserDirectory.DIRECTORY_DOWNLOAD)
            directory_uri = GLib.filename_to_uri(directory, None)
        destination_uri = "%s/%s" % (directory_uri, parsed.netloc)
        directory = Gio.File.new_for_uri(destination_uri)
        if not directory.query_exists():
            directory.make_directory_with_parents()
        for child in self.__flowbox.get_children():
            if child.uri.find(self.__filter) != -1:
                encoded = sha256(child.uri.encode("utf-8")).hexdigest()
                child_basename = child.uri.split("/")[-1]
                filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded)
                s = Gio.File.new_for_path(filepath)
                if not s.query_exists():
                    continue
                d = Gio.File.new_for_uri("%s/%s" %
                                         (destination_uri, child_basename))
                try:
                    s.move(d, Gio.FileCopyFlags.OVERWRITE, None, None, None)
                except Exception as e:
                    print("ImagesPopover::__move_images()", e)
        GLib.idle_add(self.hide)

    def __clean_cache(self):
        """
            Clean the cache
        """
        for uri in self.__cache_uris:
            encoded = sha256(uri.encode("utf-8")).hexdigest()
            filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded)
            f = Gio.File.new_for_path(filepath)
            try:
                if f.query_exists():
                    f.delete()
            except Exception as e:
                print("ImagesPopover::__clean_cache():", e)

    def __on_write_all_async(self, stream, result, uri):
        """
            Add image
            @param stream as Gio.OutputStream
            @param result as Gio.AsyncResult
            @param uri as str
        """
        try:
            stream.write_all_finish(result)
            self.__add_image(uri)
            self.__cache_uris.append(uri)
        except Exception as e:
            print("ImagesPopover::__on_write_all_async()", e)

    def __on_load_uri_content(self, uri, status, content, uris):
        """
            Load pending uris
            @param uri as str
            @param status as bool
            @param content as bytes
            @param uris as [str]
        """
        if status:
            encoded = sha256(uri.encode("utf-8")).hexdigest()
            filepath = "%s/%s" % (EOLIE_CACHE_PATH, encoded)
            f = Gio.File.new_for_path(filepath)
            stream = f.append_to(Gio.FileCreateFlags.REPLACE_DESTINATION,
                                 self.__cancellable)
            stream.write_all_async(content, GLib.PRIORITY_DEFAULT,
                                   self.__cancellable,
                                   self.__on_write_all_async, uri)
        if uris and not self.__cancellable.is_cancelled():
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri, self.__cancellable,
                                                self.__on_load_uri_content,
                                                uris)
        else:
            self.__spinner.stop()
            self.__button.set_sensitive(True)

    def __on_get_images(self, source, result):
        """
            Get result and load pending uris
            @param source as GObject.Object
            @param result as Gio.AsyncResult
        """
        uris = []
        try:
            uris = source.call_finish(result)[0]
        except Exception as e:
            print("ImagesPopover::__on_get_images()", e)
        self.__cancellable.reset()
        self.__on_load_uri_content(None, False, b"", uris)

    def __on_closed(self, popover):
        """
            Clean cache
        """
        self.__spinner.stop()
        self.__cancellable.cancel()
        self.__task_helper.run(self.__clean_cache)
예제 #27
0
class DatabaseAdblock:
    """
        Eolie adblock db
    """
    DB_PATH = "%s/adblock.db" % EOLIE_LOCAL_PATH

    __URIS = [
        "https://adaway.org/hosts.txt",
        "http://winhelp2002.mvps.org/hosts.txt",
        "http://hosts-file.net/ad_servers.txt",
        "https://pgl.yoyo.org/adservers/serverlist.php?"
        "hostformat=hosts&showintro=0&mimetype=plaintext"
    ]

    # SQLite documentation:
    # In SQLite, a column with type INTEGER PRIMARY KEY
    # is an alias for the ROWID.
    # Here, we define an id INT PRIMARY KEY but never feed it,
    # this make VACUUM not destroy rowids...
    __create_adblock = '''CREATE TABLE adblock (
                                               id INTEGER PRIMARY KEY,
                                               dns TEXT NOT NULL,
                                               mtime INT NOT NULL
                                               )'''

    def __init__(self):
        """
            Create database tables or manage update if needed
        """
        self.__cancellable = Gio.Cancellable.new()
        self.__task_helper = TaskHelper()
        f = Gio.File.new_for_path(self.DB_PATH)
        # Lazy loading if not empty
        if not f.query_exists():
            try:
                d = Gio.File.new_for_path(EOLIE_LOCAL_PATH)
                if not d.query_exists():
                    d.make_directory_with_parents()
                # Create db schema
                with SqlCursor(self) as sql:
                    sql.execute(self.__create_adblock)
                    sql.commit()
            except Exception as e:
                print("DatabaseAdblock::__init__(): %s" % e)

    def update(self):
        """
            Update database
        """
        if not Gio.NetworkMonitor.get_default().get_network_available():
            return
        # Update adblock_js repo
        git = GLib.find_program_in_path("git")
        if git is None:
            print(_("For stronger ad blocking, install git command"))
        else:
            d = Gio.File.new_for_path(ADBLOCK_JS)
            if d.query_exists():
                argv = [git, "-C", ADBLOCK_JS, "pull"]
            else:
                argv = [
                    git, "clone",
                    "https://github.com/gnumdk/eolie-adblock.git", ADBLOCK_JS
                ]
            (pid, a1, a2,
             a3) = GLib.spawn_async(argv,
                                    flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL)
            GLib.spawn_close_pid(pid)

        # Get in db mtime
        # Only update if filters older than one week
        mtime = 0
        with SqlCursor(self) as sql:
            result = sql.execute("SELECT mtime FROM adblock LIMIT 1")
            v = result.fetchone()
            if v is not None:
                mtime = v[0]
        self.__mtime = int(time())
        if self.__mtime - mtime < 604800:
            return
        # Update adblock db
        self.__cancellable.reset()
        self.__on_load_uri_content(None, False, b"", self.__URIS)

    def stop(self):
        """
            Stop update
        """
        self.__cancellable.cancel()

    def is_blocked(self, uri):
        """
            Return True if uri is blocked
            @param uri as str
            @return bool
        """
        try:
            parse = urlparse(uri)
            with SqlCursor(self) as sql:
                result = sql.execute(
                    "SELECT mtime FROM adblock\
                                      WHERE dns=?", (parse.netloc, ))
                v = result.fetchone()
                return v is not None
        except Exception as e:
            print("DatabaseAdblock::is_blocked():", e)
            return False

    def get_cursor(self):
        """
            Return a new sqlite cursor
        """
        try:
            c = sqlite3.connect(self.DB_PATH, 600.0)
            return c
        except Exception as e:
            print(e)
            exit(-1)

#######################
# PRIVATE             #
#######################

    def __save_rules(self, rules, uris):
        """
            Save rules to db
            @param rules as bytes
            @param uris as [str]
        """
        SqlCursor.add(self)
        result = rules.decode('utf-8')
        count = 0
        for line in result.split('\n'):
            if self.__cancellable.is_cancelled():
                raise IOError("Cancelled")
            if line.startswith('#'):
                continue
            array = line.replace(' ', '\t', 1).replace('\t', '@', 1).split('@')
            if len(array) <= 1:
                continue
            dns = array[1].replace(' ', '').replace('\r', '').split('#')[0]
            # Update entry if exists, create else
            with SqlCursor(self) as sql:
                sql.execute(
                    "INSERT INTO adblock\
                                  (dns, mtime)\
                                  VALUES (?, ?)", (dns, self.__mtime))
                count += 1
                if count == 1000:
                    sql.commit()
                    count = 0
        # We are the last call to save_rules()?
        # Delete removed entries and commit
        if not uris:
            with SqlCursor(self) as sql:
                sql.execute(
                    "DELETE FROM adblock\
                             WHERE mtime!=?", (self.__mtime, ))
                sql.commit()
        SqlCursor.remove(self)

    def __on_load_uri_content(self, uri, status, content, uris):
        """
            Load pending uris
            @param uri as str
            @param status as bool
            @param content as bytes
            @param uris as [str]
        """
        if status:
            self.__task_helper.run(self.__save_rules, (content, uris))
        if uris:
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri, self.__cancellable,
                                                self.__on_load_uri_content,
                                                uris)
예제 #28
0
class DatabasePhishing:
    """
        Phishing database
    """
    __DB_PATH = "%s/phishing.db" % EOLIE_DATA_PATH
    __URI = "http://data.phishtank.com/data/online-valid.json"
    __SCHEMA_VERSION = 0
    __UPDATE = 172800
    # SQLite documentation:
    # In SQLite, a column with type INTEGER PRIMARY KEY
    # is an alias for the ROWID.
    # Here, we define an id INT PRIMARY KEY but never feed it,
    # this make VACUUM not destroy rowids...
    __create_phishing = '''CREATE TABLE phishing (
                                               id INTEGER PRIMARY KEY,
                                               uri TEXT NOT NULL,
                                               mtime INT NOT NULL
                                               )'''
    __create_phishing_idx = """CREATE UNIQUE INDEX idx_phishing ON phishing(
                                               uri)"""

    def __init__(self):
        """
            Create database tables or manage update if needed
        """
        self.thread_lock = Lock()
        self.__cancellable = Gio.Cancellable.new()
        self.__task_helper = TaskHelper()
        self.__phishing_mtime = int(time())
        self.__regex = None

    def create_db(self):
        """
            Create databse
        """
        if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR):
            GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750)
        # If DB schema changed, remove it
        if GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR):
            with SqlCursor(self) as sql:
                result = sql.execute("PRAGMA user_version")
                v = result.fetchone()
                if v is None or v[0] != self.__SCHEMA_VERSION:
                    f = Gio.File.new_for_path(self.__DB_PATH)
                    f.delete()
        if not GLib.file_test(self.__DB_PATH, GLib.FileTest.IS_REGULAR):
            try:
                # Create db schema
                with SqlCursor(self) as sql:
                    sql.execute(self.__create_phishing)
                    sql.execute(self.__create_phishing_idx)
                    sql.execute("PRAGMA user_version=%s" %
                                self.__SCHEMA_VERSION)
            except Exception as e:
                Logger.error("DatabasePhishing::__init__(): %s", e)

    def update(self):
        """
            Update database
        """
        if not Gio.NetworkMonitor.get_default().get_network_available():
            return
        # DB version is last successful sync mtime
        try:
            version = load(open(EOLIE_DATA_PATH + "/phishing.bin", "rb"))
        except:
            version = 0
        self.__cancellable.reset()
        if self.__phishing_mtime - version > self.__UPDATE:
            self.__on_load_uri_content(None, False, b"", [self.__URI])

    def is_phishing(self, uri):
        """
            True if uri is phishing
            @param uri as str
            @return bool
        """
        uri = uri.rstrip("/")
        try:
            with SqlCursor(self) as sql:
                result = sql.execute(
                    "SELECT uri FROM phishing\
                                      WHERE uri=?", (uri, ))
                v = result.fetchone()
                return v is not None
        except Exception as e:
            Logger.error("DatabasePhishing::is_phishing(): %s", e)
            return False

    def stop(self):
        """
            Stop update
        """
        self.__cancellable.cancel()
        self.__stop = True

    def get_cursor(self):
        """
            Return a new sqlite cursor
        """
        try:
            c = sqlite3.connect(self.__DB_PATH, 600.0)
            return c
        except Exception as e:
            Logger.error("DatabasePhishing::get_cursor(): %s", e)
            exit(-1)

#######################
# PRIVATE             #
#######################

    def __save_rules(self, rules, uris):
        """
            Save rules to db
            @param rules as bytes
            @param uris as [str]
        """
        SqlCursor.add(self)
        try:
            result = rules.decode('utf-8')
            j = json.loads(result)
            with SqlCursor(self) as sql:
                count = 0
                for item in j:
                    if self.__cancellable.is_cancelled():
                        raise IOError("Cancelled")
                    uri = item["url"].rstrip("/")
                    try:
                        sql.execute(
                            "INSERT INTO phishing\
                                     (uri, mtime) VALUES (?, ?)",
                            (uri, self.__phishing_mtime))
                    except:
                        sql.execute(
                            "UPDATE phishing set mtime=?\
                                     WHERE uri=?",
                            (self.__phishing_mtime, uri))
                    count += 1
                    if count == 1000:
                        SqlCursor.commit(self)
                        # Do not flood sqlite
                        # this allow webkit extension to run
                        sleep(0.1)
                        count = 0
            # We are the last call to save_rules()?
            # Delete removed entries and commit
            if not uris:
                with SqlCursor(self) as sql:
                    sql.execute(
                        "DELETE FROM phishing\
                                 WHERE mtime!=?", (self.__phishing_mtime, ))
                    try:
                        dump(self.__phishing_mtime,
                             open(EOLIE_DATA_PATH + "/phishing.bin", "wb"))
                    except Exception as e:
                        Logger.error("DatabasePhishing::__save_rules(): %s", e)
        except Exception as e:
            Logger.error("DatabasePhishing::__save_rules():%s -> %s", e, rules)
        SqlCursor.remove(self)

    def __on_load_uri_content(self, uri, status, content, uris):
        """
            Load pending uris
            @param uri as str
            @param status as bool
            @param content as bytes
            @param uris as [str]
        """
        if status:
            self.__task_helper.run(self.__save_rules, content, uris)
        if uris:
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri, self.__cancellable,
                                                self.__on_load_uri_content,
                                                uris)
예제 #29
0
파일: settings.py 프로젝트: bagage/eolie
    def __init__(self, window):
        """
            Init dialog
            @param window as Window
        """
        self.__window = window
        self.__helper = PasswordsHelper()
        builder = Gtk.Builder()
        builder.add_from_resource("/org/gnome/Eolie/SettingsDialog.ui")

        self.__settings_dialog = builder.get_object("settings_dialog")
        self.__settings_dialog.set_transient_for(window)
        # self.__settings_dialog.connect("destroy", self.__on_destroy)

        if False:
            self.__settings_dialog.set_title(_("Preferences"))
        else:
            headerbar = builder.get_object("header_bar")
            headerbar.set_title(_("Preferences"))
            self.__settings_dialog.set_titlebar(headerbar)

        download_chooser = builder.get_object("download_chooser")
        dir_uri = App().settings.get_value("download-uri").get_string()
        if not dir_uri:
            directory = GLib.get_user_special_dir(
                GLib.UserDirectory.DIRECTORY_DOWNLOAD)
            if directory is not None:
                dir_uri = GLib.filename_to_uri(directory, None)
        if dir_uri:
            download_chooser.set_uri(dir_uri)
        else:
            download_chooser.set_uri("file://" + GLib.getenv("HOME"))

        open_downloads = builder.get_object("open_downloads_check")
        open_downloads.set_active(App().settings.get_value("open-downloads"))

        self.__start_page_uri = builder.get_object("start_page_uri")
        combo_start = builder.get_object("combo_start")
        start_page = App().settings.get_value("start-page").get_string()
        if start_page.startswith("http"):
            combo_start.set_active_id("address")
            self.__start_page_uri.set_text(start_page)
            self.__start_page_uri.show()
        else:
            combo_start.set_active_id(start_page)

        remember_session = builder.get_object("remember_sessions_check")
        remember_session.set_active(
            App().settings.get_value("remember-session"))

        suggestions = builder.get_object("suggestions_check")
        suggestions.set_active(App().settings.get_value("enable-suggestions"))

        enable_dev_tools = builder.get_object("dev_tools_check")
        enable_dev_tools.set_active(
            App().settings.get_value("developer-extras"))

        enable_plugins = builder.get_object("plugins_check")
        enable_plugins.set_active(App().settings.get_value("enable-plugins"))

        self.__fonts_grid = builder.get_object("fonts_grid")
        use_system_fonts = builder.get_object("system_fonts_check")
        use_system_fonts.set_active(
            App().settings.get_value("use-system-fonts"))
        self.__fonts_grid.set_sensitive(
            not App().settings.get_value("use-system-fonts"))

        sans_serif_button = builder.get_object("sans_serif_button")
        sans_serif_button.set_font_name(
            App().settings.get_value("font-sans-serif").get_string())
        serif_button = builder.get_object("serif_button")
        serif_button.set_font_name(
            App().settings.get_value("font-serif").get_string())
        monospace_button = builder.get_object("monospace_button")
        monospace_button.set_font_name(
            App().settings.get_value("font-monospace").get_string())

        min_font_size_spin = builder.get_object("min_font_size_spin")
        min_font_size_spin.set_value(
            App().settings.get_value("min-font-size").get_int32())

        monitor_model = get_current_monitor_model(window)
        zoom_levels = App().settings.get_value("default-zoom-level")
        wanted_zoom_level = 1.0
        try:
            for zoom_level in zoom_levels:
                zoom_splited = zoom_level.split('@')
                if zoom_splited[0] == monitor_model:
                    wanted_zoom_level = float(zoom_splited[1])
        except:
            pass
        default_zoom_level = builder.get_object("default_zoom_level")
        percent_zoom = int(wanted_zoom_level * 100)
        default_zoom_level.set_value(percent_zoom)
        default_zoom_level.set_text("{} %".format(percent_zoom))

        cookies_combo = builder.get_object("cookies_combo")
        storage = App().settings.get_enum("cookie-storage")
        cookies_combo.set_active_id(str(storage))

        history_combo = builder.get_object("history_combo")
        storage = App().settings.get_enum("history-storage")
        history_combo.set_active_id(str(storage))

        self.__populars_count = builder.get_object("populars_count")
        if start_page in ["popular_book", "popular_hist"]:
            self.__populars_count.show()
        max_popular_items = App().settings.get_value(
            "max-popular-items").get_int32()
        builder.get_object("popular_spin_button").set_value(max_popular_items)
        remember_passwords = builder.get_object("remember_passwords_check")
        remember_passwords.set_active(
            App().settings.get_value("remember-passwords"))

        dns_prediction_check = builder.get_object("dns_prediction_check")
        dns_prediction_check.set_active(
            App().settings.get_value("dns-prediction"))
        tracking_check = builder.get_object("tracking_check")
        tracking_check.set_active(App().settings.get_value("do-not-track"))
        self.__result_label = builder.get_object("result_label")
        self.__sync_button = builder.get_object("sync_button")
        self.__login_entry = builder.get_object("login_entry")
        self.__password_entry = builder.get_object("password_entry")
        self.__result_image = builder.get_object("result_image")
        builder.connect_signals(self)
        self.__helper.get_sync(self.__on_get_sync)

        task_helper = TaskHelper()
        task_helper.run(self.__get_sync_status)
예제 #30
0
class DatabaseAdblock:
    """
        Eolie adblock db
    """
    DB_PATH = "%s/adblock2.db" % EOLIE_DATA_PATH

    __URIS = [
        "https://adaway.org/hosts.txt",
        "https://pgl.yoyo.org/adservers/serverlist.php?" +
        "hostformat=hosts&showintro=0&startdate%5Bday%5D=" +
        "&startdate%5Bmonth%5D=&startdate%5Byear%5D=",
        "http://winhelp2002.mvps.org/hosts.txt",
        "http://hosts-file.net/ad_servers.txt",
        "https://pgl.yoyo.org/adservers/serverlist.php?"
        "hostformat=hosts&showintro=0&mimetype=plaintext"
    ]

    __CSS_URIS = ["https://easylist-downloads.adblockplus.org/easylist.txt"]

    __CSS_LOCALIZED_URIS = {
        "bg":
        "http://stanev.org/abp/adblock_bg.txt",
        "zh":
        "https://easylist-downloads.adblockplus.org/easylistchina.txt",
        "sk":
        "https://raw.github.com/tomasko126/" +
        "easylistczechandslovak/master/filters.txt",
        "cs":
        "https://raw.github.com/tomasko126/" +
        "easylistczechandslovak/master/filters.txt",
        "nl":
        "https://easylist-downloads.adblockplus.org/easylistdutch.txt",
        "de":
        "https://easylist-downloads.adblockplus.org/easylistgermany.txt",
        "he":
        "https://raw.githubusercontent.com/easylist/" +
        "EasyListHebrew/master/EasyListHebrew.txt",
        "it":
        "https://easylist-downloads.adblockplus.org/easylistitaly.txt",
        "lt":
        "http://margevicius.lt/easylistlithuania.txt",
        "es":
        "https://easylist-downloads.adblockplus.org/easylistspanish.txt",
        "lv":
        "https://notabug.org/latvian-list/" +
        "adblock-latvian/raw/master/lists/latvian-list.txt",
        "ar":
        "https://easylist-downloads.adblockplus.org/Liste_AR.txt",
        "fr":
        "https://easylist-downloads.adblockplus.org/liste_fr.txt",
        "ro":
        "http://www.zoso.ro/pages/rolist.txt",
        "ru":
        "https://easylist-downloads.adblockplus.org/advblock.txt",
        "ja":
        "http://bit.ly/11QrCfx",
        "fi":
        "https://adb.juvander.net/Finland_adb.txt",
        "cz":
        "http://adblock.dajbych.net/adblock.txt",
        "et":
        "http://gurud.ee/ab.txt",
        "hu":
        "https://raw.githubusercontent.com/szpeter80/" +
        "hufilter/master/hufilter.txt"
    }

    __UPDATE = 172800

    # SQLite documentation:
    # In SQLite, a column with type INTEGER PRIMARY KEY
    # is an alias for the ROWID.
    # Here, we define an id INT PRIMARY KEY but never feed it,
    # this make VACUUM not destroy rowids...
    __create_adblock = '''CREATE TABLE adblock (
                                               id INTEGER PRIMARY KEY,
                                               dns TEXT NOT NULL,
                                               mtime INT NOT NULL
                                               )'''
    __create_adblock_css = '''CREATE TABLE adblock_css (
                                               id INTEGER PRIMARY KEY,
                                               name TEXT NOT NULL,
                                               whitelist TEXT DEFAULT "",
                                               blacklist TEXT DEFAULT "",
                                               mtime INT NOT NULL
                                               )'''

    def __init__(self):
        """
            Create database tables or manage update if needed
        """
        self.__cancellable = Gio.Cancellable.new()
        self.__task_helper = TaskHelper()
        self.__adblock_mtime = int(time())

        # Lazy loading if not empty
        if not GLib.file_test(self.DB_PATH, GLib.FileTest.IS_REGULAR):
            try:
                if not GLib.file_test(EOLIE_DATA_PATH, GLib.FileTest.IS_DIR):
                    GLib.mkdir_with_parents(EOLIE_DATA_PATH, 0o0750)
                # Create db schema
                with SqlCursor(self) as sql:
                    sql.execute(self.__create_adblock)
                    sql.execute(self.__create_adblock_css)
                    sql.commit()
            except Exception as e:
                print("DatabaseAdblock::__init__(): %s" % e)

    def update(self):
        """
            Update database
        """
        if not Gio.NetworkMonitor.get_default().get_network_available():
            return
        # Update adblock_js repo
        git = GLib.find_program_in_path("git")
        if git is None:
            print(_("For stronger ad blocking, install git command"))
        else:
            if GLib.file_test(ADBLOCK_JS, GLib.FileTest.IS_DIR):
                argv = [git, "-C", ADBLOCK_JS, "pull"]
            else:
                argv = [
                    git, "clone",
                    "https://gitlab.gnome.org/gnumdk/eolie-adblock.git",
                    ADBLOCK_JS
                ]
            (pid, a1, a2,
             a3) = GLib.spawn_async(argv,
                                    flags=GLib.SpawnFlags.STDOUT_TO_DEV_NULL)
            GLib.spawn_close_pid(pid)

        # Check entries in DB, do we need to update?
        mtime = 0
        with SqlCursor(self) as sql:
            result = sql.execute("SELECT mtime FROM adblock\
                                  ORDER BY mtime LIMIT 1")
            v = result.fetchone()
            if v is not None:
                mtime = v[0]
        self.__cancellable.reset()
        if self.__adblock_mtime - mtime > self.__UPDATE:
            # Update host rules
            uris = list(self.__URIS)
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri, self.__cancellable,
                                                self.__on_load_uri_content,
                                                uris)
        else:
            self.__on_save_rules()

    def stop(self):
        """
            Stop update
        """
        self.__cancellable.cancel()

    def get_default_css_rules(self):
        """
            Return default css rules
        """
        rules = ""
        with SqlCursor(self) as sql:
            request = "SELECT name FROM adblock_css WHERE\
                       blacklist='' AND whitelist=''"

            result = sql.execute(request)
            for name in list(itertools.chain(*result)):
                rules += "%s,\n" % name
        return rules[:-2] + "{display: none !important;}"

    def get_css_rules(self, uri):
        """
            Return css rules for uri
            @return str
        """
        rules = ""
        parsed = urlparse(uri)
        if parsed.scheme not in ["http", "https"]:
            return ""
        netloc = remove_www(parsed.netloc)
        with SqlCursor(self) as sql:
            request = "SELECT name FROM adblock_css WHERE\
                       (blacklist!='' AND blacklist!=?) OR whitelist=?"

            result = sql.execute(request, (netloc, netloc))
            for name in list(itertools.chain(*result)):
                rules += "%s,\n" % name
        return rules[:-2] + "{display: none !important;}"

    def is_blocked(self, uri):
        """
            Return True if uri is blocked
            @param uri as str
            @return bool
        """
        try:
            parsed = urlparse(uri)
            if parsed.scheme not in ["http", "https"] or\
                    El().adblock_exceptions.find_parsed(parsed):
                return False
            with SqlCursor(self) as sql:
                result = sql.execute(
                    "SELECT mtime FROM adblock\
                                      WHERE dns=?", (parsed.netloc, ))
                v = result.fetchone()
                return v is not None
        except Exception as e:
            print("DatabaseAdblock::is_blocked():", e)
            return False

    def get_cursor(self):
        """
            Return a new sqlite cursor
        """
        try:
            c = sqlite3.connect(self.DB_PATH, 600.0)
            return c
        except Exception as e:
            print(e)
            exit(-1)

#######################
# PRIVATE             #
#######################

    def __save_rules(self, rules, uris):
        """
            Save rules to db
            @param rules bytes
            @param uris as [str]
        """
        SqlCursor.add(self)
        result = rules.decode('utf-8')
        count = 0
        for line in result.split('\n'):
            if self.__cancellable.is_cancelled():
                raise IOError("Cancelled")
            if line.startswith('#'):
                continue
            array = line.replace(' ', '\t', 1).replace('\t', '@', 1).split('@')
            if len(array) <= 1:
                continue
            dns = array[1].replace(' ', '').replace('\r', '').split('#')[0]
            # Update entry if exists, create else
            with SqlCursor(self) as sql:
                debug("Add filter: %s" % dns)
                sql.execute(
                    "INSERT INTO adblock\
                            (dns, mtime) VALUES (?, ?)",
                    (dns, self.__adblock_mtime))
                count += 1
                if count == 1000:
                    sql.commit()
                    count = 0
        # We are the last call to save_rules()?
        # Delete removed entries and commit
        if not uris:
            with SqlCursor(self) as sql:
                sql.execute(
                    "DELETE FROM adblock\
                             WHERE mtime!=?", (self.__adblock_mtime, ))
                sql.commit()
        SqlCursor.remove(self)

    def __save_css_default_rule(self, line):
        """
            Save default (without blacklist, whitelist) rule to db
            @param line as str
        """
        name = line[2:]
        # Update entry if exists, create else
        with SqlCursor(self) as sql:
            debug("Add filter: %s" % name)
            sql.execute(
                "INSERT INTO adblock_css\
                        (name, mtime) VALUES (?, ?)",
                (name, self.__adblock_mtime))

    def __save_css_domain_rule(self, line):
        """
            Save domain rule to db
            @param line as str
        """
        whitelist = ""
        blacklist = ""
        (domains, name) = line.split("##")
        for domain in domains.split(","):
            if domain.startswith("~"):
                blacklist += "@%s@" % domain[1:]
            else:
                whitelist += domain
        with SqlCursor(self) as sql:
            debug("Add filter: %s" % name)
            sql.execute(
                "INSERT INTO adblock_css\
                         (name, whitelist, blacklist, mtime)\
                         VALUES (?, ?, ?, ?)",
                (name, whitelist, blacklist, self.__adblock_mtime))

    def __save_css_rules(self, rules, uris):
        """
            Save rules to db
            @param rules as bytes
            @param uris as [str]
        """
        SqlCursor.add(self)
        result = rules.decode("utf-8")
        count = 0
        for line in result.split('\n'):
            if self.__cancellable.is_cancelled():
                raise IOError("Cancelled")
            if line.find("-abp-") != -1:
                continue
            elif line.startswith("##"):
                self.__save_css_default_rule(line)
            elif line.find("##") != -1:
                self.__save_css_domain_rule(line)
            count += 1
            if count == 1000:
                with SqlCursor(self) as sql:
                    sql.commit()
                count = 0
        # We are the last rule
        # Delete old entries
        if not uris:
            with SqlCursor(self) as sql:
                sql.execute(
                    "DELETE FROM adblock_css\
                             WHERE mtime!=?", (self.__adblock_mtime, ))
                sql.commit()
        SqlCursor.remove(self)

    def __on_save_css_rules(self, result, uris):
        """
            Load next uri
            @param result as ??
            @param uris as [str]
        """
        if uris:
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri, self.__cancellable,
                                                self.__on_load_uri_css_content,
                                                uris)

    def __on_load_uri_css_content(self, uri, status, content, uris):
        """
            Load pending uris
            @param uri as str
            @param status as bool
            @param content as bytes
            @param uris as [str]
        """
        if status:
            self.__task_helper.run(self.__save_css_rules,
                                   content,
                                   uris,
                                   callback=(self.__on_save_css_rules, uris))

    def __on_save_rules(self, result=None, uris=[]):
        """
            Load next uri, if finished, load CSS rules
            @param result as None
            @param uris as [str]
        """
        if uris:
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri, self.__cancellable,
                                                self.__on_load_uri_content,
                                                uris)
        else:
            # Check entries in DB, do we need to update?
            mtime = 0
            with SqlCursor(self) as sql:
                result = sql.execute("SELECT mtime FROM adblock_css\
                                      ORDER BY mtime LIMIT 1")
                v = result.fetchone()
                if v is not None:
                    mtime = v[0]
            # We ignore update value from rules file
            if self.__adblock_mtime - mtime < self.__UPDATE:
                return
            locales = GLib.get_language_names()
            user_locale = locales[0].split("_")[0]
            try:
                uris = [self.__CSS_LOCALIZED_URIS[user_locale]]
            except:
                uris = []
            uris += list(self.__CSS_URIS)
            uri = uris.pop(0)
            self.__task_helper.load_uri_content(uri, self.__cancellable,
                                                self.__on_load_uri_css_content,
                                                uris)

    def __on_load_uri_content(self, uri, status, content, uris):
        """
            Save loaded values
            @param uri as str
            @param status as bool
            @param content as bytes
            @param uris as [str]
        """
        if status:
            self.__task_helper.run(self.__save_rules,
                                   content,
                                   uris,
                                   callback=(self.__on_save_rules, uris))