def __init__(self, *, delete_func=None, parent=None):
"""Create a new History completion category."""
super().__init__(parent=parent)
self.name = "History"
# replace ' in timestamp-format to avoid breaking the query
timestamp_format = config.val.completion.timestamp_format
timefmt = (
"strftime('{}', last_atime, 'unixepoch', 'localtime')".format(
timestamp_format.replace("'", "`")))
self._query = sql.Query(
' '.join([
"SELECT url, title, {}".format(timefmt),
"FROM CompletionHistory",
# the incoming pattern will have literal % and _ escaped with '\'
# we need to tell sql to treat '\' as an escape character
"WHERE (url LIKE :pat escape '\\' or title LIKE :pat escape '\\')",
self._atime_expr(),
"ORDER BY last_atime DESC",
]),
forward_only=False)
# advertise that this model filters by URL and title
self.columns_to_filter = [0, 1]
self.delete_func = delete_func
def test_prepare_error(self):
with pytest.raises(sql.BugError) as excinfo:
sql.Query('invalid')
expected = ('Failed to prepare query "invalid": "near "invalid": '
'syntax error Unable to execute statement"')
assert str(excinfo.value) == expected
def _rebuild_completion(self):
data = {'url': [], 'title': [], 'last_atime': []}
# select the latest entry for each url
q = sql.Query('SELECT url, title, max(atime) AS atime FROM History '
'WHERE NOT redirect and url NOT LIKE "qute://back%" '
'GROUP BY url ORDER BY atime asc')
for entry in q.run():
url = QUrl(entry.url)
if self._is_excluded(url):
continue
data['url'].append(self._format_completion_url(url))
data['title'].append(entry.title)
data['last_atime'].append(entry.atime)
self.completion.insert_batch(data, replace=True)
sql.Query('pragma user_version = {}'.format(_USER_VERSION)).run()
def _rebuild_completion(self):
data: Mapping[str, MutableSequence[str]] = {
'url': [],
'title': [],
'last_atime': []
}
self._progress.start(
"<b>Rebuilding completion...</b><br>"
"This is a one-time operation and happens because the database version "
"or <i>completion.web_history.exclude</i> was changed."
)
# Delete old entries
self.completion.delete_all()
QApplication.processEvents()
# Select the latest entry for each url
q = sql.Query('SELECT url, title, max(atime) AS atime FROM History '
'WHERE NOT redirect '
'GROUP BY url ORDER BY atime asc')
result = q.run()
QApplication.processEvents()
entries = list(result)
self._progress.set_maximum(len(entries))
for entry in entries:
self._progress.tick()
url = QUrl(entry.url)
if self._is_excluded_from_completion(url):
continue
data['url'].append(self._format_completion_url(url))
data['title'].append(entry.title)
data['last_atime'].append(entry.atime)
self._progress.set_maximum(0)
# We might have caused fragmentation - let's clean up.
sql.Query('VACUUM').run()
QApplication.processEvents()
self.completion.insert_batch(data, replace=True)
QApplication.processEvents()
self._progress.finish()
def __init__(self, progress, parent=None):
super().__init__("History", ['url', 'title', 'atime', 'redirect'],
constraints={
'url': 'NOT NULL',
'title': 'NOT NULL',
'atime': 'NOT NULL',
'redirect': 'NOT NULL'
},
parent=parent)
self._progress = progress
# Store the last saved url to avoid duplicate immediate saves.
self._last_url = None
self.completion = CompletionHistory(parent=self)
self.metainfo = CompletionMetaInfo(parent=self)
if sql.Query('pragma user_version').run().value() < _USER_VERSION:
self.completion.delete_all()
# Get a string of all patterns
patterns = config.instance.get_str('completion.web_history.exclude')
# If patterns changed, update them in database and rebuild completion
if self.metainfo['excluded_patterns'] != patterns:
self.metainfo['excluded_patterns'] = patterns
self.completion.delete_all()
if not self.completion:
# either the table is out-of-date or the user wiped it manually
self._rebuild_completion()
self.create_index('HistoryIndex', 'url')
self.create_index('HistoryAtimeIndex', 'atime')
self._contains_query = self.contains_query('url')
self._between_query = sql.Query('SELECT * FROM History '
'where not redirect '
'and not url like "qute://%" '
'and atime > :earliest '
'and atime <= :latest '
'ORDER BY atime desc')
self._before_query = sql.Query('SELECT * FROM History '
'where not redirect '
'and not url like "qute://%" '
'and atime <= :latest '
'ORDER BY atime desc '
'limit :limit offset :offset')
def test_recovery_no_table(self, metainfo):
sql.Query("DROP TABLE CompletionMetaInfo").run()
with pytest.raises(sql.BugError,
match='no such table: CompletionMetaInfo'):
metainfo['force_rebuild']
metainfo.try_recover()
assert not metainfo['force_rebuild']
def __init__(self, progress, parent=None):
super().__init__("History", ['url', 'title', 'atime', 'redirect'],
constraints={
'url': 'NOT NULL',
'title': 'NOT NULL',
'atime': 'NOT NULL',
'redirect': 'NOT NULL'
},
parent=parent)
self._progress = progress
# Store the last saved url to avoid duplicate immedate saves.
self._last_url = None
self.completion = CompletionHistory(parent=self)
self.metainfo = CompletionMetaInfo(parent=self)
if sql.Query('pragma user_version').run().value() < _USER_VERSION:
self.completion.delete_all()
if self.metainfo['force_rebuild']:
self.completion.delete_all()
self.metainfo['force_rebuild'] = False
if not self.completion:
# either the table is out-of-date or the user wiped it manually
self._rebuild_completion()
self.create_index('HistoryIndex', 'url')
self.create_index('HistoryAtimeIndex', 'atime')
self._contains_query = self.contains_query('url')
self._between_query = sql.Query('SELECT * FROM History '
'where not redirect '
'and not url like "qute://%" '
'and atime > :earliest '
'and atime <= :latest '
'ORDER BY atime desc')
self._before_query = sql.Query('SELECT * FROM History '
'where not redirect '
'and not url like "qute://%" '
'and atime <= :latest '
'ORDER BY atime desc '
'limit :limit offset :offset')
config.instance.changed.connect(self._on_config_changed)
def __init__(self, parent=None):
super().__init__("History", ['url', 'title', 'atime', 'redirect'],
parent=parent)
self.completion = CompletionHistory(parent=self)
self.create_index('HistoryIndex', 'url')
self.create_index('HistoryAtimeIndex', 'atime')
self._contains_query = self.contains_query('url')
self._between_query = sql.Query('SELECT * FROM History '
'where not redirect '
'and not url like "qute://%" '
'and atime > :earliest '
'and atime <= :latest '
'ORDER BY atime desc')
self._before_query = sql.Query('SELECT * FROM History '
'where not redirect '
'and not url like "qute://%" '
'and atime <= :latest '
'ORDER BY atime desc '
'limit :limit offset :offset')
def set_pattern(self, pattern):
"""Set the pattern used to filter results.
Args:
pattern: string pattern to filter by.
"""
# escape to treat a user input % or _ as a literal, not a wildcard
pattern = pattern.replace('%', '\\%')
pattern = pattern.replace('_', '\\_')
words = ['%{}%'.format(w) for w in pattern.split(' ')]
# build a where clause to match all of the words in any order
# given the search term "a b", the WHERE clause would be:
# (url LIKE '%a%' OR title LIKE '%a%') AND
# (url LIKE '%b%' OR title LIKE '%b%')
where_clause = ' AND '.join(
"(url LIKE :{val} escape '\\' OR title LIKE :{val} escape '\\')"
.format(val=i) for i in range(len(words)))
# replace ' in timestamp-format to avoid breaking the query
timestamp_format = config.val.completion.timestamp_format or ''
timefmt = ("strftime('{}', last_atime, 'unixepoch', 'localtime')"
.format(timestamp_format.replace("'", "`")))
try:
if (not self._query or
len(words) != len(self._query.bound_values())):
# if the number of words changed, we need to generate a new
# query otherwise, we can reuse the prepared query for
# performance
self._query = sql.Query(' '.join([
"SELECT url, title, {}".format(timefmt),
"FROM CompletionHistory",
# the incoming pattern will have literal % and _ escaped we
# need to tell SQL to treat '\' as an escape character
'WHERE ({})'.format(where_clause),
self._atime_expr(),
"ORDER BY last_atime DESC",
]), forward_only=False)
with debug.log_time('sql', 'Running completion query'):
self._query.run(**{
str(i): w for i, w in enumerate(words)})
except sql.KnownError as e:
# Sometimes, the query we built up was invalid, for example,
# due to a large amount of words.
# Also catches failures in the DB we can't solve.
message.error("Error with SQL query: {}".format(e.text()))
return
self.setQuery(self._query.query)
def _cleanup_history(self):
"""Do a one-time cleanup of the entire history.
This is run only once after the v2.0.0 upgrade, based on the database's
user_version.
"""
terms = [
'data:%',
'view-source:%',
'qute://back%',
'qute://pdfjs%',
]
where_clause = ' OR '.join(f"url LIKE '{term}'" for term in terms)
q = sql.Query(f'DELETE FROM History WHERE {where_clause}')
entries = q.run()
log.sql.debug(f"Cleanup removed {entries.rows_affected()} items")
def set_pattern(self, pattern):
"""Set the pattern used to filter results.
Args:
pattern: string pattern to filter by.
"""
# escape to treat a user input % or _ as a literal, not a wildcard
pattern = pattern.replace('%', '\\%')
pattern = pattern.replace('_', '\\_')
words = ['%{}%'.format(w) for w in pattern.split(' ')]
# build a where clause to match all of the words in any order
# given the search term "a b", the WHERE clause would be:
# ((url || ' ' || title) LIKE '%a%') AND
# ((url || ' ' || title) LIKE '%b%')
where_clause = ' AND '.join(
"(url || ' ' || title) LIKE :{} escape '\\'".format(i)
for i in range(len(words)))
# replace ' in timestamp-format to avoid breaking the query
timestamp_format = config.val.completion.timestamp_format or ''
timefmt = (
"strftime('{}', last_atime, 'unixepoch', 'localtime')".format(
timestamp_format.replace("'", "`")))
if not self._query or len(words) != len(self._query.bound_values()):
# if the number of words changed, we need to generate a new query
# otherwise, we can reuse the prepared query for performance
self._query = sql.Query(
' '.join([
"SELECT url, title, {}".format(timefmt),
"FROM CompletionHistory",
# the incoming pattern will have literal % and _ escaped
# we need to tell sql to treat '\' as an escape character
'WHERE ({})'.format(where_clause),
self._atime_expr(),
"ORDER BY last_atime DESC",
]),
forward_only=False)
with debug.log_time('sql', 'Running completion query'):
self._query.run(**{str(i): w for i, w in enumerate(words)})
self.setQuery(self._query.query)
def _atime_expr(self):
"""If max_items is set, return an expression to limit the query."""
max_items = config.val.completion.web_history_max_items
# HistoryCategory should not be added to the completion in that case.
assert max_items != 0
if max_items < 0:
return ''
min_atime = sql.Query(' '.join([
'SELECT min(last_atime) FROM',
'(SELECT last_atime FROM CompletionHistory',
'ORDER BY last_atime DESC LIMIT :limit)',
])).run(limit=max_items).value()
if not min_atime:
# if there are no history items, min_atime may be '' (issue #2849)
return ''
return "AND last_atime >= {}".format(min_atime)
def test_forward_only(self, forward_only):
q = sql.Query('SELECT 0 WHERE 0', forward_only=forward_only)
assert q.query.isForwardOnly() == forward_only
def test_num_rows_affected(self, condition):
table = sql.SqlTable('Foo', ['name'])
table.insert({'name': 'helloworld'})
q = sql.Query(f'DELETE FROM Foo WHERE {condition}')
q.run()
assert q.rows_affected() == condition
def test_bound_values(self):
q = sql.Query('SELECT :answer')
q.run(answer=42)
assert q.bound_values() == {':answer': 42}
def test_num_rows_affected_not_active(self):
with pytest.raises(AssertionError):
q = sql.Query('SELECT 0')
q.rows_affected()
def test_num_rows_affected_select(self):
with pytest.raises(AssertionError):
q = sql.Query('SELECT 0')
q.run()
q.rows_affected()
def test_run_batch_missing_binding(self):
q = sql.Query('SELECT :answer')
with pytest.raises(sql.BugError, match='Missing bound values!'):
q.run_batch(values={})
def test_value_missing(self):
q = sql.Query('SELECT 0 WHERE 0')
q.run()
with pytest.raises(sql.BugError,
match='No result for single-result query'):
q.value()
def test_run_binding(self):
q = sql.Query('SELECT :answer')
q.run(answer=42)
assert q.value() == 42
def test_run_batch(self):
q = sql.Query('SELECT :answer')
q.run_batch(values={'answer': [42]})
assert q.value() == 42
def test_iter(self):
q = sql.Query('SELECT 0 AS col')
q.run()
result = next(iter(q))
assert result.col == 0
def test_iter_multiple(self):
q = sql.Query('VALUES (1), (2), (3);')
res = list(q.run())
assert len(res) == 3
assert res[0].column1 == 1
def test_iter_empty(self):
q = sql.Query('SELECT 0 AS col WHERE 0')
q.run()
with pytest.raises(StopIteration):
next(iter(q))
def test_iter_inactive(self):
q = sql.Query('SELECT 0')
with pytest.raises(sql.BugError,
match='Cannot iterate inactive query'):
next(iter(q))
def __init__(self, progress, parent=None):
super().__init__("History", ['url', 'title', 'atime', 'redirect'],
constraints={
'url': 'NOT NULL',
'title': 'NOT NULL',
'atime': 'NOT NULL',
'redirect': 'NOT NULL'
},
parent=parent)
self._progress = progress
# Store the last saved url to avoid duplicate immediate saves.
self._last_url = None
self.completion = CompletionHistory(parent=self)
self.metainfo = CompletionMetaInfo(parent=self)
try:
rebuild_completion = self.metainfo['force_rebuild']
except sql.BugError: # pragma: no cover
log.sql.warning("Failed to access meta info, trying to recover...",
exc_info=True)
self.metainfo.try_recover()
rebuild_completion = self.metainfo['force_rebuild']
if sql.user_version_changed():
# If the DB user version changed, run a full cleanup and rebuild the
# completion history.
#
# In the future, this could be improved to only be done when actually needed
# - but version changes happen very infrequently, rebuilding everything
# gives us less corner-cases to deal with, and we can run a VACUUM to make
# things smaller.
self._cleanup_history()
rebuild_completion = True
# Get a string of all patterns
patterns = config.instance.get_str('completion.web_history.exclude')
# If patterns changed, update them in database and rebuild completion
if self.metainfo['excluded_patterns'] != patterns:
self.metainfo['excluded_patterns'] = patterns
rebuild_completion = True
if rebuild_completion and self:
# If no history exists, we don't need to spawn a dialog for
# cleaning it up.
self._rebuild_completion()
self.create_index('HistoryIndex', 'url')
self.create_index('HistoryAtimeIndex', 'atime')
self._contains_query = self.contains_query('url')
self._between_query = sql.Query('SELECT * FROM History '
'where not redirect '
'and not url like "qute://%" '
'and atime > :earliest '
'and atime <= :latest '
'ORDER BY atime desc')
self._before_query = sql.Query('SELECT * FROM History '
'where not redirect '
'and not url like "qute://%" '
'and atime <= :latest '
'ORDER BY atime desc '
'limit :limit offset :offset')
def __getitem__(self, key):
self._check_key(key)
query = sql.Query('SELECT value FROM CompletionMetaInfo '
'WHERE key = :key')
return query.run(key=key).value()
def test_num_rows_affected(self):
q = sql.Query('SELECT 0')
q.run()
assert q.rows_affected() == 0