def load_sqlite_backup(backup_date: str, test, **kwargs):
if test:
logger.debug("load_sqlite_backup task")
return
with db_factory.get_instance() as db:
assert isinstance(db, CSqliteExtDatabase)
backup_filename = os.path.join(config.get('sqlite', 'backup_dir'),
_BACKUP_FORMAT % (backup_date))
db_backup = CSqliteExtDatabase(backup_filename)
db_backup.backup(db)
def __init__(self, database=None):
if database is None:
database = SqliteExtDatabase(':memory:')
if HAVE_C_EXTENSION:
register_json_contains(database)
else:
database.register_function(_json_contains, 'json_contains')
self.database = database
self.V, self.E = self.create_models()
def __init__(self, filename, cache_size=32, thread_safe=True, **params):
if SqliteDatabase is None:
raise ImproperlyConfigured('Cannot use SqliteCache - peewee is '
'not installed')
self._filename = filename
self._cache_size = cache_size # In MiB.
self._thread_safe = thread_safe
self._db = SqliteDatabase(
self._filename,
thread_safe=self._thread_safe,
pragmas={
'cache_size': self._cache_size * -1000,
'journal_mode': 'wal', # Multiple readers + one writer.
'synchronous': 0,
'wal_synchronous': 0})
class Cache(Model):
key = TextField(primary_key=True)
value = BlobField(null=True)
expires = FloatField()
class Meta:
database = self._db
indexes = (
(('key', 'expires'), False),
)
without_rowid = True
self.cache = Cache
super(SqliteCache, self).__init__(**params)
def get_instance(self, instance: str = None):
if not instance:
instance = self.defaut_instance
if instance not in self.instances.keys():
if instance == 'sqlite':
instance_obj = CSqliteExtDatabase(self.sqlite_db_path, autoconnect=False)
elif instance == 'sqlite-app-test':
PACKAGR_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
instance_obj = CSqliteExtDatabase(os.path.join(PACKAGR_DIR, 'mediastrends_test.db'))
elif instance == 'memory':
instance_obj = CSqliteExtDatabase(':memory:')
else:
raise ValueError("Instance %s not defined" % (instance))
self.instances[instance] = instance_obj
instance = self.instances[instance]
self.database_proxy.initialize(instance)
return instance
def prepare_database_with_table(name: str, rows: list):
from peewee import IntegerField, Proxy, CharField, Model
from playhouse.sqlite_ext import CSqliteExtDatabase
db = Proxy()
db.initialize(CSqliteExtDatabase(':memory:', bloomfilter=True))
NameModel = type(
name, (Model, ), {
'id_': IntegerField(primary_key=True, column_name='id'),
'name': CharField(column_name='name')
})
table: Model = NameModel()
table.bind(db)
db.create_tables([NameModel])
for row in rows:
table.insert(row).execute()
return db
from playhouse.sqlite_ext import *
try:
from playhouse.sqlite_ext import CSqliteExtDatabase as SqliteExtDatabase
except ImportError:
pass
import lcars.settings as settings
indexDir = settings.data_root / "searchIndex"
(settings.data_root / "searchIndex").mkdir(parents=True, exist_ok=True)
db = SqliteExtDatabase(
indexDir / 'search_db.sqlite3',
regexp_function=True,
timeout=60, #timeout not working?
pragmas={
'journal_mode': 'wal',
'cache_size': -1 * 64000, # 64MB
'foreign_keys': 1,
'ignore_check_constraints': 0,
'synchronous': 0
})
import requests
session = requests.Session()
class Document(Model):
# Canonical source of data, stored in a regular table.
url = TextField(null=False, unique=True)
title = TextField(unique=False, default="")
body = TextField(default="")
class SqliteCache(Cache):
def __init__(self, filename, cache_size=32, thread_safe=True, **params):
if SqliteDatabase is None:
raise ImproperlyConfigured('Cannot use SqliteCache - peewee is '
'not installed')
self._filename = filename
self._cache_size = cache_size # In MiB.
self._thread_safe = thread_safe
self._db = SqliteDatabase(
self._filename,
thread_safe=self._thread_safe,
pragmas={
'cache_size': self._cache_size * -1000,
'journal_mode': 'wal', # Multiple readers + one writer.
'synchronous': 0,
'wal_synchronous': 0})
class Cache(Model):
key = TextField(primary_key=True)
value = BlobField(null=True)
expires = FloatField()
class Meta:
database = self._db
indexes = (
(('key', 'expires'), False),
)
without_rowid = True
self.cache = Cache
super(SqliteCache, self).__init__(**params)
def open(self):
if not self._db.is_closed(): return False
self._db.connect()
self.cache.create_table()
return True
def close(self):
if self._db.is_closed(): return False
self._db.close()
return True
def _get(self, key):
query = (self.cache
.select(self.cache.value)
.where((self.cache.key == key) &
(self.cache.expires >= time.time()))
.limit(1)
.tuples())
try:
return query.get()[0]
except self.cache.DoesNotExist:
pass
def _get_many(self, keys):
query = (self.cache
.select(self.cache.key, self.cache.value)
.where(self.cache.key.in_(keys) &
(self.cache.expires >= time.time()))
.tuples())
return dict(query)
def _set(self, key, value, timeout):
expires = time.time() + timeout
self.cache.replace(key=key, value=value, expires=expires).execute()
def _set_many(self, data, timeout):
expires = time.time() + timeout
normalized = [(key, value, expires) for key, value in data.items()]
fields = [self.cache.key, self.cache.value, self.cache.expires]
return (self.cache
.replace_many(normalized, fields=fields)
.execute())
def _delete(self, key):
return self.cache.delete().where(self.cache.key == key).execute()
def _delete_many(self, keys):
return self.cache.delete().where(self.cache.key.in_(keys)).execute()
def _flush(self):
query = self.cache.delete()
if self.prefix:
prefix = decode(self.prefix)
query = query.where(
fn.SUBSTR(self.cache.key, 1, len(prefix)) == prefix)
return query.execute()
def clean_expired(self, ndays=0):
timestamp = time.time() - (ndays * 86400)
return (self.cache
.delete()
.where(self.cache.expires <= timestamp)
.execute())