def __create_cache(self, uri_list, smart=-1):
"""
create cache and return the number of newly created meta caches
smart is how fast you want to do that:
* 0 force regeneration of entire meta cache
* 1 regenerate cache when hash differs (it would need to open every kitab)
* 2 regenerate when mtime differs
* -1 do not update cache for exiting meta (even if the file is changed)
"""
cn = self._getConnection()
c = cn.cursor()
r = 0
uri_set = set(uri_list)
#c.execute('BEGIN TRANSACTION')
# remove meta for kitab that no longer exists
deleted = filter(lambda i: i not in uri_set, self.__meta_uri_list)
for uri in deleted:
c.execute(SQL_MCACHE_DROP, (uri, ))
r += 1
# update meta for the rest (in a smart way)
for uri in uri_list:
if not os.access(toFs(uri), os.R_OK): continue
if smart == 0:
# force recreation of cache, drop all, then create all
r += self.__cache(c, uri, uri in self.__meta_uri_list)
continue
meta = None
drop_old_needed = False
cache_needed = False
if uri not in self.__meta_uri_list:
cache_needed = True
else:
drop_old_needed = True
cache_needed = True
if smart == -1: continue # don't replace existing cache
elif smart == 2: # rely of mtime
if abs(
os.path.getmtime(toFs(uri)) -
self.getByUri(uri)['mtime']) < 1e-5:
continue
elif smart == 1: # rely on a hash saved inside the database
old_meta = self.getByUri(uri)
meta = self.load_from_uri(uri)
if not meta or old_meta['hash'] == meta['hash']: continue
if cache_needed:
r += self.__cache(c, uri, meta)
#c.execute('END TRANSACTION')
cn.commit()
return r
def __create_cache(self, uri_list, smart = -1):
"""
create cache and return the number of newly created meta caches
smart is how fast you want to do that:
* 0 force regeneration of entire meta cache
* 1 regenerate cache when hash differs (it would need to open every kitab)
* 2 regenerate when mtime differs
* -1 do not update cache for exiting meta (even if the file is changed)
"""
cn = self._getConnection()
c = cn.cursor()
r = 0
uri_set = set(uri_list)
#c.execute('BEGIN TRANSACTION')
# remove meta for kitab that no longer exists
deleted = filter(lambda i: i not in uri_set, self.__meta_uri_list)
for uri in deleted:
c.execute(SQL_MCACHE_DROP, (uri,))
r += 1
# update meta for the rest (in a smart way)
for uri in uri_list:
if not os.access(toFs(uri), os.R_OK): continue
if smart == 0:
# force recreation of cache, drop all, then create all
r+=self.__cache(c, uri, uri in self.__meta_uri_list)
continue
meta = None
drop_old_needed = False
cache_needed = False
if uri not in self.__meta_uri_list:
cache_needed = True
else:
drop_old_needed = True
cache_needed = True
if smart == -1: continue # don't replace existing cache
elif smart == 2: # rely of mtime
if abs(os.path.getmtime(toFs(uri)) - self.getByUri(uri)['mtime']) < 1e-5:
continue
elif smart == 1: # rely on a hash saved inside the database
old_meta = self.getByUri(uri)
meta = self.load_from_uri(uri)
if not meta or old_meta['hash'] == meta['hash']:
continue
if cache_needed:
r += self.__cache(c, uri, meta)
#c.execute('END TRANSACTION')
cn.commit()
return r
def __cache(self, c, uri, meta=None):
if not meta: meta = self.load_from_uri(uri)
if not meta: return 0
#if drop_old_needed:
meta['uri'] = uri
meta['mtime'] = os.path.getmtime(toFs(uri))
meta['flags'] = 0
c.execute(SQL_MCACHE_ADD, meta)
return 1
def __cache(self, c, uri, meta=None): if not meta: meta=self.load_from_uri(uri) if not meta: return 0 #if drop_old_needed: meta['uri']=uri meta['mtime']=os.path.getmtime(toFs(uri)) meta['flags']=0 c.execute(SQL_MCACHE_ADD,meta) return 1
def __init__(self,uri, is_tmp = False, th = None, meta = None):
"""
open the Kitab pointed by uri (or try to create it)
is_tmp should be set to True when we are creating a new kitab from scratch in temporary
th is ThawabManaget to which this book belongs
meta is meta cache entry of this kitab
Note: don't rely on meta having uri, mtime, flags unless th is set (use uri property instead)
"""
self._cn_h = {} # per-thread sqlite connection
# node generators
self.grouped_rows_to_node = (self.grouped_rows_to_node0,
self.grouped_rows_to_node1,
self.grouped_rows_to_node2,
self.grouped_rows_to_node3)
self.row_to_node = (self.row_to_node0, self.row_to_node1)
# TODO: do we need a mode = r|w ?
self.uri = uri
self.is_tmp = is_tmp
self.th = th
self.meta = meta
if not meta: self.getMCache()
if meta and meta.get('originalKitab',None):
self.originalKi = self.th.getCachedKitabByNameV(meta['originalKitab'] + \
u"-" + \
meta['originalVersion'])
else: self.originalKi = None
# the logic to open the uri goes here
# check if fn exists, if not then set the flag sql_create_schema
if is_tmp or not os.path.exists(toFs(uri)):
sql_create_schema = True
else:
sql_create_schema = False
cn = self.cn()
# FIXME: do we really need this
cn.create_function("th_enumerate", 0, self.rowsEnumerator)
# NOTE: we have a policy, no saving of cursors in object attributes for thread safty
c = cn.cursor()
self.toc = KitabToc(self)
# private
self.__tags = {} # a hash by of tags data by tag name
self.__tags_loaded = False
self.__counter = 0 # used to renumber rows
self.inc_size = 1 << 10
# TODO: make a decision, should the root node be saved in SQL,
# if so a lower bound checks to Kitab.getSliceBoundary() and an exception into Kitab.getNodeByIdNum()
self.root = Node(kitab = self,
idNum = 0,
parent = -1,
depth = 0,
content = '',
tags = {})
if sql_create_schema:
c.executescript(SQL_DATA_MODEL)
# create standard tags
for t in STD_TAGS_ARGS:
c.execute(SQL_ADD_TAG, t)
def getManagedUriList(self):
"""list of all managed uri (absolute filenames for a Kitab)
this is low level as the user should work with kitabName, title, and rest of meta data"""
if self.__meta:
return self.__meta.getUriList()
r=[]
for i in self.prefixes:
a=glob(toFs(os.path.join(fromFs(i),u'db',th_ext_glob)))
p=map(lambda j: fromFs(j), a)
r.extend(p)
return r
def __init__(self, uri, is_tmp=False, th=None, meta=None):
"""
open the Kitab pointed by uri (or try to create it)
is_tmp should be set to True when we are creating a new kitab from scratch in temporary
th is ThawabManaget to which this book belongs
meta is meta cache entry of this kitab
Note: don't rely on meta having uri, mtime, flags unless th is set (use uri property instead)
"""
self._cn_h = {} # per-thread sqlite connection
# node generators
self.grouped_rows_to_node = (
self.grouped_rows_to_node0,
self.grouped_rows_to_node1,
self.grouped_rows_to_node2,
self.grouped_rows_to_node3,
)
self.row_to_node = (self.row_to_node0, self.row_to_node1)
# TODO: do we need a mode = r|w ?
self.uri = uri
self.is_tmp = is_tmp
self.th = th
self.meta = meta
if not meta:
self.getMCache()
if meta and meta.get("originalKitab", None):
self.originalKi = self.th.getCachedKitabByNameV(meta["originalKitab"] + u"-" + meta["originalVersion"])
else:
self.originalKi = None
# the logic to open the uri goes here
# check if fn exists, if not then set the flag sql_create_schema
if is_tmp or not os.path.exists(toFs(uri)):
sql_create_schema = True
else:
sql_create_schema = False
cn = self.cn()
# FIXME: do we really need this
cn.create_function("th_enumerate", 0, self.rowsEnumerator)
# NOTE: we have a policy, no saving of cursors in object attributes for thread safty
c = cn.cursor()
self.toc = KitabToc(self)
# private
self.__tags = {} # a hash by of tags data by tag name
self.__tags_loaded = False
self.__counter = 0 # used to renumber rows
self.inc_size = 1 << 10
# TODO: make a decision, should the root node be saved in SQL,
# if so a lower bound checks to Kitab.getSliceBoundary() and an exception into Kitab.getNodeByIdNum()
self.root = Node(kitab=self, idNum=0, parent=-1, depth=0, content="", tags={})
if sql_create_schema:
c.executescript(SQL_DATA_MODEL)
# create standard tags
for t in STD_TAGS_ARGS:
c.execute(SQL_ADD_TAG, t)