Ejemplo n.º 1
0
 def __create_cache(self, uri_list, smart=-1):
     """
 create cache and return the number of newly created meta caches
 
 smart is how fast you want to do that:
   * 0 force regeneration of entire meta cache
   * 1 regenerate cache when hash differs (it would need to open every kitab)
   * 2 regenerate when mtime differs
   * -1 do not update cache for exiting meta (even if the file is changed)
 """
     cn = self._getConnection()
     c = cn.cursor()
     r = 0
     uri_set = set(uri_list)
     #c.execute('BEGIN TRANSACTION')
     # remove meta for kitab that no longer exists
     deleted = filter(lambda i: i not in uri_set, self.__meta_uri_list)
     for uri in deleted:
         c.execute(SQL_MCACHE_DROP, (uri, ))
         r += 1
     # update meta for the rest (in a smart way)
     for uri in uri_list:
         if not os.access(toFs(uri), os.R_OK): continue
         if smart == 0:
             # force recreation of cache, drop all, then create all
             r += self.__cache(c, uri, uri in self.__meta_uri_list)
             continue
         meta = None
         drop_old_needed = False
         cache_needed = False
         if uri not in self.__meta_uri_list:
             cache_needed = True
         else:
             drop_old_needed = True
             cache_needed = True
             if smart == -1: continue  # don't replace existing cache
             elif smart == 2:  # rely of mtime
                 if abs(
                         os.path.getmtime(toFs(uri)) -
                         self.getByUri(uri)['mtime']) < 1e-5:
                     continue
             elif smart == 1:  # rely on a hash saved inside the database
                 old_meta = self.getByUri(uri)
                 meta = self.load_from_uri(uri)
                 if not meta or old_meta['hash'] == meta['hash']: continue
         if cache_needed:
             r += self.__cache(c, uri, meta)
     #c.execute('END TRANSACTION')
     cn.commit()
     return r
Ejemplo n.º 2
0
 def __create_cache(self, uri_list, smart = -1):
     """
     create cache and return the number of newly created meta caches
     
     smart is how fast you want to do that:
         * 0 force regeneration of entire meta cache
         * 1 regenerate cache when hash differs (it would need to open every kitab)
         * 2 regenerate when mtime differs
         * -1 do not update cache for exiting meta (even if the file is changed)
     """
     cn = self._getConnection()
     c = cn.cursor()
     r = 0
     uri_set = set(uri_list)
     #c.execute('BEGIN TRANSACTION')
     # remove meta for kitab that no longer exists
     deleted = filter(lambda i: i not in uri_set, self.__meta_uri_list)
     for uri in deleted:
         c.execute(SQL_MCACHE_DROP, (uri,))
         r += 1
     # update meta for the rest (in a smart way)
     for uri in uri_list:
         if not os.access(toFs(uri), os.R_OK): continue
         if smart == 0:
             # force recreation of cache, drop all, then create all
             r+=self.__cache(c, uri, uri in self.__meta_uri_list)
             continue
         meta = None
         drop_old_needed = False
         cache_needed = False
         if uri not in self.__meta_uri_list:
             cache_needed = True
         else:
             drop_old_needed = True
             cache_needed = True
             if smart == -1: continue # don't replace existing cache
             elif smart == 2: # rely of mtime
                 if abs(os.path.getmtime(toFs(uri)) - self.getByUri(uri)['mtime']) < 1e-5:
                     continue
             elif smart == 1: # rely on a hash saved inside the database
                 old_meta = self.getByUri(uri)
                 meta = self.load_from_uri(uri)
                 if not meta or old_meta['hash'] == meta['hash']:
                     continue
         if cache_needed:
             r += self.__cache(c, uri, meta)
     #c.execute('END TRANSACTION')
     cn.commit()
     return r
Ejemplo n.º 3
0
 def __cache(self, c, uri, meta=None):
     if not meta: meta = self.load_from_uri(uri)
     if not meta: return 0
     #if drop_old_needed:
     meta['uri'] = uri
     meta['mtime'] = os.path.getmtime(toFs(uri))
     meta['flags'] = 0
     c.execute(SQL_MCACHE_ADD, meta)
     return 1
Ejemplo n.º 4
0
 def __cache(self, c, uri, meta=None):
   if not meta: meta=self.load_from_uri(uri)
   if not meta: return 0
   #if drop_old_needed: 
   meta['uri']=uri
   meta['mtime']=os.path.getmtime(toFs(uri))
   meta['flags']=0
   c.execute(SQL_MCACHE_ADD,meta)
   return 1
Ejemplo n.º 5
0
 def __init__(self,uri, is_tmp = False, th = None, meta = None):
     """
     open the Kitab pointed by uri (or try to create it)
     is_tmp should be set to True when we are creating a new kitab from scratch in temporary 
     th is ThawabManaget to which this book belongs
     meta is meta cache entry of this kitab
     
     Note: don't rely on meta having uri, mtime, flags unless th is set (use uri property instead)
     """
     self._cn_h = {} # per-thread sqlite connection
     # node generators
     self.grouped_rows_to_node = (self.grouped_rows_to_node0,
                                  self.grouped_rows_to_node1,
                                  self.grouped_rows_to_node2,
                                  self.grouped_rows_to_node3)
     self.row_to_node = (self.row_to_node0, self.row_to_node1)
     # TODO: do we need a mode = r|w ?
     self.uri = uri
     self.is_tmp = is_tmp
     self.th = th
     self.meta = meta
     if not meta: self.getMCache()
     if meta and meta.get('originalKitab',None):
         self.originalKi = self.th.getCachedKitabByNameV(meta['originalKitab'] + \
                                                         u"-" + \
                                                         meta['originalVersion'])
     else: self.originalKi = None
     # the logic to open the uri goes here
     # check if fn exists, if not then set the flag sql_create_schema
     if is_tmp or not os.path.exists(toFs(uri)):
         sql_create_schema = True
     else:
         sql_create_schema = False
     cn = self.cn()
     # FIXME: do we really need this
     cn.create_function("th_enumerate", 0, self.rowsEnumerator) 
     # NOTE: we have a policy, no saving of cursors in object attributes for thread safty
     c = cn.cursor()
     self.toc = KitabToc(self)
     # private
     self.__tags = {} # a hash by of tags data by tag name
     self.__tags_loaded = False
     self.__counter = 0 # used to renumber rows
     self.inc_size = 1 << 10
     # TODO: make a decision, should the root node be saved in SQL,
     # if so a lower bound checks to Kitab.getSliceBoundary() and an exception into Kitab.getNodeByIdNum()
     self.root = Node(kitab = self,
                      idNum = 0,
                      parent = -1,
                      depth = 0,
                      content = '',
                      tags = {})
     if sql_create_schema:
         c.executescript(SQL_DATA_MODEL)
         # create standard tags
         for t in STD_TAGS_ARGS:
             c.execute(SQL_ADD_TAG, t)
Ejemplo n.º 6
0
 def getManagedUriList(self):
   """list of all managed uri (absolute filenames for a Kitab)
    this is low level as the user should work with kitabName, title, and rest of meta data"""
   if self.__meta:
     return self.__meta.getUriList()
   r=[]
   for i in self.prefixes:
     a=glob(toFs(os.path.join(fromFs(i),u'db',th_ext_glob)))
     p=map(lambda j: fromFs(j), a)
     r.extend(p)
   return r
Ejemplo n.º 7
0
 def __init__(self, uri, is_tmp=False, th=None, meta=None):
     """
     open the Kitab pointed by uri (or try to create it)
     is_tmp should be set to True when we are creating a new kitab from scratch in temporary 
     th is ThawabManaget to which this book belongs
     meta is meta cache entry of this kitab
     
     Note: don't rely on meta having uri, mtime, flags unless th is set (use uri property instead)
     """
     self._cn_h = {}  # per-thread sqlite connection
     # node generators
     self.grouped_rows_to_node = (
         self.grouped_rows_to_node0,
         self.grouped_rows_to_node1,
         self.grouped_rows_to_node2,
         self.grouped_rows_to_node3,
     )
     self.row_to_node = (self.row_to_node0, self.row_to_node1)
     # TODO: do we need a mode = r|w ?
     self.uri = uri
     self.is_tmp = is_tmp
     self.th = th
     self.meta = meta
     if not meta:
         self.getMCache()
     if meta and meta.get("originalKitab", None):
         self.originalKi = self.th.getCachedKitabByNameV(meta["originalKitab"] + u"-" + meta["originalVersion"])
     else:
         self.originalKi = None
     # the logic to open the uri goes here
     # check if fn exists, if not then set the flag sql_create_schema
     if is_tmp or not os.path.exists(toFs(uri)):
         sql_create_schema = True
     else:
         sql_create_schema = False
     cn = self.cn()
     # FIXME: do we really need this
     cn.create_function("th_enumerate", 0, self.rowsEnumerator)
     # NOTE: we have a policy, no saving of cursors in object attributes for thread safty
     c = cn.cursor()
     self.toc = KitabToc(self)
     # private
     self.__tags = {}  # a hash by of tags data by tag name
     self.__tags_loaded = False
     self.__counter = 0  # used to renumber rows
     self.inc_size = 1 << 10
     # TODO: make a decision, should the root node be saved in SQL,
     # if so a lower bound checks to Kitab.getSliceBoundary() and an exception into Kitab.getNodeByIdNum()
     self.root = Node(kitab=self, idNum=0, parent=-1, depth=0, content="", tags={})
     if sql_create_schema:
         c.executescript(SQL_DATA_MODEL)
         # create standard tags
         for t in STD_TAGS_ARGS:
             c.execute(SQL_ADD_TAG, t)