def __init__(self, prefixes=None, isMonolithic = True, indexerQueueSize = 0): """Create a new Thawab instance given a user writable directory and an optional system-wide read-only directory prefixes a list of directories all are read-only except the first the first writable directory can be os.path.expanduser('~/.thawab') os.path.join([os.path.dirname(sys.argv[0]),'..','data']) isMonolithic = True if we should use locks and reconnect to sqlite indexerQueueSize is the size of threaded index queue (0 infinite, -1 disabled) the first thing you should do is to call loadMCache() """ if not prefixes: prefixes = guess_prefixes() try: if not os.path.isdir(prefixes[0]): os.makedirs(prefixes[0]) except: raise OSError self.prefixes = filter(lambda i:os.path.isdir(i), [os.path.realpath(os.path.abspath(p)) for p in prefixes]) # make sure it's unique p = self.prefixes[0] s = set(self.prefixes[1:]) if p in s: s.remove(p) if len(s)<len(self.prefixes)-1: self.prefixes=[p]+sorted(list(s)) #print self.prefixes self.othman = othman self.__meta = None self.read_only = self.assertManagedTree() self.conf = self.prase_conf() self.searchEngine = SearchEngine(self) self.user_db = UserDb(self, os.path.join(self.prefixes[0], "user.db")) if indexerQueueSize >= 0: self.asyncIndexer = AsyncIndex(self.searchEngine, indexerQueueSize) else: self.asyncIndexer = None self.isMonolithic = isMonolithic if not self.isMonolithic: import threading lock1 = threading.Lock() else: lock1 = None self.kutubCache = ObjectsCache(lock = lock1)
def __init__(self, prefixes=None, isMonolithic=True, indexerQueueSize=0): """Create a new Thawab instance given a user writable directory and an optional system-wide read-only directory prefixes a list of directories all are read-only except the first the first writable directory can be os.path.expanduser('~/.thawab') os.path.join([os.path.dirname(sys.argv[0]),'..','data']) isMonolithic = True if we should use locks and reconnect to sqlite indexerQueueSize is the size of threaded index queue (0 infinite, -1 disabled) the first thing you should do is to call loadMCache() """ if not prefixes: prefixes = guess_prefixes() try: if not os.path.isdir(prefixes[0]): os.makedirs(prefixes[0]) except: raise OSError self.prefixes = filter(lambda i: os.path.isdir(i), [os.path.realpath(os.path.abspath(p)) for p in prefixes]) # make sure it's unique p = self.prefixes[0] s = set(self.prefixes[1:]) if p in s: s.remove(p) if len(s) < len(self.prefixes) - 1: self.prefixes = [p] + sorted(list(s)) # print self.prefixes self.othman = othman self.__meta = None self.read_only = self.assertManagedTree() self.conf = self.prase_conf() self.searchEngine = SearchEngine(self) self.user_db = UserDb(self, os.path.join(self.prefixes[0], "user.db")) if indexerQueueSize >= 0: self.asyncIndexer = AsyncIndex(self.searchEngine, indexerQueueSize) else: self.asyncIndexer = None self.isMonolithic = isMonolithic if not self.isMonolithic: import threading lock1 = threading.Lock() else: lock1 = None self.kutubCache = ObjectsCache(lock=lock1)
class ThawabMan (object): def __init__(self, prefixes=None, isMonolithic=True, indexerQueueSize=0): """Create a new Thawab instance given a user writable directory and an optional system-wide read-only directory prefixes a list of directories all are read-only except the first the first writable directory can be os.path.expanduser('~/.thawab') os.path.join([os.path.dirname(sys.argv[0]),'..','data']) isMonolithic=True if we should use locks and reconnect to sqlite indexerQueueSize is the size of threaded index queue (0 infinite, -1 disabled) the first thing you should do is to call loadMCache() """ if not prefixes: prefixes=guess_prefixes() try: if not os.path.isdir(prefixes[0]): os.makedirs(prefixes[0]) except: raise OSError self.prefixes=filter(lambda i:os.path.isdir(i), [os.path.realpath(os.path.abspath(p)) for p in prefixes]) # make sure it's unique p=self.prefixes[0] s=set(self.prefixes[1:]) if p in s: s.remove(p) if len(s)<len(self.prefixes)-1: self.prefixes=[p]+sorted(list(s)) #print self.prefixes self.othman=othman self.__meta=None self.read_only = self.assertManagedTree() self.conf = self.prase_conf() self.searchEngine=SearchEngine(self) self.user_db=UserDb(self, os.path.join(self.prefixes[0],"user.db") ) if indexerQueueSize>=0: self.asyncIndexer=AsyncIndex(self.searchEngine, indexerQueueSize) else: self.asyncIndexer=None self.isMonolithic=isMonolithic if not self.isMonolithic: import threading lock1=threading.Lock(); else: lock1=None self.kutubCache=ObjectsCache(lock=lock1) def prase_conf(self): r={} fn=os.path.join(self.prefixes[0], 'conf', 'main.txt') if not os.path.exists(fn): return {} try: f=open(fn) t=f.readlines() f.close() except: return {} for l in t: a=l.strip().split("=",1) if len(a)!=2: continue r[a[0].strip()]=a[1].strip() return r def assertManagedTree(self): """create the hierarchy inside the user-managed prefix # db contains Kitab files [.thawab] # index contains search index # conf application configuration # cache contains the metadata cache for all containers""" P=self.prefixes[0] if not os.access(P, os.W_OK): return False for i in ['db','index','conf','cache', 'tmp', 'themes']: p=os.path.join(P,i) if not os.path.isdir(p): os.makedirs(p) return True def mktemp(self): h,fn=mkstemp(th_ext, 'THAWAB_',os.path.join(self.prefixes[0],'tmp')) return Kitab(fn,True) def getCachedKitab(self, uri): """ try to get a kitab by uri from cache, if it's not in the cache, it will be opened and cached """ ki=self.kutubCache.get(uri) if not ki: ki=self.getKitabByUri(uri) if ki: self.kutubCache.append(uri, ki) #elif not self.isMonolithic: ki.connect() # FIXME: no longer needed, kept to trace other usage of isMonolithic return ki def getCachedKitabByNameV(self, kitabNameV): a=kitabNameV.split(u'-') l=len(a) if l==1: m=self.getMeta().getLatestKitab(kitabNameV) elif l==2: m=self.getMeta().getLatestKitabV(*a) else: m=self.getMeta().getLatestKitabVr(*a) if m: return self.getCachedKitab(m['uri']) return None def getUriByKitabName(self,kitabName): """ return uri for the latest kitab with the given name """ m=self.getMeta().getLatestKitab(kitabName) if not m: return None return m['uri'] def getKitab(self,kitabName): m=self.getMeta().getLatestKitab(kitabName) if m: return Kitab(m['uri'], th=self, meta=m) return None def getKitabByUri(self,uri): m=self.getMeta().getByUri(uri) if m: return Kitab(uri, th=self, meta=m) return Kitab(uri, th=self) def getKitabList(self): """ return a list of managed kitab's name """ return self.getMeta().getKitabList() def getManagedUriList(self): """list of all managed uri (absolute filenames for a Kitab) this is low level as the user should work with kitabName, title, and rest of meta data""" if self.__meta: return self.__meta.getUriList() r=[] for i in self.prefixes: a=glob(toFs(os.path.join(fromFs(i),u'db',th_ext_glob))) p=map(lambda j: fromFs(j), a) r.extend(p) return r def getMeta(self): if not self.__meta: self.loadMeta() return self.__meta def loadMeta(self): self.__meta=None p=os.path.join(self.prefixes[0],'cache','meta.db') self.__meta=MCache(p, self.getManagedUriList()) return self.__meta def reconstructMetaIndexedFlags(self): m=self.loadMeta() # NOTE: getMeta is not used because we want to make sure we are using a fresh one l1=m.getIndexedList() l2=m.getUnindexedList() #l3=m.getDirtyIndexList() # NOTE: Dirty are kept as is for i in l1: v=self.searchEngine.getIndexedVersion(i['kitab']) if not v or metaVrr(i)!=v: m.setIndexedFlags(i['uri'], 0) # mark as unindexed for i in l2: v=self.searchEngine.getIndexedVersion(i['kitab']) if v and metaVrr(i)==v: m.setIndexedFlags(i['uri']) # mark as indexed if same version