def __init__(self, prefixes=None, isMonolithic = True, indexerQueueSize = 0):
"""Create a new Thawab instance given a user writable directory and an optional system-wide read-only directory
prefixes a list of directories all are read-only except the first
the first writable directory can be
os.path.expanduser('~/.thawab')
os.path.join([os.path.dirname(sys.argv[0]),'..','data'])
isMonolithic = True if we should use locks and reconnect to sqlite
indexerQueueSize is the size of threaded index queue (0 infinite, -1 disabled)
the first thing you should do is to call loadMCache()
"""
if not prefixes:
prefixes = guess_prefixes()
try:
if not os.path.isdir(prefixes[0]):
os.makedirs(prefixes[0])
except:
raise OSError
self.prefixes = filter(lambda i:os.path.isdir(i),
[os.path.realpath(os.path.abspath(p)) for p in prefixes])
# make sure it's unique
p = self.prefixes[0]
s = set(self.prefixes[1:])
if p in s:
s.remove(p)
if len(s)<len(self.prefixes)-1:
self.prefixes=[p]+sorted(list(s))
#print self.prefixes
self.othman = othman
self.__meta = None
self.read_only = self.assertManagedTree()
self.conf = self.prase_conf()
self.searchEngine = SearchEngine(self)
self.user_db = UserDb(self, os.path.join(self.prefixes[0], "user.db"))
if indexerQueueSize >= 0:
self.asyncIndexer = AsyncIndex(self.searchEngine, indexerQueueSize)
else:
self.asyncIndexer = None
self.isMonolithic = isMonolithic
if not self.isMonolithic:
import threading
lock1 = threading.Lock()
else:
lock1 = None
self.kutubCache = ObjectsCache(lock = lock1)
def __init__(self, prefixes=None, isMonolithic=True, indexerQueueSize=0):
"""Create a new Thawab instance given a user writable directory and an optional system-wide read-only directory
prefixes a list of directories all are read-only except the first
the first writable directory can be
os.path.expanduser('~/.thawab')
os.path.join([os.path.dirname(sys.argv[0]),'..','data'])
isMonolithic = True if we should use locks and reconnect to sqlite
indexerQueueSize is the size of threaded index queue (0 infinite, -1 disabled)
the first thing you should do is to call loadMCache()
"""
if not prefixes:
prefixes = guess_prefixes()
try:
if not os.path.isdir(prefixes[0]):
os.makedirs(prefixes[0])
except:
raise OSError
self.prefixes = filter(lambda i: os.path.isdir(i), [os.path.realpath(os.path.abspath(p)) for p in prefixes])
# make sure it's unique
p = self.prefixes[0]
s = set(self.prefixes[1:])
if p in s:
s.remove(p)
if len(s) < len(self.prefixes) - 1:
self.prefixes = [p] + sorted(list(s))
# print self.prefixes
self.othman = othman
self.__meta = None
self.read_only = self.assertManagedTree()
self.conf = self.prase_conf()
self.searchEngine = SearchEngine(self)
self.user_db = UserDb(self, os.path.join(self.prefixes[0], "user.db"))
if indexerQueueSize >= 0:
self.asyncIndexer = AsyncIndex(self.searchEngine, indexerQueueSize)
else:
self.asyncIndexer = None
self.isMonolithic = isMonolithic
if not self.isMonolithic:
import threading
lock1 = threading.Lock()
else:
lock1 = None
self.kutubCache = ObjectsCache(lock=lock1)
class ThawabMan (object):
def __init__(self, prefixes=None, isMonolithic=True, indexerQueueSize=0):
"""Create a new Thawab instance given a user writable directory and an optional system-wide read-only directory
prefixes a list of directories all are read-only except the first
the first writable directory can be
os.path.expanduser('~/.thawab')
os.path.join([os.path.dirname(sys.argv[0]),'..','data'])
isMonolithic=True if we should use locks and reconnect to sqlite
indexerQueueSize is the size of threaded index queue (0 infinite, -1 disabled)
the first thing you should do is to call loadMCache()
"""
if not prefixes: prefixes=guess_prefixes()
try:
if not os.path.isdir(prefixes[0]): os.makedirs(prefixes[0])
except:
raise OSError
self.prefixes=filter(lambda i:os.path.isdir(i), [os.path.realpath(os.path.abspath(p)) for p in prefixes])
# make sure it's unique
p=self.prefixes[0]
s=set(self.prefixes[1:])
if p in s: s.remove(p)
if len(s)<len(self.prefixes)-1: self.prefixes=[p]+sorted(list(s))
#print self.prefixes
self.othman=othman
self.__meta=None
self.read_only = self.assertManagedTree()
self.conf = self.prase_conf()
self.searchEngine=SearchEngine(self)
self.user_db=UserDb(self, os.path.join(self.prefixes[0],"user.db") )
if indexerQueueSize>=0:
self.asyncIndexer=AsyncIndex(self.searchEngine, indexerQueueSize)
else:
self.asyncIndexer=None
self.isMonolithic=isMonolithic
if not self.isMonolithic:
import threading
lock1=threading.Lock();
else:
lock1=None
self.kutubCache=ObjectsCache(lock=lock1)
def prase_conf(self):
r={}
fn=os.path.join(self.prefixes[0], 'conf', 'main.txt')
if not os.path.exists(fn): return {}
try:
f=open(fn)
t=f.readlines()
f.close()
except: return {}
for l in t:
a=l.strip().split("=",1)
if len(a)!=2: continue
r[a[0].strip()]=a[1].strip()
return r
def assertManagedTree(self):
"""create the hierarchy inside the user-managed prefix
# db contains Kitab files [.thawab]
# index contains search index
# conf application configuration
# cache contains the metadata cache for all containers"""
P=self.prefixes[0]
if not os.access(P, os.W_OK): return False
for i in ['db','index','conf','cache', 'tmp', 'themes']:
p=os.path.join(P,i)
if not os.path.isdir(p): os.makedirs(p)
return True
def mktemp(self):
h,fn=mkstemp(th_ext, 'THAWAB_',os.path.join(self.prefixes[0],'tmp'))
return Kitab(fn,True)
def getCachedKitab(self, uri):
"""
try to get a kitab by uri from cache,
if it's not in the cache, it will be opened and cached
"""
ki=self.kutubCache.get(uri)
if not ki:
ki=self.getKitabByUri(uri)
if ki: self.kutubCache.append(uri, ki)
#elif not self.isMonolithic: ki.connect() # FIXME: no longer needed, kept to trace other usage of isMonolithic
return ki
def getCachedKitabByNameV(self, kitabNameV):
a=kitabNameV.split(u'-')
l=len(a)
if l==1:
m=self.getMeta().getLatestKitab(kitabNameV)
elif l==2:
m=self.getMeta().getLatestKitabV(*a)
else:
m=self.getMeta().getLatestKitabVr(*a)
if m: return self.getCachedKitab(m['uri'])
return None
def getUriByKitabName(self,kitabName):
"""
return uri for the latest kitab with the given name
"""
m=self.getMeta().getLatestKitab(kitabName)
if not m: return None
return m['uri']
def getKitab(self,kitabName):
m=self.getMeta().getLatestKitab(kitabName)
if m: return Kitab(m['uri'], th=self, meta=m)
return None
def getKitabByUri(self,uri):
m=self.getMeta().getByUri(uri)
if m: return Kitab(uri, th=self, meta=m)
return Kitab(uri, th=self)
def getKitabList(self):
"""
return a list of managed kitab's name
"""
return self.getMeta().getKitabList()
def getManagedUriList(self):
"""list of all managed uri (absolute filenames for a Kitab)
this is low level as the user should work with kitabName, title, and rest of meta data"""
if self.__meta:
return self.__meta.getUriList()
r=[]
for i in self.prefixes:
a=glob(toFs(os.path.join(fromFs(i),u'db',th_ext_glob)))
p=map(lambda j: fromFs(j), a)
r.extend(p)
return r
def getMeta(self):
if not self.__meta: self.loadMeta()
return self.__meta
def loadMeta(self):
self.__meta=None
p=os.path.join(self.prefixes[0],'cache','meta.db')
self.__meta=MCache(p, self.getManagedUriList())
return self.__meta
def reconstructMetaIndexedFlags(self):
m=self.loadMeta() # NOTE: getMeta is not used because we want to make sure we are using a fresh one
l1=m.getIndexedList()
l2=m.getUnindexedList()
#l3=m.getDirtyIndexList() # NOTE: Dirty are kept as is
for i in l1:
v=self.searchEngine.getIndexedVersion(i['kitab'])
if not v or metaVrr(i)!=v: m.setIndexedFlags(i['uri'], 0) # mark as unindexed
for i in l2:
v=self.searchEngine.getIndexedVersion(i['kitab'])
if v and metaVrr(i)==v: m.setIndexedFlags(i['uri']) # mark as indexed if same version
