def __init__(self):
"""
Sets up the DataBase if it has not been initialized yet,
sets up the cache and clears the tmp dir.
Sets a uuid to identify the instance.
"""
super(KnowledgeManager, self).__init__()
self.logger = logging.getLogger("pyphant")
self._cache = []
self._cache_size = 0
if KM_DBASE == u'default':
self.dbase = os.path.join(getPyphantPath('sqlite3'),
"km_meta.sqlite3")
else:
self.dbase = KM_DBASE
self.any_value = AnyValue()
self.node = None # for hooking up a KnowledgeNode
self.uuid = uuid1().urn
tmpdir = getPyphantPath(os.path.join(KM_PATH, 'tmp'))
if os.path.isdir(tmpdir):
from shutil import rmtree
try:
rmtree(tmpdir)
except OSError:
self.logger.warn("Could not delete '%s'." % tmpdir)
with SQLiteWrapper(self.dbase) as wrapper:
rebuild = wrapper.dbase_broken()
if rebuild:
self.logger.info("dbase needs rebuild")
self.rebuildIndex()
else:
with SQLiteWrapper(self.dbase) as wrapper:
wrapper.setup_sqlite()
def getDataContainer(self, dc_id, use_cache=True, try_remote=True):
"""
Returns DataContainer matching the given id.
dc_id -- Unique ID of the DataContainer (emd5)
use_cache -- Try local cache first and cache DC for further
lookups (default: True)
try_remote -- Try to get DC from remote KMs (default: True)
"""
filename = None
with SQLiteWrapper(self.dbase) as wrapper:
try:
filename = wrapper[dc_id]['storage']
except KeyError:
pass
if filename != None:
if use_cache:
return self.getDCFromCache(dc_id, filename)
with self.getH5FileHandler(filename) as handler:
dc = handler.loadDataContainer(dc_id)
return dc
elif try_remote and self.node != None:
try:
return self.node.get_datacontainer(dc_id)
except DCNotFoundError:
pass
msg = "Could not find DC with id '%s'." % dc_id
self.logger.error(msg)
raise DCNotFoundError(msg)
def isTemporary(self, dcid):
"""
Returns whether the given DC is stored temporarily.
"""
with SQLiteWrapper(self.dbase) as wrapper:
is_tmp = wrapper.is_temporary(dcid)
return is_tmp
def hasDataContainer(self, dcid):
"""
Returns whether the given DC is stored locally.
"""
with SQLiteWrapper(self.dbase) as wrapper:
has_entry = wrapper.has_entry(dcid)
return has_entry
def setTemporary(self, dcid, temporary):
"""
Sets the given entry to temporary, which means it will be
deleted upon restart or removes the temporary flag.
- dcid: emd5 of DataContainer
- temporary: boolean
"""
with SQLiteWrapper(self.dbase) as wrapper:
wrapper.set_temporary(dcid, temporary)
def search(self, result_keys, search_dict={}, order_by=None,
order_asc=True, limit=-1, offset=0, distinct=False):
"""
see SQLiteWrapper.get_andsearch_result
"""
with SQLiteWrapper(self.dbase) as wrapper:
return wrapper.get_andsearch_result(
result_keys, search_dict, order_by, order_asc,
limit, offset, distinct)
def rebuildIndex(self):
self.logger.info("rebuilding dbase...")
oldname = self.dbase + ".bak"
if os.path.exists(oldname):
os.remove(oldname)
os.rename(self.dbase, oldname)
with SQLiteWrapper(self.dbase) as wrapper:
wrapper.setup_dbase()
self.updateIndex()
self.logger.info("done rebuilding dbase")
def getSummary(self, dc_id):
"""
This method returns a dictionary with meta information about
the given DC.
"""
with SQLiteWrapper(self.dbase) as wrapper:
rowwrapper = wrapper[dc_id]
keys = list(SQLiteWrapper.all_keys)
if dc_id.endswith('field'):
keys.remove('columns')
elif dc_id.endswith('sample'):
keys.remove('unit')
keys.remove('dimensions')
summary = dict([(key, rowwrapper[key]) for key in keys])
return summary
def registerH5(self, filename, temporary=False):
"""
Adds the given file to the knowledge pool. If you want the data to
be copied to the .pyphant directory, use registerURL() instead.
filename -- path to the HDF5 file
temporary -- flag that marks data to be deleted upon next
instantiation of a KM Singleton
"""
h5fh = self.getH5FileHandler(filename)
with h5fh:
summaryDict = h5fh.loadSummary()
with SQLiteWrapper(self.dbase) as wrapper:
for dcId, summary in summaryDict.items():
if dcId == im_id:
wrapper.set_entry(summary, None, temporary)
else:
wrapper.set_entry(summary, filename, temporary)
def __init__(self, dc_id, kn):
with SQLiteWrapper(kn.km.dbase) as wrapper:
attrs = wrapper[dc_id]['attributes']
rows = [('attribute', 'value')]
rows.extend(attrs.items())
HTMLTable.__init__(self, rows)
def getEmd5List(self):
"""
returns a list with all locally known DataContainer ids.
"""
with SQLiteWrapper(self.dbase) as wrapper:
return wrapper.get_emd5_list()