def __init__(self):
"""
Sets up the DataBase if it has not been initialized yet,
sets up the cache and clears the tmp dir.
Sets a uuid to identify the instance.
"""
super(KnowledgeManager, self).__init__()
self.logger = logging.getLogger("pyphant")
self._cache = []
self._cache_size = 0
if KM_DBASE == u'default':
self.dbase = os.path.join(getPyphantPath('sqlite3'),
"km_meta.sqlite3")
else:
self.dbase = KM_DBASE
self.any_value = AnyValue()
self.node = None # for hooking up a KnowledgeNode
self.uuid = uuid1().urn
tmpdir = getPyphantPath(os.path.join(KM_PATH, 'tmp'))
if os.path.isdir(tmpdir):
from shutil import rmtree
try:
rmtree(tmpdir)
except OSError:
self.logger.warn("Could not delete '%s'." % tmpdir)
with SQLiteWrapper(self.dbase) as wrapper:
rebuild = wrapper.dbase_broken()
if rebuild:
self.logger.info("dbase needs rebuild")
self.rebuildIndex()
else:
with SQLiteWrapper(self.dbase) as wrapper:
wrapper.setup_sqlite()
def __init__(self):
"""
Sets up the DataBase if it has not been initialized yet,
sets up the cache and clears the tmp dir.
Sets a uuid to identify the instance.
"""
super(KnowledgeManager, self).__init__()
self.logger = logging.getLogger("pyphant")
self._cache = []
self._cache_size = 0
if KM_DBASE == u'default':
self.dbase = os.path.join(getPyphantPath('sqlite3'),
"km_meta.sqlite3")
else:
self.dbase = KM_DBASE
self.any_value = AnyValue()
self.node = None # for hooking up a KnowledgeNode
self.uuid = uuid1().urn
tmpdir = getPyphantPath(os.path.join(KM_PATH, 'tmp'))
if os.path.isdir(tmpdir):
from shutil import rmtree
try:
rmtree(tmpdir)
except OSError:
self.logger.warn("Could not delete '%s'." % tmpdir)
with SQLiteWrapper(self.dbase) as wrapper:
rebuild = wrapper.dbase_broken()
if rebuild:
self.logger.info("dbase needs rebuild")
self.rebuildIndex()
else:
with SQLiteWrapper(self.dbase) as wrapper:
wrapper.setup_sqlite()
def updateIndex(self):
file_list = []
def accumulate_files(flist, directory, files):
for fname in [
afname for afname in files if afname.endswith('.h5')
]:
flist.append(os.path.realpath(os.path.join(directory, fname)))
os.path.walk(getPyphantPath(KM_PATH), accumulate_files, file_list)
try:
from wx import (ProgressDialog, PyNoAppError)
try:
pdial = ProgressDialog('Rebuilding index...', ' ' * 100,
maximum=len(file_list))
except PyNoAppError:
pdial = None
except ImportError:
pdial = None
count = 1
for realname in file_list:
if pdial is not None:
pdial.Update(count, os.path.basename(realname))
count += 1
try:
self.registerH5(realname)
except:
self.logger.warn("Could not extract meta data from '%s'."\
% realname)
if pdial is not None:
pdial.Destroy()
def updateIndex(self):
file_list = []
def accumulate_files(flist, directory, files):
for fname in [
afname for afname in files if afname.endswith('.h5')
]:
flist.append(os.path.realpath(os.path.join(directory, fname)))
os.path.walk(getPyphantPath(KM_PATH), accumulate_files, file_list)
try:
from wx import (ProgressDialog, PyNoAppError)
try:
pdial = ProgressDialog('Rebuilding index...',
' ' * 100,
maximum=len(file_list))
except PyNoAppError:
pdial = None
except ImportError:
pdial = None
count = 1
for realname in file_list:
if pdial is not None:
pdial.Update(count, os.path.basename(realname))
count += 1
try:
self.registerH5(realname)
except:
self.logger.warn("Could not extract meta data from '%s'."\
% realname)
if pdial is not None:
pdial.Destroy()
def __init__(self, local_km=None,
host=u'127.0.0.1', port=8080, start=False,
web_interface=False, dbase=u'default'):
"""
Arguments:
- `local_km`: Local KnowledgeManager instance to hook up to.
If set to `None`, KnowledgeManager.getInstance() is used.
- `host`: hostname to listen on
- `port`: port to listen on
- `start`: flag that indicates whether to start the server
- `web_interface`: flag that indicates whether to enable
the web interface. You can enable/disable it anytime by
setting (KN instance).web_interface.enabled to `True`/`False`.
- `dbase`: leave this to 'default', other values are allowed for
debug purposes
"""
RoutingHTTPServer.__init__(self, host, port, start)
if local_km == None:
local_km = KnowledgeManager.getInstance()
self.km = local_km
self.remotes = []
if dbase == u'default':
self._dbase = os.path.join(getPyphantPath('sqlite3'),
'kn_remotes.sqlite3')
else:
self._dbase = dbase
self._restore_remotes()
self._setup_routes()
self._tempdir = mkdtemp(prefix='HDF5Wrap')
tpl_path = pkg_resources.resource_filename('pyphant', 'web/templates')
if not tpl_path in pyphant.core.bottle.TEMPLATE_PATH:
pyphant.core.bottle.TEMPLATE_PATH.append(tpl_path)
from pyphant.core.WebInterface import WebInterface
self.web_interface = WebInterface(self, web_interface)
self.km.node = self
def registerURL(self, url, temporary=False):
"""
Registers an HDF5 or FMF file downloadable from given URL and stores it
in the .pyphant directory. The content of the file is made
available to the KnowledgeManager.
HTTP redirects are resolved. The filetype is determined by the
extension.
url -- URL of the HDF5 or FMF file
temporary -- set to True in order to mark the data to be deleted upon
next instantiation of a KM singleton
"""
parsed = urlparse(url)
tmp_extension = ''
if temporary:
tmp_extension = 'tmp'
try:
remote_path = parsed.path
except AttributeError:
remote_path = parsed[2]
directory = os.path.join(KM_PATH, tmp_extension,
'registered')
filename = os.path.join(getPyphantPath(directory),
os.path.basename(remote_path))
if os.path.exists(filename):
i = 0
directory = os.path.dirname(filename)
basename = os.path.basename(filename)
split = basename.split('.')
ext = split.pop()
fnwoext = ''
for part in split:
fnwoext += (part + '.')
from sys import maxint
while i < maxint:
fill = str(i).zfill(10)
tryfn = os.path.join(directory,
"%s%s.%s" % (fnwoext, fill, ext))
if os.path.exists(tryfn):
i += 1
else:
filename = tryfn
break
self.logger.info("Retrieving url '%s'..." % url)
self.logger.info("Using local file '%s'." % filename)
savedto, headers = urllib.urlretrieve(url, filename)
if REFMF.match(filename.lower()) != None:
self.registerFMF(filename, temporary)
elif REHDF5.match(filename.lower()) != None:
self.registerH5(filename, temporary)
else:
msg = "Could not guess type of '%s'" % url
self.logger.error(msg)
raise ValueError(msg)
def registerURL(self, url, temporary=False):
"""
Registers an HDF5 or FMF file downloadable from given URL and stores it
in the .pyphant directory. The content of the file is made
available to the KnowledgeManager.
HTTP redirects are resolved. The filetype is determined by the
extension.
url -- URL of the HDF5 or FMF file
temporary -- set to True in order to mark the data to be deleted upon
next instantiation of a KM singleton
"""
parsed = urlparse(url)
tmp_extension = ''
if temporary:
tmp_extension = 'tmp'
try:
remote_path = parsed.path
except AttributeError:
remote_path = parsed[2]
directory = os.path.join(KM_PATH, tmp_extension,
'registered')
filename = os.path.join(getPyphantPath(directory),
os.path.basename(remote_path))
if os.path.exists(filename):
i = 0
directory = os.path.dirname(filename)
basename = os.path.basename(filename)
split = basename.split('.')
ext = split.pop()
fnwoext = ''
for part in split:
fnwoext += (part + '.')
from sys import maxint
while i < maxint:
fill = str(i).zfill(10)
tryfn = os.path.join(directory,
"%s%s.%s" % (fnwoext, fill, ext))
if os.path.exists(tryfn):
i += 1
else:
filename = tryfn
break
self.logger.info("Retrieving url '%s'..." % url)
self.logger.info("Using local file '%s'." % filename)
savedto, headers = urllib.urlretrieve(url, filename)
if REFMF.match(filename.lower()) != None:
self.registerFMF(filename, temporary)
elif REHDF5.match(filename.lower()) != None:
self.registerH5(filename, temporary)
else:
msg = "Could not guess type of '%s'" % url
self.logger.error(msg)
raise ValueError(msg)
def getFilenameFromDcId(dcId, temporary=False):
"""
Returns a unique filename for the given emd5.
"""
emd5list = (dcId + '.h5')[7:].split('/')
emd5path = os.path.join(
*(emd5list[:-2] + [emd5list[-2][:10],
emd5list[-2][11:] + '.' + emd5list[-1]]))
#needed for windows version of pytables:
emd5path = emd5path.replace(':', '.')
directory = os.path.dirname(emd5path)
filename = os.path.basename(emd5path)
if temporary:
subdir = os.path.join('tmp', 'by_emd5')
else:
subdir = 'by_emd5'
return os.path.join(getPyphantPath(
os.path.join(KM_PATH, subdir, directory)), filename)
def getFilenameFromDcId(dcId, temporary=False):
"""
Returns a unique filename for the given emd5.
"""
emd5list = (dcId + '.h5')[7:].split('/')
emd5path = os.path.join(
*(emd5list[:-2] +
[emd5list[-2][:10], emd5list[-2][11:] + '.' + emd5list[-1]]))
#needed for windows version of pytables:
emd5path = emd5path.replace(':', '.')
directory = os.path.dirname(emd5path)
filename = os.path.basename(emd5path)
if temporary:
subdir = os.path.join('tmp', 'by_emd5')
else:
subdir = 'by_emd5'
return os.path.join(
getPyphantPath(os.path.join(KM_PATH, subdir, directory)), filename)
def __init__(self,
local_km=None,
host=u'127.0.0.1',
port=8080,
start=False,
web_interface=False,
dbase=u'default'):
"""
Arguments:
- `local_km`: Local KnowledgeManager instance to hook up to.
If set to `None`, KnowledgeManager.getInstance() is used.
- `host`: hostname to listen on
- `port`: port to listen on
- `start`: flag that indicates whether to start the server
- `web_interface`: flag that indicates whether to enable
the web interface. You can enable/disable it anytime by
setting (KN instance).web_interface.enabled to `True`/`False`.
- `dbase`: leave this to 'default', other values are allowed for
debug purposes
"""
RoutingHTTPServer.__init__(self, host, port, start)
if local_km == None:
local_km = KnowledgeManager.getInstance()
self.km = local_km
self.remotes = []
if dbase == u'default':
self._dbase = os.path.join(getPyphantPath('sqlite3'),
'kn_remotes.sqlite3')
else:
self._dbase = dbase
self._restore_remotes()
self._setup_routes()
self._tempdir = mkdtemp(prefix='HDF5Wrap')
tpl_path = os.path.join(pyphant_source_path[0], 'web', 'templates')
if not tpl_path in pyphant.core.bottle.TEMPLATE_PATH:
pyphant.core.bottle.TEMPLATE_PATH.append(tpl_path)
from pyphant.core.WebInterface import WebInterface
self.web_interface = WebInterface(self, web_interface)
self.km.node = self
def log(self):
if not self.enabled:
return template('disabled')
with open(os.path.join(getPyphantPath(), 'pyphant.log')) as logfile:
loglines = logfile.readlines()
return template('log', loglines=''.join(loglines), url=self.url_link)
def log(self):
if not self.enabled:
return template("disabled")
with open(os.path.join(getPyphantPath(), "pyphant.log")) as logfile:
loglines = logfile.readlines()
return template("log", loglines="".join(loglines), url=self.url_link)
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
u"""
"""
__id__ = "$Id$"
__author__ = "$Author$"
__version__ = "Sprint"
# $Source$
import os, os.path, pkg_resources
from pyphant.core.Helpers import getPyphantPath
LOGDIR = getPyphantPath()
import logging
import logging.handlers
# logging.basicConfig(level=logging.DEBUG)
#else:
logging.basicConfig(level=logging.DEBUG,
filename=os.path.join(LOGDIR, u'pyphant.log'),
filemode='w',
format="%(asctime)s - %(levelname)s:%(name)s:%(thread)"\
"d:%(module)s.%(funcName)s(l %(lineno)d):%(message)s")
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
logbuffer = logging.handlers.MemoryHandler(1000)
logging.getLogger('').addHandler(console)
logging.getLogger('').addHandler(logbuffer)
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
u"""
"""
__id__ = "$Id$"
__author__ = "$Author$"
__version__ = "Sprint"
# $Source$
import os, os.path, pkg_resources
from pyphant.core.Helpers import getPyphantPath
LOGDIR = getPyphantPath()
import logging
from logging.handlers import MemoryHandler
logging.basicConfig(level=logging.NOTSET,
filename=os.path.join(LOGDIR, u'pyphant.log'),
filemode='w',
format="%(asctime)s - %(levelname)s:%(name)s:%(thread)"\
"d:%(module)s.%(funcName)s(l %(lineno)d):%(message)s")
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
pdmh = MemoryHandler(1000, flushLevel=logging.CRITICAL + 1)
pdmh.setLevel(logging.WARNING)
logging.getLogger('').addHandler(pdmh)
logging.getLogger('').addHandler(console)
import sys
