def __init__(self): """ Sets up the DataBase if it has not been initialized yet, sets up the cache and clears the tmp dir. Sets a uuid to identify the instance. """ super(KnowledgeManager, self).__init__() self.logger = logging.getLogger("pyphant") self._cache = [] self._cache_size = 0 if KM_DBASE == u'default': self.dbase = os.path.join(getPyphantPath('sqlite3'), "km_meta.sqlite3") else: self.dbase = KM_DBASE self.any_value = AnyValue() self.node = None # for hooking up a KnowledgeNode self.uuid = uuid1().urn tmpdir = getPyphantPath(os.path.join(KM_PATH, 'tmp')) if os.path.isdir(tmpdir): from shutil import rmtree try: rmtree(tmpdir) except OSError: self.logger.warn("Could not delete '%s'." % tmpdir) with SQLiteWrapper(self.dbase) as wrapper: rebuild = wrapper.dbase_broken() if rebuild: self.logger.info("dbase needs rebuild") self.rebuildIndex() else: with SQLiteWrapper(self.dbase) as wrapper: wrapper.setup_sqlite()
def updateIndex(self): file_list = [] def accumulate_files(flist, directory, files): for fname in [ afname for afname in files if afname.endswith('.h5') ]: flist.append(os.path.realpath(os.path.join(directory, fname))) os.path.walk(getPyphantPath(KM_PATH), accumulate_files, file_list) try: from wx import (ProgressDialog, PyNoAppError) try: pdial = ProgressDialog('Rebuilding index...', ' ' * 100, maximum=len(file_list)) except PyNoAppError: pdial = None except ImportError: pdial = None count = 1 for realname in file_list: if pdial is not None: pdial.Update(count, os.path.basename(realname)) count += 1 try: self.registerH5(realname) except: self.logger.warn("Could not extract meta data from '%s'."\ % realname) if pdial is not None: pdial.Destroy()
def __init__(self, local_km=None, host=u'127.0.0.1', port=8080, start=False, web_interface=False, dbase=u'default'): """ Arguments: - `local_km`: Local KnowledgeManager instance to hook up to. If set to `None`, KnowledgeManager.getInstance() is used. - `host`: hostname to listen on - `port`: port to listen on - `start`: flag that indicates whether to start the server - `web_interface`: flag that indicates whether to enable the web interface. You can enable/disable it anytime by setting (KN instance).web_interface.enabled to `True`/`False`. - `dbase`: leave this to 'default', other values are allowed for debug purposes """ RoutingHTTPServer.__init__(self, host, port, start) if local_km == None: local_km = KnowledgeManager.getInstance() self.km = local_km self.remotes = [] if dbase == u'default': self._dbase = os.path.join(getPyphantPath('sqlite3'), 'kn_remotes.sqlite3') else: self._dbase = dbase self._restore_remotes() self._setup_routes() self._tempdir = mkdtemp(prefix='HDF5Wrap') tpl_path = pkg_resources.resource_filename('pyphant', 'web/templates') if not tpl_path in pyphant.core.bottle.TEMPLATE_PATH: pyphant.core.bottle.TEMPLATE_PATH.append(tpl_path) from pyphant.core.WebInterface import WebInterface self.web_interface = WebInterface(self, web_interface) self.km.node = self
def registerURL(self, url, temporary=False): """ Registers an HDF5 or FMF file downloadable from given URL and stores it in the .pyphant directory. The content of the file is made available to the KnowledgeManager. HTTP redirects are resolved. The filetype is determined by the extension. url -- URL of the HDF5 or FMF file temporary -- set to True in order to mark the data to be deleted upon next instantiation of a KM singleton """ parsed = urlparse(url) tmp_extension = '' if temporary: tmp_extension = 'tmp' try: remote_path = parsed.path except AttributeError: remote_path = parsed[2] directory = os.path.join(KM_PATH, tmp_extension, 'registered') filename = os.path.join(getPyphantPath(directory), os.path.basename(remote_path)) if os.path.exists(filename): i = 0 directory = os.path.dirname(filename) basename = os.path.basename(filename) split = basename.split('.') ext = split.pop() fnwoext = '' for part in split: fnwoext += (part + '.') from sys import maxint while i < maxint: fill = str(i).zfill(10) tryfn = os.path.join(directory, "%s%s.%s" % (fnwoext, fill, ext)) if os.path.exists(tryfn): i += 1 else: filename = tryfn break self.logger.info("Retrieving url '%s'..." % url) self.logger.info("Using local file '%s'." % filename) savedto, headers = urllib.urlretrieve(url, filename) if REFMF.match(filename.lower()) != None: self.registerFMF(filename, temporary) elif REHDF5.match(filename.lower()) != None: self.registerH5(filename, temporary) else: msg = "Could not guess type of '%s'" % url self.logger.error(msg) raise ValueError(msg)
def getFilenameFromDcId(dcId, temporary=False): """ Returns a unique filename for the given emd5. """ emd5list = (dcId + '.h5')[7:].split('/') emd5path = os.path.join( *(emd5list[:-2] + [emd5list[-2][:10], emd5list[-2][11:] + '.' + emd5list[-1]])) #needed for windows version of pytables: emd5path = emd5path.replace(':', '.') directory = os.path.dirname(emd5path) filename = os.path.basename(emd5path) if temporary: subdir = os.path.join('tmp', 'by_emd5') else: subdir = 'by_emd5' return os.path.join(getPyphantPath( os.path.join(KM_PATH, subdir, directory)), filename)
def getFilenameFromDcId(dcId, temporary=False): """ Returns a unique filename for the given emd5. """ emd5list = (dcId + '.h5')[7:].split('/') emd5path = os.path.join( *(emd5list[:-2] + [emd5list[-2][:10], emd5list[-2][11:] + '.' + emd5list[-1]])) #needed for windows version of pytables: emd5path = emd5path.replace(':', '.') directory = os.path.dirname(emd5path) filename = os.path.basename(emd5path) if temporary: subdir = os.path.join('tmp', 'by_emd5') else: subdir = 'by_emd5' return os.path.join( getPyphantPath(os.path.join(KM_PATH, subdir, directory)), filename)
def __init__(self, local_km=None, host=u'127.0.0.1', port=8080, start=False, web_interface=False, dbase=u'default'): """ Arguments: - `local_km`: Local KnowledgeManager instance to hook up to. If set to `None`, KnowledgeManager.getInstance() is used. - `host`: hostname to listen on - `port`: port to listen on - `start`: flag that indicates whether to start the server - `web_interface`: flag that indicates whether to enable the web interface. You can enable/disable it anytime by setting (KN instance).web_interface.enabled to `True`/`False`. - `dbase`: leave this to 'default', other values are allowed for debug purposes """ RoutingHTTPServer.__init__(self, host, port, start) if local_km == None: local_km = KnowledgeManager.getInstance() self.km = local_km self.remotes = [] if dbase == u'default': self._dbase = os.path.join(getPyphantPath('sqlite3'), 'kn_remotes.sqlite3') else: self._dbase = dbase self._restore_remotes() self._setup_routes() self._tempdir = mkdtemp(prefix='HDF5Wrap') tpl_path = os.path.join(pyphant_source_path[0], 'web', 'templates') if not tpl_path in pyphant.core.bottle.TEMPLATE_PATH: pyphant.core.bottle.TEMPLATE_PATH.append(tpl_path) from pyphant.core.WebInterface import WebInterface self.web_interface = WebInterface(self, web_interface) self.km.node = self
def log(self): if not self.enabled: return template('disabled') with open(os.path.join(getPyphantPath(), 'pyphant.log')) as logfile: loglines = logfile.readlines() return template('log', loglines=''.join(loglines), url=self.url_link)
def log(self): if not self.enabled: return template("disabled") with open(os.path.join(getPyphantPath(), "pyphant.log")) as logfile: loglines = logfile.readlines() return template("log", loglines="".join(loglines), url=self.url_link)
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. u""" """ __id__ = "$Id$" __author__ = "$Author$" __version__ = "Sprint" # $Source$ import os, os.path, pkg_resources from pyphant.core.Helpers import getPyphantPath LOGDIR = getPyphantPath() import logging import logging.handlers # logging.basicConfig(level=logging.DEBUG) #else: logging.basicConfig(level=logging.DEBUG, filename=os.path.join(LOGDIR, u'pyphant.log'), filemode='w', format="%(asctime)s - %(levelname)s:%(name)s:%(thread)"\ "d:%(module)s.%(funcName)s(l %(lineno)d):%(message)s") console = logging.StreamHandler() console.setLevel(logging.WARNING) logbuffer = logging.handlers.MemoryHandler(1000) logging.getLogger('').addHandler(console) logging.getLogger('').addHandler(logbuffer)
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. u""" """ __id__ = "$Id$" __author__ = "$Author$" __version__ = "Sprint" # $Source$ import os, os.path, pkg_resources from pyphant.core.Helpers import getPyphantPath LOGDIR = getPyphantPath() import logging from logging.handlers import MemoryHandler logging.basicConfig(level=logging.NOTSET, filename=os.path.join(LOGDIR, u'pyphant.log'), filemode='w', format="%(asctime)s - %(levelname)s:%(name)s:%(thread)"\ "d:%(module)s.%(funcName)s(l %(lineno)d):%(message)s") console = logging.StreamHandler() console.setLevel(logging.WARNING) pdmh = MemoryHandler(1000, flushLevel=logging.CRITICAL + 1) pdmh.setLevel(logging.WARNING) logging.getLogger('').addHandler(pdmh) logging.getLogger('').addHandler(console) import sys