def start_http(self, app, args):
port = int(args[1])
notebook_path = unicode(args[2])
# connect to notebook on disk
conn = NoteBookConnectionFS()
conn.connect(notebook_path)
# start server in another thread
host = "localhost"
url = "http://%s:%d/" % (host, port)
server = NoteBookHttpServer(conn, host="localhost", port=port)
if port in self._ports:
raise Exception("Server already on port %d" % port)
self._ports[port] = server
keepnote.log_message("starting server:\n%s\n" % url)
thread.start_new_thread(server.serve_forever, ())
if host == "localhost":
keepnote.log_message(
"NOTE: server is local only. Use ssh port forwarding for security.\n"
)
def compact(self):
"""
Try to compact the index by reclaiming space
"""
keepnote.log_message("compacting index '%s'\n" % self._index_file)
self.con.execute("VACUUM;")
self.con.comment()
def compact(self):
"""
Try to compact the index by reclaiming space
"""
keepnote.log_message("compacting index '%s'\n" % self._index_file)
self.con.execute("VACUUM;")
self.con.comment()
def install_extension(self, filename):
"""Install a new extension from package 'filename'"""
userdir = get_user_extensions_dir()
newfiles = []
try:
# unzip and record new files
for fn in unzip(filename, userdir):
newfiles.append(fn)
# rescan user extensions
exts = set(self._extensions.keys())
self._scan_extension_path(userdir, "user")
# find new extensions
new_names = set(self._extensions.keys()) - exts
new_exts = [self.get_extension(name) for name in new_names]
except Exception, e:
self.error(_("Unable to install extension '%s'") % filename,
e, tracebk=sys.exc_info()[2])
# delete newfiles
for fn in newfiles:
try:
keepnote.log_message(_("removing '%s'") % newfile)
os.remove(newfile)
except:
# delete may fail, continue
pass
return []
def write_anchor(self, dom, anchor, xhtml=True):
"""Write an anchor object"""
for tag_writer in self._tag_writers:
if isinstance(anchor, tag_writer.tagclass):
tag_writer.write(self._out, dom, xhtml)
return
# warning
log_message("unknown anchor element", anchor)
def check_index(self):
"""Check filesystem to see if index is up to date"""
keepnote.log_message("checking index... ")
start = time.time()
mtime_index = self.get_mtime()
mtime = keepnote.notebook.connection.fs.last_node_change(
self._nconn._get_node_path(self._nconn.get_rootid()))
keepnote.log_message("%f seconds\n" % (time.time() - start))
return (mtime > mtime_index)
def check_index(self):
"""Check filesystem to see if index is up to date"""
keepnote.log_message("checking index... ")
start = time.time()
mtime_index = self.get_mtime()
mtime = keepnote.notebook.connection.fs.last_node_change(
self._nconn._get_node_path(self._nconn.get_rootid()))
keepnote.log_message("%f seconds\n" % (time.time() - start))
return (mtime > mtime_index)
def write_anchor(self, dom, anchor, xhtml=True):
"""Write an anchor object"""
for tag_writer in self._tag_writers:
if isinstance(anchor, tag_writer.tagclass):
tag_writer.write(self._out, dom, xhtml)
return
# warning
log_message("unknown anchor element", anchor)
def convert_node_attr(filename, filename2, attr_defs=g_attr_defs_lookup):
"""Convert a node.xml file from version 5 to 6"""
keepnote.log_message("converting '%s'...\n" % filename2)
try:
attr = read_attr_v5(filename, attr_defs)
attr["version"] = 6
write_attr_v6(filename2, attr)
except Exception, e:
keepnote.log_error("cannot convert %s: %s\n" % (filename, str(e)),
sys.exc_info()[2])
def convert_node_attr(filename, filename2, attr_defs=g_attr_defs_lookup):
"""Convert a node.xml file from version 5 to 6"""
keepnote.log_message("converting '%s'...\n" % filename2)
try:
attr = read_attr_v5(filename, attr_defs)
attr["version"] = 6
write_attr_v6(filename2, attr)
except Exception, e:
keepnote.log_error("cannot convert %s: %s\n" % (filename, str(e)),
sys.exc_info()[2])
def stop_http(self, app, args):
port = int(args[1])
if port not in self._ports:
raise Exception("No server is on port %d" % port)
server = self._ports[port]
keepnote.log_message("stopping server on port %d...\n" % port)
server.shutdown()
del self._ports[port]
def stop_http(self, app, args):
port = int(args[1])
if port not in self._ports:
raise Exception("No server is on port %d" % port)
server = self._ports[port]
keepnote.log_message("stopping server on port %d...\n" % port)
server.shutdown()
del self._ports[port]
def init_index(self, auto_clear=True):
"""Initialize the tables in the index if they do not exist"""
con = self.con
try:
# check database version
version = self._get_version()
if version is None or version != INDEX_VERSION:
# version does not match, drop all tables
self._drop_tables()
# update version
self._set_version()
self._need_index = True
# init NodeGraph table
con.execute(
u"""CREATE TABLE IF NOT EXISTS NodeGraph
(nodeid TEXT,
parentid TEXT,
basename TEXT,
mtime FLOAT,
symlink BOOLEAN,
UNIQUE(nodeid) ON CONFLICT REPLACE);
"""
)
con.execute(
u"""CREATE INDEX IF NOT EXISTS IdxNodeGraphNodeid
ON NodeGraph (nodeid);"""
)
con.execute(
u"""CREATE INDEX IF NOT EXISTS IdxNodeGraphParentid
ON NodeGraph (parentid);"""
)
# init attribute indexes
self.init_attrs(self.cur)
con.commit()
# check whether index is uptodate
# if not self._need_index:
# self._need_index = self.check_index()
except sqlite.DatabaseError, e:
self._on_corrupt(e, sys.exc_info()[2])
keepnote.log_message("reinitializing index '%s'\n" % self._index_file)
self.clear()
def _move_to_lostdir(self, filename):
"""Moves a file/dir to the lost_found directory"""
lostdir = self._get_lostdir()
if not os.path.exists(lostdir):
os.makedirs(lostdir)
new_filename = keepnote.notebook.get_unique_filename(
lostdir, os.path.basename(filename), sep=u"-")
keepnote.log_message(u"moving data to lostdir '%s' => '%s'\n" %
(filename, new_filename))
try:
os.rename(filename, new_filename)
except OSError, e:
raise ConnectionError(u"unable to store lost file '%s'"
% filename, e)
def _move_to_lostdir(self, filename):
"""Moves a file/dir to the lost_found directory"""
lostdir = self._get_lostdir()
if not os.path.exists(lostdir):
os.makedirs(lostdir)
new_filename = keepnote.notebook.get_unique_filename(
lostdir, os.path.basename(filename), sep=u"-")
keepnote.log_message(u"moving data to lostdir '%s' => '%s'\n" %
(filename, new_filename))
try:
os.rename(filename, new_filename)
except OSError, e:
raise ConnectionError(u"unable to store lost file '%s'" % filename,
e)
def init_index(self, auto_clear=True):
"""Initialize the tables in the index if they do not exist"""
con = self.con
try:
# check database version
version = self._get_version()
if version is None or version != INDEX_VERSION:
# version does not match, drop all tables
self._drop_tables()
# update version
self._set_version()
self._need_index = True
# init NodeGraph table
con.execute(u"""CREATE TABLE IF NOT EXISTS NodeGraph
(nodeid TEXT,
parentid TEXT,
basename TEXT,
mtime FLOAT,
symlink BOOLEAN,
UNIQUE(nodeid) ON CONFLICT REPLACE);
""")
con.execute(u"""CREATE INDEX IF NOT EXISTS IdxNodeGraphNodeid
ON NodeGraph (nodeid);""")
con.execute(u"""CREATE INDEX IF NOT EXISTS IdxNodeGraphParentid
ON NodeGraph (parentid);""")
# init attribute indexes
self.init_attrs(self.cur)
con.commit()
# check whether index is uptodate
#if not self._need_index:
# self._need_index = self.check_index()
except sqlite.DatabaseError as e:
self._on_corrupt(e, sys.exc_info()[2])
keepnote.log_message("reinitializing index '%s'\n" %
self._index_file)
self.clear()
def install_extension(self, filename):
"""Install a new extension from package 'filename'"""
userdir = get_user_extensions_dir()
newfiles = []
try:
# unzip and record new files
for fn in unzip(filename, userdir):
newfiles.append(fn)
# rescan user extensions
exts = set(self._extensions.keys())
self._scan_extension_path(userdir, "user")
# find new extensions
new_names = set(self._extensions.keys()) - exts
new_exts = [self.get_extension(name) for name in new_names]
except Exception as e:
self.error(_("Unable to install extension '%s'") % filename,
e,
tracebk=sys.exc_info()[2])
# delete newfiles
for fn in newfiles:
try:
keepnote.log_message(_("removing '%s'") % newfile)
os.remove(newfile)
except:
# delete may fail, continue
pass
return []
# enable new extensions
log_message(_("Enabling new extensions:\n"))
for ext in new_exts:
log_message(_("enabling extension '%s'\n") % ext.key)
ext.enable(True)
return new_exts
def _reindex_node(self, nodeid, parentid, path, attr, mtime, warn=True):
"""Reindex a node that has been tampered"""
if warn:
keepnote.log_message(
u"Unmanaged change detected. Reindexing '%s'\n" % path)
# TODO: to prevent a full recurse I could index children but
# use 0 for mtime, so that they will still trigger an index for them
# selves
# reindex all children in case their parentid's changed
#for path2 in iter_child_node_paths(path):
# attr2 = self._read_node(nodeid, path2, _full=False,
# _force_index=True)
#self._index.add_node(
# attr2["nodeid"], nodeid,
# os.path.basename(path2), attr2,
# get_path_mtime(path2))
# reindex this node
self._index.add_node(nodeid, parentid, os.path.basename(path), attr,
mtime)
def on_upgrade_notebook(self, app, args):
version = keepnote.notebook.NOTEBOOK_FORMAT_VERSION
i = 1
while i < len(args):
if args[i] == "v":
try:
version = int(args[i+1])
i += 2
except:
raise Exception("excepted version number")
else:
break
files = args[i:]
for filename in files:
keepnote.log_message("upgrading notebook to version %d: %s\n" %
(version, filename))
keepnote.notebook.update.update_notebook(filename, version,
verify=True)
def _reindex_node(self, nodeid, parentid, path, attr, mtime, warn=True):
"""Reindex a node that has been tampered"""
if warn:
keepnote.log_message(
u"Unmanaged change detected. Reindexing '%s'\n" % path)
# TODO: to prevent a full recurse I could index children but
# use 0 for mtime, so that they will still trigger an index for them
# selves
# reindex all children in case their parentid's changed
#for path2 in iter_child_node_paths(path):
# attr2 = self._read_node(nodeid, path2, _full=False,
# _force_index=True)
#self._index.add_node(
# attr2["nodeid"], nodeid,
# os.path.basename(path2), attr2,
# get_path_mtime(path2))
# reindex this node
self._index.add_node(
nodeid, parentid, os.path.basename(path), attr, mtime)
def on_upgrade_notebook(self, app, args):
version = keepnote.notebook.NOTEBOOK_FORMAT_VERSION
i = 1
while i < len(args):
if args[i] == "v":
try:
version = int(args[i+1])
i += 2
except:
raise Exception("excepted version number")
else:
break
files = args[i:]
for filename in files:
keepnote.log_message("upgrading notebook to version %d: %s\n" %
(version, filename))
keepnote.notebook.update.update_notebook(filename, version,
verify=True)
def search_node_contents_manual(self, cur, words):
"""Recursively search nodes under node for occurrence of words"""
keepnote.log_message("manual search\n")
nodeid = self._nconn.get_rootid()
stack = [nodeid]
while len(stack) > 0:
nodeid = stack.pop()
title = self._nconn.read_node(nodeid).get("title", "").lower()
infile = chain([title],
read_data_as_plain_text(self._nconn, nodeid))
if match_words(infile, words):
yield nodeid
else:
# return frequently so that search does not block long
yield None
children = self._nconn._list_children_nodeids(nodeid)
stack.extend(children)
def search_node_contents_manual(self, cur, words):
"""Recursively search nodes under node for occurrence of words"""
keepnote.log_message("manual search\n")
nodeid = self._nconn.get_rootid()
stack = [nodeid]
while len(stack) > 0:
nodeid = stack.pop()
title = self._nconn.read_node(nodeid).get("title", "").lower()
infile = chain([title],
read_data_as_plain_text(self._nconn, nodeid))
if match_words(infile, words):
yield nodeid
else:
# return frequently so that search does not block long
yield None
children = self._nconn._list_children_nodeids(nodeid)
stack.extend(children)
def start_http(self, app, args):
port = int(args[1])
notebook_path = unicode(args[2])
# connect to notebook on disk
conn = NoteBookConnectionFS()
conn.connect(notebook_path)
# start server in another thread
host = "localhost"
url = "http://%s:%d/" % (host, port)
server = NoteBookHttpServer(conn, host="localhost", port=port)
if port in self._ports:
raise Exception("Server already on port %d" % port)
self._ports[port] = server
keepnote.log_message("starting server:\n%s\n" % url)
thread.start_new_thread(server.serve_forever, ())
if host == "localhost":
keepnote.log_message("NOTE: server is local only. Use ssh port forwarding for security.\n")
def error(self, text, error=None, tracebk=None):
"""Display an error message"""
keepnote.log_message(text)
if error is not None:
keepnote.log_error(error, tracebk)
def generate_catalog_folders(node, filename, task=None):
"""
Import a folder tree as a subfolder of the current item
node -- node to attach folder to
filename -- filename of folder to import
task -- Task object to track progress
"""
# TODO: Exceptions, intelligent error handling
# For windows:
# Deep paths are handled by unicode "\\?\" extension to filename.
if task is None:
# create dummy task if needed
task = tasklib.Task()
# Determine number of files in advance so we can have a progress bar
nfiles = 0
for root, dirs, files in os.walk(filename):
nfiles += len(files) # Add files found in current dir
task.set_message(("text", "Found %i files..." % nfiles))
# Make a node based on the root - so we have an origin to import to
rootnode = node.new_child(CONTENT_TYPE_PAGE, os.path.basename(filename))
rootnode.set_attr("title", os.path.basename(filename))
filename2node = {filename: rootnode}
nfilescomplete = 0 # updates progress bar
# Walk directory we're importing and create nodes
for root, dirs, files in os.walk(filename):
# create node for directory
if root == filename:
parent = rootnode
else:
parent2 = filename2node.get(os.path.dirname(root), None)
if parent2 is None:
keepnote.log_message("parent node not found '%s'.\n" % root)
continue
parent = parent2.new_child(CONTENT_TYPE_PAGE,
os.path.basename(root))
parent.set_attr("title", os.path.basename(root))
filename2node[root] = parent
# create nodes for files
fileList = "" ;
for shortName in files:
#if keepnote.get_platform() is "windows":
# fn = "\\\\?\\" + os.path.join(root, fn)
#else:
# fn = os.path.join(root, fn)
#child = attach_file(fn, parent)
fn = os.path.join(root,shortName)
fileSize = os.stat(fn).st_size
#fileTime = time.asctime(time.localtime(os.stat(fn).st_mtime))
ft = time.localtime(os.stat(fn).st_mtime)
fileLine = '<a href="%s">%s</a> %s %02d-%02d-%02d %02d:%02d:%02d ' % (fn,shortName,formatFileSize(fileSize),ft.tm_year,ft.tm_mon,ft.tm_mday,ft.tm_hour,ft.tm_min,ft.tm_sec)
fileList += fileLine + "\n" # Will be converted to <br> when page inserted
nfilescomplete += 1
task.set_message(("text", "Imported %i / %i files..." %
(nfilescomplete, nfiles)))
task.set_percent(float(nfilescomplete) / float(nfiles))
make_catalog_page(parent,fileList,task)
task.finish()
# """)
# initialize attribute tables
for attr in self._attrs.itervalues():
attr.init(self.cur)
con.commit()
# check whether index is uptodate
#if not self._need_index:
# self._need_index = self.check_index()
except sqlite.DatabaseError, e:
self._on_corrupt(e, sys.exc_info()[2])
keepnote.log_message("reinitializing index '%s'\n" %
self._index_file)
self.clear()
def is_corrupt(self):
"""Return True if database appear corrupt"""
return self._corrupt
def check_index(self):
"""Check filesystem to see if index is up to date"""
keepnote.log_message("checking index... ")
start = time.time()
mtime_index = self.get_mtime()
mtime = keepnote.notebook.connection.fs.last_node_change(
self._nconn._get_node_path(self._nconn.get_rootid()))
keepnote.log_message("%f seconds\n" % (time.time() - start))
def generate_catalog_folders(node, filename, task=None):
"""
Import a folder tree as a subfolder of the current item
node -- node to attach folder to
filename -- filename of folder to import
task -- Task object to track progress
"""
# TODO: Exceptions, intelligent error handling
# For windows:
# Deep paths are handled by unicode "\\?\" extension to filename.
if task is None:
# create dummy task if needed
task = tasklib.Task()
# Determine number of files in advance so we can have a progress bar
nfiles = 0
for root, dirs, files in os.walk(filename):
nfiles += len(files) # Add files found in current dir
task.set_message(("text", "Found %i files..." % nfiles))
# Make a node based on the root - so we have an origin to import to
rootnode = node.new_child(CONTENT_TYPE_PAGE, os.path.basename(filename))
rootnode.set_attr("title", os.path.basename(filename))
filename2node = {filename: rootnode}
nfilescomplete = 0 # updates progress bar
# Walk directory we're importing and create nodes
for root, dirs, files in os.walk(filename):
# create node for directory
if root == filename:
parent = rootnode
else:
parent2 = filename2node.get(os.path.dirname(root), None)
if parent2 is None:
keepnote.log_message("parent node not found '%s'.\n" % root)
continue
parent = parent2.new_child(CONTENT_TYPE_PAGE,
os.path.basename(root))
parent.set_attr("title", os.path.basename(root))
filename2node[root] = parent
# create nodes for files
fileList = "" ;
for shortName in files:
#if keepnote.get_platform() is "windows":
# fn = "\\\\?\\" + os.path.join(root, fn)
#else:
# fn = os.path.join(root, fn)
#child = attach_file(fn, parent)
fn = os.path.join(root,shortName)
fileSize = os.stat(fn).st_size
#fileTime = time.asctime(time.localtime(os.stat(fn).st_mtime))
ft = time.localtime(os.stat(fn).st_mtime)
fileLine = '<a href="%s">%s</a> %s %02d-%02d-%02d %02d:%02d:%02d ' % (fn,shortName,formatFileSize(fileSize),ft.tm_year,ft.tm_mon,ft.tm_mday,ft.tm_hour,ft.tm_min,ft.tm_sec)
fileList += fileLine + "\n" # Will be converted to <br> when page inserted
nfilescomplete += 1
task.set_message(("text", "Imported %i / %i files..." %
(nfilescomplete, nfiles)))
task.set_percent(float(nfilescomplete) / float(nfiles))
make_catalog_page(parent,fileList,task)
task.finish()
def init_index(self, auto_clear=True):
"""Initialize the tables in the index if they do not exist"""
con = self.con
try:
# check database version
version = self._get_version()
if version is None or version != INDEX_VERSION:
# version does not match, drop all tables
self._drop_tables()
# update version
self._set_version()
self._need_index = True
# init NodeGraph table
con.execute(u"""CREATE TABLE IF NOT EXISTS NodeGraph
(nodeid TEXT,
parentid TEXT,
basename TEXT,
mtime FLOAT,
symlink BOOLEAN,
UNIQUE(nodeid) ON CONFLICT REPLACE);
""")
con.execute(u"""CREATE INDEX IF NOT EXISTS IdxNodeGraphNodeid
ON NodeGraph (nodeid);""")
con.execute(u"""CREATE INDEX IF NOT EXISTS IdxNodeGraphParentid
ON NodeGraph (parentid);""")
# full text table
try:
# test for fts3 availability
con.execute(u"DROP TABLE IF EXISTS fts3test;")
con.execute(
"CREATE VIRTUAL TABLE fts3test USING fts3(col TEXT);")
con.execute("DROP TABLE fts3test;")
# create fulltext table if it does not already exist
if not list(
con.execute(u"""SELECT 1 FROM sqlite_master
WHERE name == 'fulltext';""")):
con.execute(u"""CREATE VIRTUAL TABLE
fulltext USING
fts3(nodeid TEXT, content TEXT,
tokenize=porter);""")
self._has_fulltext = True
except Exception as e:
keepnote.log_error(e)
self._has_fulltext = False
# TODO: make an Attr table
# this will let me query whether an attribute is currently being
# indexed and in what table it is in.
#con.execute(u"""CREATE TABLE IF NOT EXISTS AttrDefs
# (attr TEXT,
# type );
# """)
# initialize attribute tables
for attr in self._attrs.itervalues():
attr.init(self.cur)
con.commit()
# check whether index is uptodate
#if not self._need_index:
# self._need_index = self.check_index()
except sqlite.DatabaseError as e:
self._on_corrupt(e, sys.exc_info()[2])
keepnote.log_message("reinitializing index '%s'\n" %
self._index_file)
self.clear()
def error(self, text, error=None, tracebk=None):
"""Display an error message"""
keepnote.log_message(text)
if error is not None:
keepnote.log_error(error, tracebk)
# """)
# initialize attribute tables
for attr in self._attrs.itervalues():
attr.init(self.cur)
con.commit()
# check whether index is uptodate
#if not self._need_index:
# self._need_index = self.check_index()
except sqlite.DatabaseError, e:
self._on_corrupt(e, sys.exc_info()[2])
keepnote.log_message("reinitializing index '%s'\n" %
self._index_file)
self.clear()
def is_corrupt(self):
"""Return True if database appear corrupt"""
return self._corrupt
def check_index(self):
"""Check filesystem to see if index is up to date"""
keepnote.log_message("checking index... ")
start = time.time()
mtime_index = self.get_mtime()
mtime = keepnote.notebook.connection.fs.last_node_change(
