def loadBranches(self, branches):
"""Bulk addition of branches.
'branches' is a list of (parent_oid, oid) pairs that will be
added to the tree. If parent_oid is 'ROOT' the tree will be used as
parent.
"""
start = time.time()
created_branches = []
for parent_oid, oid in branches:
if self.branchExists(oid):
raise errors.SiptrackError(
'a branch with oid %s already exists' % (oid))
branch = Branch(self, parent_oid, oid)
self.addedBranch(oid, branch)
created_branches.append(branch)
for branch in self.oid_mapping.itervalues():
if type(branch.parent) in [str, unicode]:
if branch.parent == 'ROOT':
parent = self
else:
parent = self.oid_mapping.get(branch.parent)
if not parent:
log.msg('WARNING: unable to locate parent %s for oid %s' %
(branch.parent, branch.oid))
# raise errors.SiptrackError(
# 'unable to locate parent %s for oid %s' % (branch.parent, branch.oid))
else:
branch.parent = parent
parent.branches.append(branch)
def getText(self, pk_password, user):
text = self._text.get()
pk = self.password_key
if pk:
try:
if pk.canEncryptDecrypt(password=pk_password, user=user):
text = pk.decrypt(text, self.lock_data, pk_password, user)
else:
text = ''
except errors.SiptrackError as e:
# Decryption failed
text = ''
if self.unicode and type(text) != unicode:
try:
text = text.decode('utf-8')
except UnicodeDecodeError:
text = ''
except Exception as e:
text = ''
tbmsg = traceback.format_exc()
log.msg(tbmsg)
return text
def getText(self, pk_password, user):
text = self._text.get()
pk = self.password_key
if pk:
try:
if pk.canEncryptDecrypt(password=pk_password, user=user):
text = pk.decrypt(
text,
self.lock_data,
pk_password,
user
)
else:
text = ''
except errors.SiptrackError as e:
# Decryption failed
text = ''
if self.unicode and type(text) != unicode:
try:
text = text.decode('utf-8')
except UnicodeDecodeError:
text = ''
except Exception as e:
text = ''
tbmsg = traceback.format_exc()
log.msg(tbmsg)
return text
def loadBranches(self, branches):
"""Bulk addition of branches.
'branches' is a list of (parent_oid, oid) pairs that will be
added to the tree. If parent_oid is 'ROOT' the tree will be used as
parent.
"""
start = time.time()
created_branches = []
for parent_oid, oid in branches:
if self.branchExists(oid):
raise errors.SiptrackError(
'a branch with oid %s already exists' % (oid))
branch = Branch(self, parent_oid, oid)
self.addedBranch(oid, branch)
created_branches.append(branch)
for branch in self.oid_mapping.itervalues():
if type(branch.parent) in [str, unicode]:
if branch.parent == 'ROOT':
parent = self
else:
parent = self.oid_mapping.get(branch.parent)
if not parent:
log.msg('WARNING: unable to locate parent %s for oid %s' % (branch.parent, branch.oid))
# raise errors.SiptrackError(
# 'unable to locate parent %s for oid %s' % (branch.parent, branch.oid))
else:
branch.parent = parent
parent.branches.append(branch)
def runPendingSubKeys(self, password):
updated = []
for psk in list(self.listChildren(include = ['pending subkey'])):
updated.append(psk)
# Don't pass on errors if this fails, it would abort the login.
try:
updated += psk.connectPasswordKey(self, password, remove_self = False)
except Exception, e:
log.msg('WARNING: error connecting pending subkey: %s' % (str(e)))
finally:
def quicksearch(self,
search_pattern,
attr_limit=[],
include=[],
exclude=[],
user=None,
fuzzy=True,
default_fields=[],
max_results=None):
"""Quick search for text strings.
Uses the searcher interface from search.py.
search_pattern : text pattern to search for.
include : include only node types listed
exclude : exclude node types listed
"""
if not self.searcher:
raise errors.SiptrackError(
'no searcher selected, quicksearch unavailable')
returned = {}
count = 0
for oid in self.searcher.search(search_pattern,
fuzzy,
default_fields,
max_results=None):
try:
node = self.getOID(oid)
except errors.NonExistent:
log.msg(
'quicksearch matched non-existent oid, something is wrong: %s'
% (oid))
continue
local_types = [
'attribute', 'versioned attribute', 'encrypted attribute'
]
if node.class_name in local_types:
if len(attr_limit) > 0 and node.name not in attr_limit:
continue
# Get the attributes nearest _non-attribute_ parent.
node = node.getParentNode()
if not node.hasReadPermission(user):
continue
if node.oid in returned:
continue
if len(include) > 0 and node.class_name not in include:
continue
if node.class_name in exclude:
continue
returned[node.oid] = True
if max_results and count >= max_results:
break
count += 1
yield node
def get_searcher(name = None, *args, **kwargs):
if name == 'memory':
searcher = MemorySearch(*args, **kwargs)
elif name == 'whoosh':
if not _have_whoosh:
raise errors.SiptrackError('Sorry, whoosh search is unavailable.')
searcher = WhooshSearch(*args, **kwargs)
else:
raise errors.SiptrackError('Unknown searcher "%s"' % (name))
log.msg('Using %s searcher' % (name))
return searcher
def _buildIndex(self, object_store):
if self._using_existing_index:
return
log.msg('WhooshSearch building index, hang on.')
self._indexed = True
attr_types = ['attribute', 'versioned attribute']
writer = self.ix.writer()
for node in object_store.view_tree.traverse(exclude = attr_types):
self._setNode(node, writer)
writer.commit()
log.msg('WhooshSearch index building complete.')
def runPendingSubKeys(self, password):
updated = []
for psk in list(self.listChildren(include=['pending subkey'])):
updated.append(psk)
# Don't pass on errors if this fails, it would abort the login.
try:
updated += psk.connectPasswordKey(self,
password,
remove_self=False)
except Exception, e:
log.msg('WARNING: error connecting pending subkey: %s' %
(str(e)))
finally:
def run(self, event_type, event_trigger, *args, **kwargs):
if event_type in ['node add', 'node remove', 'node update', 'node relocate']:
node = args[0]
elif event_type in ['node associate', 'node disassociate']:
node1, node2 = args
try:
exec self._code.get()
if self._error_timestamp.get() != 0:
self._error_timestamp.set(0)
self._error.set('')
except Exception, e:
log.msg('Exception caught when running %s: %s' % (self, str(e)))
self._error_timestamp.set(int(time.time()))
self._error.set(str(e))
def triggerEvent(self, event_type, *event_args, **event_kwargs):
# Don't trigger any more events while in an event trigger,
# otherwise we could end up in a nasty loop.
# This means that no triggers will ever be run from what
# happens inside a trigger (node creation etc).
if not self.event_triggers_enabled:
return
self.event_triggers_enabled = False
# Make sure trigger failures don't affect anything else.
try:
for event_trigger in self.event_triggers:
event_trigger.triggerEvent(event_type, *event_args, **event_kwargs)
except Exception, e:
log.msg('Trigger %s raised unhandled exception: %s' % (event_trigger, str(e)))
def hasWritePermission(self, user, recurse=True):
if not user or user.administrator:
return True
if self.require_admin and not user.administrator:
return False
# Users have access to their own users.
if self == user.user:
return True
perm = self.getPermission(user, recurse)
if perm and perm.write_access.get():
return True
log.msg('FAIL: write permission failed for node:%s user:%s' %
(self, user))
return False
def triggerEvent(self, event_type, *event_args, **event_kwargs):
# Don't trigger any more events while in an event trigger,
# otherwise we could end up in a nasty loop.
# This means that no triggers will ever be run from what
# happens inside a trigger (node creation etc).
if not self.event_triggers_enabled:
return
self.event_triggers_enabled = False
# Make sure trigger failures don't affect anything else.
try:
for event_trigger in self.event_triggers:
event_trigger.triggerEvent(event_type, *event_args,
**event_kwargs)
except Exception, e:
log.msg('Trigger %s raised unhandled exception: %s' %
(event_trigger, str(e)))
def loadAssociations(self, associations):
"""Bulk addition of associations.
'associations' is a list of (oid, associated_oid)
pairs.
"""
for oid, associated_oid in associations:
branch = self.getBranch(oid)
if not branch:
log.msg('WARNING: unable to find oid %s (-> %s) when loading associations' % (oid, associated_oid))
continue
associated_branch = self.getBranch(associated_oid)
if not associated_branch:
log.msg('WARNING: unable to find associated oid %s (-> %s) when loading associations' % (associated_oid, oid))
continue
branch.associate(associated_branch)
def preLoad(self):
"""Preload node data.
The following happens:
* All node data is grabbed from storage.
* The object tree is walked grabbing ext_data from each bran
which causes the node to load (but it's _loaded method is
node called due to ObjectStore.call_loaded not being set.
* The nodes data is passed to it via it's _loaded method.
The node is free to do whatever it likes with the data,
several node types currently don't have a _loaded method
despite the fact that they have storage data. They should
really be added.
If no data exists for the oid an empty dict is passed in.
The layout of the data passed to the node is a dict of one or
more of:
data['storage_data_name] = (data_type, data)
"""
print 'Loading OID data'
data_mapping = yield self.storage.makeOIDData()
print 'OID data loaded'
self.call_loaded = False
try:
for branch in self.object_tree.traverse():
# if branch.hasExtData():
# continue
node = branch.ext_data
if not node:
log.msg(
'ObjectStore.preLoad branch %s returned no ext_data node, skipping'
% (branch))
continue
data = {}
if branch.oid in data_mapping:
data = data_mapping[branch.oid]
# Calling _loaded must be done with call_loaded enabled.
# This is due to the possibility that a nodes _loaded
# will use getOID to fetch another node which might not
# have been loaded yet. If that happens with call_loaded =
# False the node will be loaded without ever having
# _loaded called, which might be very bad.
self.call_loaded = True
node._loaded(data)
self.call_loaded = False
finally:
self.call_loaded = True
defer.returnValue(True)
def _remove(self, user):
"""Remove a single object. Called from branch callbacks."""
if not self.hasWritePermission(user):
raise errors.PermissionDenied()
self.object_store.triggerEvent('node remove', self)
log.msg('Removed node %s' % (self))
for reference in list(self.references):
reference.disassociate(self)
for assoc in list(self.associations):
self.disassociate(assoc)
self.branch = None
# self.oid = None
self.removed = True
self.setModified()
self.storageAction('remove_node')
self.searcherAction('remove_node')
self.removeStorageAction('create_node')
def preLoad(self):
"""Preload node data.
The following happens:
* All node data is grabbed from storage.
* The object tree is walked grabbing ext_data from each bran
which causes the node to load (but it's _loaded method is
node called due to ObjectStore.call_loaded not being set.
* The nodes data is passed to it via it's _loaded method.
The node is free to do whatever it likes with the data,
several node types currently don't have a _loaded method
despite the fact that they have storage data. They should
really be added.
If no data exists for the oid an empty dict is passed in.
The layout of the data passed to the node is a dict of one or
more of:
data['storage_data_name] = (data_type, data)
"""
print 'Loading OID data'
data_mapping = yield self.storage.makeOIDData()
print 'OID data loaded'
self.call_loaded = False
try:
for branch in self.object_tree.traverse():
# if branch.hasExtData():
# continue
node = branch.ext_data
if not node:
log.msg('ObjectStore.preLoad branch %s returned no ext_data node, skipping' % (branch))
continue
data = {}
if branch.oid in data_mapping:
data = data_mapping[branch.oid]
# Calling _loaded must be done with call_loaded enabled.
# This is due to the possibility that a nodes _loaded
# will use getOID to fetch another node which might not
# have been loaded yet. If that happens with call_loaded =
# False the node will be loaded without ever having
# _loaded called, which might be very bad.
self.call_loaded = True
node._loaded(data)
self.call_loaded = False
finally:
self.call_loaded = True
defer.returnValue(True)
def quicksearch(self, search_pattern, attr_limit = [], include = [], exclude = [], user = None,
fuzzy = True, default_fields = [], max_results = None):
"""Quick search for text strings.
Uses the searcher interface from search.py.
search_pattern : text pattern to search for.
include : include only node types listed
exclude : exclude node types listed
"""
if not self.searcher:
raise errors.SiptrackError('no searcher selected, quicksearch unavailable')
returned = {}
count = 0
for oid in self.searcher.search(search_pattern, fuzzy, default_fields, max_results=None):
try:
node = self.getOID(oid)
except errors.NonExistent:
log.msg('quicksearch matched non-existent oid, something is wrong: %s' % (oid))
continue
local_types = [
'attribute',
'versioned attribute',
'encrypted attribute'
]
if node.class_name in local_types:
if len(attr_limit) > 0 and node.name not in attr_limit:
continue
# Get the attributes nearest _non-attribute_ parent.
node = node.getParentNode()
if not node.hasReadPermission(user):
continue
if node.oid in returned:
continue
if len(include) > 0 and node.class_name not in include:
continue
if node.class_name in exclude:
continue
returned[node.oid] = True
if max_results and count >= max_results:
break
count += 1
yield node
def loadAssociations(self, associations):
"""Bulk addition of associations.
'associations' is a list of (oid, associated_oid)
pairs.
"""
for oid, associated_oid in associations:
branch = self.getBranch(oid)
if not branch:
log.msg(
'WARNING: unable to find oid %s (-> %s) when loading associations'
% (oid, associated_oid))
continue
associated_branch = self.getBranch(associated_oid)
if not associated_branch:
log.msg(
'WARNING: unable to find associated oid %s (-> %s) when loading associations'
% (associated_oid, oid))
continue
branch.associate(associated_branch)
def search(self, queries, fuzzy = True, default_fields = [], max_results = None):
if type(queries) != list:
queries = [queries]
if type(default_fields) != list:
default_fields = [default_fields]
if fuzzy and len(queries) == 1 and len(queries[0].split()) == 1 and ':' not in queries[0] and '*' not in queries[0]:
queries = ['*%s*' % (queries[0])]
for query in queries:
if type(query) != unicode:
query = query.decode('utf-8')
log.msg('search query: %s' % (query))
with self.ix.searcher() as searcher:
parser = MultifieldParser(default_fields, self.ix.schema)
parser.remove_plugin_class(plugins.WildcardPlugin)
parser.add_plugin(WildcardPlugin)
query = parser.parse(query)
log.msg('search query parsed: %s' % (query))
results = searcher.search(query, limit = None)
count = 0
for result in results:
yield result['oid']
count += 1
if max_results and count >= max_results:
break
class BaseNode(object):
"""Base class for all objects in the tree.
This class is inherited by all regular tree objects, views,
containers etc.
"""
require_admin = False
def __init__(self, oid, branch):
self.oid = oid
self.branch = branch
# Contains a list of all StorageValue instances used by the node.
# Updated by StorageValue.__init__.
self._storage_actions = []
self._searcher_actions = []
self.object_store = self.branch.tree.ext_data
self.searcher = self.object_store.searcher
self.removed = False
# Creation time.
self.ctime = storagevalue.StorageValue(self, 'ctime', 0)
# Modification time, for internal use.
self.modtime = time.time()
global perm_cache
self.perm_cache = perm_cache
def __str__(self):
return '<%s:%s>' % (self.class_name, self.oid)
def commit(self):
return self.object_store.commit(self)
def _treeFree(self):
"""Free a BaseNode.
This will free any memory references used by the node.
The node will be unusable after this is called.
Called by the object tree branch when it's freed.
"""
# self.oid = None
self.branch = None
self.object_store = None
self.searcher = None
self.perm_cache = None
self._storage_actions = None
def storageAction(self, action, args=None):
self._storage_actions.append({'action': action, 'args': args})
def searcherAction(self, action, args=None):
self._searcher_actions.append({'action': action, 'args': args})
def removeStorageAction(self, rm_action):
if not self._storage_actions:
return
rm = []
for action in self._storage_actions:
if action['action'] == rm_action:
rm.append(action)
for action in rm:
self._storage_actions.remove(action)
def addChildByID(self, user, class_id, *args, **kwargs):
"""Create a new child of type class_id.
Checks the object registry that class_id is a valid child of
the current object. Also creates a new branch in the object tree
to hold the object.
"""
if not object_registry.isValidChild(self.class_id, class_id):
raise errors.SiptrackError(
'trying to create child of invalid type \'%s\'' % (class_id))
if not self.hasWritePermission(user):
raise errors.PermissionDenied()
child = object_registry.createObject(class_id, self.branch, *args,
**kwargs)
try:
# Called only for newly created objects.
child._created(user)
except Exception, e:
child.remove(recursive=False)
raise
log.msg('Added node %s' % (child))
self.branch.tree.ext_data.oid_class_mapping[child.oid] = class_id
return child
def logPermissionCache(self, user=None):
for perm in self.perm_cache.get(self):
log.msg(str(perm))
if user and perm.matchesUser(user):
log.msg('matches user %s: true' % (user))
class Password(treenodes.BaseNode):
"""Store passwords.
Used to store passwords, if an optional key is given (as a PasswordKey
object) the password will be encrypted on disk and in memory and only
accessible if the PasswordKey is unlocked.
"""
class_id = 'P'
class_name = 'password'
def __init__(self,
oid,
branch,
password=None,
key=None,
unicode=True,
pk_password=None):
"""Init.
password is the plaintext password.
key is an optional password key.
unicode determines whether the password should go through
character conversion (if it's a unicode password)
"""
super(Password, self).__init__(oid, branch)
if type(password) in [str, unicode] and unicode:
password = password.encode('utf-8')
self._password = storagevalue.StorageValue(self, 'password', password)
self._password_key = storagevalue.StorageNode(self, 'key', key)
self._lock_data = storagevalue.StorageValue(self, 'password-lockdata')
self.unicode = unicode
self._pk_password = pk_password
def _created(self, user):
"""Store the password and extra data."""
super(Password, self)._created(user)
if type(self._password.get()) not in [str, unicode]:
raise errors.SiptrackError('invalid password in password object')
self._password_key.commit()
pk = self.password_key
if pk:
if not pk.canEncryptDecrypt(self._pk_password, user):
raise errors.SiptrackError('unable to access password key')
password, self.lock_data = pk.encrypt(self._password.get(),
self._pk_password, user)
self._password.set(password)
else:
self._password.commit()
self._pk_password = None
def _loaded(self, data=None):
"""Load the password from disk.
If the password has been stored encrypted it will not be decrypted
here.
"""
super(Password, self)._loaded(data)
self._password_key.preload(data)
self._lock_data.preload(data)
self._password.preload(data)
self._pk_password = None
def getPassword(self, pk_password, user):
"""Returns the password.
If it has been encrypted with a PasswordKey decrypt it first.
"""
password = self._password.get()
pk = self.password_key
if pk:
try:
if pk.canEncryptDecrypt(password=pk_password, user=user):
password = self.password_key.decrypt(
password, self.lock_data, pk_password, user)
else:
password = ''
# Decryption failed.
except errors.SiptrackError, e:
password = ''
if self.unicode and type(password) != unicode:
try:
password = password.decode('utf-8')
except UnicodeDecodeError:
password = ''
except Exception, e:
password = ''
tbmsg = traceback.format_exc()
log.msg(tbmsg)
