def loadBranches(self, branches): """Bulk addition of branches. 'branches' is a list of (parent_oid, oid) pairs that will be added to the tree. If parent_oid is 'ROOT' the tree will be used as parent. """ start = time.time() created_branches = [] for parent_oid, oid in branches: if self.branchExists(oid): raise errors.SiptrackError( 'a branch with oid %s already exists' % (oid)) branch = Branch(self, parent_oid, oid) self.addedBranch(oid, branch) created_branches.append(branch) for branch in self.oid_mapping.itervalues(): if type(branch.parent) in [str, unicode]: if branch.parent == 'ROOT': parent = self else: parent = self.oid_mapping.get(branch.parent) if not parent: log.msg('WARNING: unable to locate parent %s for oid %s' % (branch.parent, branch.oid)) # raise errors.SiptrackError( # 'unable to locate parent %s for oid %s' % (branch.parent, branch.oid)) else: branch.parent = parent parent.branches.append(branch)
def getText(self, pk_password, user): text = self._text.get() pk = self.password_key if pk: try: if pk.canEncryptDecrypt(password=pk_password, user=user): text = pk.decrypt(text, self.lock_data, pk_password, user) else: text = '' except errors.SiptrackError as e: # Decryption failed text = '' if self.unicode and type(text) != unicode: try: text = text.decode('utf-8') except UnicodeDecodeError: text = '' except Exception as e: text = '' tbmsg = traceback.format_exc() log.msg(tbmsg) return text
def getText(self, pk_password, user): text = self._text.get() pk = self.password_key if pk: try: if pk.canEncryptDecrypt(password=pk_password, user=user): text = pk.decrypt( text, self.lock_data, pk_password, user ) else: text = '' except errors.SiptrackError as e: # Decryption failed text = '' if self.unicode and type(text) != unicode: try: text = text.decode('utf-8') except UnicodeDecodeError: text = '' except Exception as e: text = '' tbmsg = traceback.format_exc() log.msg(tbmsg) return text
def runPendingSubKeys(self, password): updated = [] for psk in list(self.listChildren(include = ['pending subkey'])): updated.append(psk) # Don't pass on errors if this fails, it would abort the login. try: updated += psk.connectPasswordKey(self, password, remove_self = False) except Exception, e: log.msg('WARNING: error connecting pending subkey: %s' % (str(e))) finally:
def quicksearch(self, search_pattern, attr_limit=[], include=[], exclude=[], user=None, fuzzy=True, default_fields=[], max_results=None): """Quick search for text strings. Uses the searcher interface from search.py. search_pattern : text pattern to search for. include : include only node types listed exclude : exclude node types listed """ if not self.searcher: raise errors.SiptrackError( 'no searcher selected, quicksearch unavailable') returned = {} count = 0 for oid in self.searcher.search(search_pattern, fuzzy, default_fields, max_results=None): try: node = self.getOID(oid) except errors.NonExistent: log.msg( 'quicksearch matched non-existent oid, something is wrong: %s' % (oid)) continue local_types = [ 'attribute', 'versioned attribute', 'encrypted attribute' ] if node.class_name in local_types: if len(attr_limit) > 0 and node.name not in attr_limit: continue # Get the attributes nearest _non-attribute_ parent. node = node.getParentNode() if not node.hasReadPermission(user): continue if node.oid in returned: continue if len(include) > 0 and node.class_name not in include: continue if node.class_name in exclude: continue returned[node.oid] = True if max_results and count >= max_results: break count += 1 yield node
def get_searcher(name = None, *args, **kwargs): if name == 'memory': searcher = MemorySearch(*args, **kwargs) elif name == 'whoosh': if not _have_whoosh: raise errors.SiptrackError('Sorry, whoosh search is unavailable.') searcher = WhooshSearch(*args, **kwargs) else: raise errors.SiptrackError('Unknown searcher "%s"' % (name)) log.msg('Using %s searcher' % (name)) return searcher
def _buildIndex(self, object_store): if self._using_existing_index: return log.msg('WhooshSearch building index, hang on.') self._indexed = True attr_types = ['attribute', 'versioned attribute'] writer = self.ix.writer() for node in object_store.view_tree.traverse(exclude = attr_types): self._setNode(node, writer) writer.commit() log.msg('WhooshSearch index building complete.')
def runPendingSubKeys(self, password): updated = [] for psk in list(self.listChildren(include=['pending subkey'])): updated.append(psk) # Don't pass on errors if this fails, it would abort the login. try: updated += psk.connectPasswordKey(self, password, remove_self=False) except Exception, e: log.msg('WARNING: error connecting pending subkey: %s' % (str(e))) finally:
def run(self, event_type, event_trigger, *args, **kwargs): if event_type in ['node add', 'node remove', 'node update', 'node relocate']: node = args[0] elif event_type in ['node associate', 'node disassociate']: node1, node2 = args try: exec self._code.get() if self._error_timestamp.get() != 0: self._error_timestamp.set(0) self._error.set('') except Exception, e: log.msg('Exception caught when running %s: %s' % (self, str(e))) self._error_timestamp.set(int(time.time())) self._error.set(str(e))
def triggerEvent(self, event_type, *event_args, **event_kwargs): # Don't trigger any more events while in an event trigger, # otherwise we could end up in a nasty loop. # This means that no triggers will ever be run from what # happens inside a trigger (node creation etc). if not self.event_triggers_enabled: return self.event_triggers_enabled = False # Make sure trigger failures don't affect anything else. try: for event_trigger in self.event_triggers: event_trigger.triggerEvent(event_type, *event_args, **event_kwargs) except Exception, e: log.msg('Trigger %s raised unhandled exception: %s' % (event_trigger, str(e)))
def hasWritePermission(self, user, recurse=True): if not user or user.administrator: return True if self.require_admin and not user.administrator: return False # Users have access to their own users. if self == user.user: return True perm = self.getPermission(user, recurse) if perm and perm.write_access.get(): return True log.msg('FAIL: write permission failed for node:%s user:%s' % (self, user)) return False
def loadAssociations(self, associations): """Bulk addition of associations. 'associations' is a list of (oid, associated_oid) pairs. """ for oid, associated_oid in associations: branch = self.getBranch(oid) if not branch: log.msg('WARNING: unable to find oid %s (-> %s) when loading associations' % (oid, associated_oid)) continue associated_branch = self.getBranch(associated_oid) if not associated_branch: log.msg('WARNING: unable to find associated oid %s (-> %s) when loading associations' % (associated_oid, oid)) continue branch.associate(associated_branch)
def preLoad(self): """Preload node data. The following happens: * All node data is grabbed from storage. * The object tree is walked grabbing ext_data from each bran which causes the node to load (but it's _loaded method is node called due to ObjectStore.call_loaded not being set. * The nodes data is passed to it via it's _loaded method. The node is free to do whatever it likes with the data, several node types currently don't have a _loaded method despite the fact that they have storage data. They should really be added. If no data exists for the oid an empty dict is passed in. The layout of the data passed to the node is a dict of one or more of: data['storage_data_name] = (data_type, data) """ print 'Loading OID data' data_mapping = yield self.storage.makeOIDData() print 'OID data loaded' self.call_loaded = False try: for branch in self.object_tree.traverse(): # if branch.hasExtData(): # continue node = branch.ext_data if not node: log.msg( 'ObjectStore.preLoad branch %s returned no ext_data node, skipping' % (branch)) continue data = {} if branch.oid in data_mapping: data = data_mapping[branch.oid] # Calling _loaded must be done with call_loaded enabled. # This is due to the possibility that a nodes _loaded # will use getOID to fetch another node which might not # have been loaded yet. If that happens with call_loaded = # False the node will be loaded without ever having # _loaded called, which might be very bad. self.call_loaded = True node._loaded(data) self.call_loaded = False finally: self.call_loaded = True defer.returnValue(True)
def _remove(self, user): """Remove a single object. Called from branch callbacks.""" if not self.hasWritePermission(user): raise errors.PermissionDenied() self.object_store.triggerEvent('node remove', self) log.msg('Removed node %s' % (self)) for reference in list(self.references): reference.disassociate(self) for assoc in list(self.associations): self.disassociate(assoc) self.branch = None # self.oid = None self.removed = True self.setModified() self.storageAction('remove_node') self.searcherAction('remove_node') self.removeStorageAction('create_node')
def preLoad(self): """Preload node data. The following happens: * All node data is grabbed from storage. * The object tree is walked grabbing ext_data from each bran which causes the node to load (but it's _loaded method is node called due to ObjectStore.call_loaded not being set. * The nodes data is passed to it via it's _loaded method. The node is free to do whatever it likes with the data, several node types currently don't have a _loaded method despite the fact that they have storage data. They should really be added. If no data exists for the oid an empty dict is passed in. The layout of the data passed to the node is a dict of one or more of: data['storage_data_name] = (data_type, data) """ print 'Loading OID data' data_mapping = yield self.storage.makeOIDData() print 'OID data loaded' self.call_loaded = False try: for branch in self.object_tree.traverse(): # if branch.hasExtData(): # continue node = branch.ext_data if not node: log.msg('ObjectStore.preLoad branch %s returned no ext_data node, skipping' % (branch)) continue data = {} if branch.oid in data_mapping: data = data_mapping[branch.oid] # Calling _loaded must be done with call_loaded enabled. # This is due to the possibility that a nodes _loaded # will use getOID to fetch another node which might not # have been loaded yet. If that happens with call_loaded = # False the node will be loaded without ever having # _loaded called, which might be very bad. self.call_loaded = True node._loaded(data) self.call_loaded = False finally: self.call_loaded = True defer.returnValue(True)
def quicksearch(self, search_pattern, attr_limit = [], include = [], exclude = [], user = None, fuzzy = True, default_fields = [], max_results = None): """Quick search for text strings. Uses the searcher interface from search.py. search_pattern : text pattern to search for. include : include only node types listed exclude : exclude node types listed """ if not self.searcher: raise errors.SiptrackError('no searcher selected, quicksearch unavailable') returned = {} count = 0 for oid in self.searcher.search(search_pattern, fuzzy, default_fields, max_results=None): try: node = self.getOID(oid) except errors.NonExistent: log.msg('quicksearch matched non-existent oid, something is wrong: %s' % (oid)) continue local_types = [ 'attribute', 'versioned attribute', 'encrypted attribute' ] if node.class_name in local_types: if len(attr_limit) > 0 and node.name not in attr_limit: continue # Get the attributes nearest _non-attribute_ parent. node = node.getParentNode() if not node.hasReadPermission(user): continue if node.oid in returned: continue if len(include) > 0 and node.class_name not in include: continue if node.class_name in exclude: continue returned[node.oid] = True if max_results and count >= max_results: break count += 1 yield node
def loadAssociations(self, associations): """Bulk addition of associations. 'associations' is a list of (oid, associated_oid) pairs. """ for oid, associated_oid in associations: branch = self.getBranch(oid) if not branch: log.msg( 'WARNING: unable to find oid %s (-> %s) when loading associations' % (oid, associated_oid)) continue associated_branch = self.getBranch(associated_oid) if not associated_branch: log.msg( 'WARNING: unable to find associated oid %s (-> %s) when loading associations' % (associated_oid, oid)) continue branch.associate(associated_branch)
def search(self, queries, fuzzy = True, default_fields = [], max_results = None): if type(queries) != list: queries = [queries] if type(default_fields) != list: default_fields = [default_fields] if fuzzy and len(queries) == 1 and len(queries[0].split()) == 1 and ':' not in queries[0] and '*' not in queries[0]: queries = ['*%s*' % (queries[0])] for query in queries: if type(query) != unicode: query = query.decode('utf-8') log.msg('search query: %s' % (query)) with self.ix.searcher() as searcher: parser = MultifieldParser(default_fields, self.ix.schema) parser.remove_plugin_class(plugins.WildcardPlugin) parser.add_plugin(WildcardPlugin) query = parser.parse(query) log.msg('search query parsed: %s' % (query)) results = searcher.search(query, limit = None) count = 0 for result in results: yield result['oid'] count += 1 if max_results and count >= max_results: break
class BaseNode(object): """Base class for all objects in the tree. This class is inherited by all regular tree objects, views, containers etc. """ require_admin = False def __init__(self, oid, branch): self.oid = oid self.branch = branch # Contains a list of all StorageValue instances used by the node. # Updated by StorageValue.__init__. self._storage_actions = [] self._searcher_actions = [] self.object_store = self.branch.tree.ext_data self.searcher = self.object_store.searcher self.removed = False # Creation time. self.ctime = storagevalue.StorageValue(self, 'ctime', 0) # Modification time, for internal use. self.modtime = time.time() global perm_cache self.perm_cache = perm_cache def __str__(self): return '<%s:%s>' % (self.class_name, self.oid) def commit(self): return self.object_store.commit(self) def _treeFree(self): """Free a BaseNode. This will free any memory references used by the node. The node will be unusable after this is called. Called by the object tree branch when it's freed. """ # self.oid = None self.branch = None self.object_store = None self.searcher = None self.perm_cache = None self._storage_actions = None def storageAction(self, action, args=None): self._storage_actions.append({'action': action, 'args': args}) def searcherAction(self, action, args=None): self._searcher_actions.append({'action': action, 'args': args}) def removeStorageAction(self, rm_action): if not self._storage_actions: return rm = [] for action in self._storage_actions: if action['action'] == rm_action: rm.append(action) for action in rm: self._storage_actions.remove(action) def addChildByID(self, user, class_id, *args, **kwargs): """Create a new child of type class_id. Checks the object registry that class_id is a valid child of the current object. Also creates a new branch in the object tree to hold the object. """ if not object_registry.isValidChild(self.class_id, class_id): raise errors.SiptrackError( 'trying to create child of invalid type \'%s\'' % (class_id)) if not self.hasWritePermission(user): raise errors.PermissionDenied() child = object_registry.createObject(class_id, self.branch, *args, **kwargs) try: # Called only for newly created objects. child._created(user) except Exception, e: child.remove(recursive=False) raise log.msg('Added node %s' % (child)) self.branch.tree.ext_data.oid_class_mapping[child.oid] = class_id return child
def logPermissionCache(self, user=None): for perm in self.perm_cache.get(self): log.msg(str(perm)) if user and perm.matchesUser(user): log.msg('matches user %s: true' % (user))
class Password(treenodes.BaseNode): """Store passwords. Used to store passwords, if an optional key is given (as a PasswordKey object) the password will be encrypted on disk and in memory and only accessible if the PasswordKey is unlocked. """ class_id = 'P' class_name = 'password' def __init__(self, oid, branch, password=None, key=None, unicode=True, pk_password=None): """Init. password is the plaintext password. key is an optional password key. unicode determines whether the password should go through character conversion (if it's a unicode password) """ super(Password, self).__init__(oid, branch) if type(password) in [str, unicode] and unicode: password = password.encode('utf-8') self._password = storagevalue.StorageValue(self, 'password', password) self._password_key = storagevalue.StorageNode(self, 'key', key) self._lock_data = storagevalue.StorageValue(self, 'password-lockdata') self.unicode = unicode self._pk_password = pk_password def _created(self, user): """Store the password and extra data.""" super(Password, self)._created(user) if type(self._password.get()) not in [str, unicode]: raise errors.SiptrackError('invalid password in password object') self._password_key.commit() pk = self.password_key if pk: if not pk.canEncryptDecrypt(self._pk_password, user): raise errors.SiptrackError('unable to access password key') password, self.lock_data = pk.encrypt(self._password.get(), self._pk_password, user) self._password.set(password) else: self._password.commit() self._pk_password = None def _loaded(self, data=None): """Load the password from disk. If the password has been stored encrypted it will not be decrypted here. """ super(Password, self)._loaded(data) self._password_key.preload(data) self._lock_data.preload(data) self._password.preload(data) self._pk_password = None def getPassword(self, pk_password, user): """Returns the password. If it has been encrypted with a PasswordKey decrypt it first. """ password = self._password.get() pk = self.password_key if pk: try: if pk.canEncryptDecrypt(password=pk_password, user=user): password = self.password_key.decrypt( password, self.lock_data, pk_password, user) else: password = '' # Decryption failed. except errors.SiptrackError, e: password = '' if self.unicode and type(password) != unicode: try: password = password.decode('utf-8') except UnicodeDecodeError: password = '' except Exception, e: password = '' tbmsg = traceback.format_exc() log.msg(tbmsg)