Esempio n. 1
0
 def _store(self, o, oname):
     ''' Store an OID as oidname in database but don't write rootoid yet. '''
     #print "Saving %s into %s" % (o, self._rootoid)
     # If using pdscache then o is actually a coid.
     if isinstance(o, pdscache._CachedOid):
         o = pdscache.write_coid(o)
     self._rootoid = self._ptrieObj.insert(self._rootoid, oname, o)
Esempio n. 2
0
 def _writeRootoid(self):
     ''' writes the root OID to file '''
     rootoid = self._rootoid
     # If PDSCache in use then write through the coid first
     if isinstance(rootoid, pdscache._CachedOid):
         rootoid = pdscache.write_coid(rootoid)
     rootoidPath = os.path.join(self._storpath, OidFS.rootoid_filename)
     fobj = open(rootoidPath, "w")
     cPickle.dump(rootoid, fobj, 2)
     fobj.close()
Esempio n. 3
0
 def gc(self):
     ''' Garbage collects OidFS's internal Ptrie PStor. Saving only
     self._rootoid. '''
     # Run GC on OID's pstor first.
     self._collect_pstor()
     # Save oidfs's _rootoid
     o = self._rootoid
     if isinstance(self._rootoid, pdscache._CachedOid):
         o = pdscache.write_coid(self._rootoid)
     o, = self._oidPstor.keepOids([o])
     o = pdscache.read_oid(o)
     self._rootoid = o
     self._writeRootoid()
Esempio n. 4
0
 def _collect_pstor(self):
     ''' Run GC on OID's pstructstor. OIDs stored in OidFS will be moved
     as a result. Note this function assumes that stored oids can belong
     to different PStors, which is currently allowed. In the future,
     PStor should probably take possession of OidFS so that there is only
     one PStor in a single OidFS. '''
     # Since PStor's GC function moves OIDs, we have to make a new OID
     # ptrie with the same oidname and new OID references.
     pstordict = {}
     for orec in self.oriter():
         oname, o = orec
         if isinstance(o, pdscache._CachedOid):
             # o is type 'coid' and o.pstor is a PStructStor object
             pstor = o.pstor
         else:
             # o is type 'OID' and o.pstor is a string
             pstor = pstructstor.PStructStor.mkpstor(o.pstor)
         if pstor not in pstordict:
             # pstor dictionary's value is a (onames, ovalues) pair
             pstordict[pstor] = [[], []]
         onames, ovalues = pstordict[pstor]
         onames.append(oname)
         if isinstance(o, pdscache._CachedOid):
             # Must convert back to real Oid
             o = pdscache.write_coid(o)
         ovalues.append(o)
     if not len(pstordict):
         return
     # Now send to pstructstor for GC and essentially re-create our
     # internal oid ptrie with new oid values
     for pstor in pstordict:
         onames, ovalues = pstordict[pstor]
         pstordict[pstor][1] = ovalues = pstor.keepOids(ovalues)
         for oname, o in zip(onames, ovalues):
             self._store(o, oname)
     self._writeRootoid()
Esempio n. 5
0
    def testloop(self):
        while True:
            try:
                inputtext = raw_input(">> ");
            except EOFError as e:
                print "Goodbye!"
                self.ofs.close()
                self.pstor.close()
                return
            args = inputtext.split()
            if len(args) == 0:
                cmd = "help"
            else:
                cmd = args.pop(0)

            if cmd == "help":
                print """Type a command. Commands are:
help quit read load find delete insert dfwalk bfwalk save ls gc inspect stats"""
            elif cmd == "quit":
                ans = raw_input("Save? (y/n)")
                if ans == "y":
                    self.ofs.store(self.root, "examplePtrie")
                    self.ofs.close()
                    self.pstor.close()
                    print "Ptrie saved as \"examplePtrie\""
                break
            elif cmd == "print":
                try:
                    print eval(args[0])
                except Exception as e:
                    print e
            elif cmd == "reinit":
                pdspath = args[0]
                if os.path.exists(pdspath) and os.path.isabs(pdspath):
                    self._reinit(pdspath)
                else:
                    print "Must be an existing absolute path!"
            elif cmd == "read":
                # Insert words from a list
                try:
                    fobj = open(args[0])
                    words = fobj.read()
                    words = eval(words) # words must be in the format of a python list
                    fobj.close()
                    print words
                    # Insert words into trie
                    for w in words:
                        self.root = self.ptrieObj.insert(
                            self.root, w, 1, lambda v1, v2: v1 + v2)
                except Exception as e:
                    print e
            elif cmd == "load":
                # Load a previously saved ptrie
                oidname = args[0]
                self.root = self.ofs.load(oidname)
                if not self.root:
                    print "Not found: %s" % oidname
            elif cmd == "find":
                self.findWords(args)
            elif cmd == "delete":
                self.delete(args[0])
            elif cmd == "insert":
                if len(args) == 1:
                    self.root = self.ptrieObj.insert(
                        self.root, args[0], 1, lambda v1, v2: v1 + v2)
                elif len(args) == 2:
                    self.root = self.ptrieObj.insert(
                        self.root, args[0], args[1], lambda v1, v2: v2)
            elif cmd == "dfwalk":
                print "Depth-First Walk:"
                for node in self.ptrieObj.dfiter(self.root):
                    self.printNode(node)
            elif cmd == "bfwalk":
                print "Breadth-First Walk:"
                self.bfwalk()
            elif cmd == "gc":
                ans = raw_input(
                    """GC will destroy any unsaved OIDs. After GC is complete,
you must re-load your saved OID because current OID references may be stale.
Continue? (y/n)""")
                if ans == "y":
                    print "Garbage Collecting oidfs."
                    self.ofs.gc()
            elif cmd == "save":
                # Save the root in oidfs
                if len(args):
                    oidname = args[0]
                else:
                    oidname = "examplePtrie"
                self.ofs.store(self.root, oidname)
            elif cmd == "ls":
                for orec in self.ofs.oriter():
                    print orec
            elif cmd == "rm":
                if len(args):
                    oidname = args[0]
                else:
                    oidname = "examplePtrie"
                self.ofs.delete(oidname)
            elif cmd == "inspect":
                if "pdscache" in sys.modules:
                    pdscache.write_coid(self.root)
                self.ptrieObj.inspect(self.root)
                self.pstor.print_stats()
            elif cmd == "stats":
                self.pstor.print_stats()
            else:
                print "Bad command: type 'quit' to quit"