def _store(self, o, oname):
''' Store an OID as oidname in database but don't write rootoid yet. '''
#print "Saving %s into %s" % (o, self._rootoid)
# If using pdscache then o is actually a coid.
if isinstance(o, pdscache._CachedOid):
o = pdscache.write_coid(o)
self._rootoid = self._ptrieObj.insert(self._rootoid, oname, o)
def _writeRootoid(self):
''' writes the root OID to file '''
rootoid = self._rootoid
# If PDSCache in use then write through the coid first
if isinstance(rootoid, pdscache._CachedOid):
rootoid = pdscache.write_coid(rootoid)
rootoidPath = os.path.join(self._storpath, OidFS.rootoid_filename)
fobj = open(rootoidPath, "w")
cPickle.dump(rootoid, fobj, 2)
fobj.close()
def gc(self):
''' Garbage collects OidFS's internal Ptrie PStor. Saving only
self._rootoid. '''
# Run GC on OID's pstor first.
self._collect_pstor()
# Save oidfs's _rootoid
o = self._rootoid
if isinstance(self._rootoid, pdscache._CachedOid):
o = pdscache.write_coid(self._rootoid)
o, = self._oidPstor.keepOids([o])
o = pdscache.read_oid(o)
self._rootoid = o
self._writeRootoid()
def _collect_pstor(self):
''' Run GC on OID's pstructstor. OIDs stored in OidFS will be moved
as a result. Note this function assumes that stored oids can belong
to different PStors, which is currently allowed. In the future,
PStor should probably take possession of OidFS so that there is only
one PStor in a single OidFS. '''
# Since PStor's GC function moves OIDs, we have to make a new OID
# ptrie with the same oidname and new OID references.
pstordict = {}
for orec in self.oriter():
oname, o = orec
if isinstance(o, pdscache._CachedOid):
# o is type 'coid' and o.pstor is a PStructStor object
pstor = o.pstor
else:
# o is type 'OID' and o.pstor is a string
pstor = pstructstor.PStructStor.mkpstor(o.pstor)
if pstor not in pstordict:
# pstor dictionary's value is a (onames, ovalues) pair
pstordict[pstor] = [[], []]
onames, ovalues = pstordict[pstor]
onames.append(oname)
if isinstance(o, pdscache._CachedOid):
# Must convert back to real Oid
o = pdscache.write_coid(o)
ovalues.append(o)
if not len(pstordict):
return
# Now send to pstructstor for GC and essentially re-create our
# internal oid ptrie with new oid values
for pstor in pstordict:
onames, ovalues = pstordict[pstor]
pstordict[pstor][1] = ovalues = pstor.keepOids(ovalues)
for oname, o in zip(onames, ovalues):
self._store(o, oname)
self._writeRootoid()
def testloop(self):
while True:
try:
inputtext = raw_input(">> ");
except EOFError as e:
print "Goodbye!"
self.ofs.close()
self.pstor.close()
return
args = inputtext.split()
if len(args) == 0:
cmd = "help"
else:
cmd = args.pop(0)
if cmd == "help":
print """Type a command. Commands are:
help quit read load find delete insert dfwalk bfwalk save ls gc inspect stats"""
elif cmd == "quit":
ans = raw_input("Save? (y/n)")
if ans == "y":
self.ofs.store(self.root, "examplePtrie")
self.ofs.close()
self.pstor.close()
print "Ptrie saved as \"examplePtrie\""
break
elif cmd == "print":
try:
print eval(args[0])
except Exception as e:
print e
elif cmd == "reinit":
pdspath = args[0]
if os.path.exists(pdspath) and os.path.isabs(pdspath):
self._reinit(pdspath)
else:
print "Must be an existing absolute path!"
elif cmd == "read":
# Insert words from a list
try:
fobj = open(args[0])
words = fobj.read()
words = eval(words) # words must be in the format of a python list
fobj.close()
print words
# Insert words into trie
for w in words:
self.root = self.ptrieObj.insert(
self.root, w, 1, lambda v1, v2: v1 + v2)
except Exception as e:
print e
elif cmd == "load":
# Load a previously saved ptrie
oidname = args[0]
self.root = self.ofs.load(oidname)
if not self.root:
print "Not found: %s" % oidname
elif cmd == "find":
self.findWords(args)
elif cmd == "delete":
self.delete(args[0])
elif cmd == "insert":
if len(args) == 1:
self.root = self.ptrieObj.insert(
self.root, args[0], 1, lambda v1, v2: v1 + v2)
elif len(args) == 2:
self.root = self.ptrieObj.insert(
self.root, args[0], args[1], lambda v1, v2: v2)
elif cmd == "dfwalk":
print "Depth-First Walk:"
for node in self.ptrieObj.dfiter(self.root):
self.printNode(node)
elif cmd == "bfwalk":
print "Breadth-First Walk:"
self.bfwalk()
elif cmd == "gc":
ans = raw_input(
"""GC will destroy any unsaved OIDs. After GC is complete,
you must re-load your saved OID because current OID references may be stale.
Continue? (y/n)""")
if ans == "y":
print "Garbage Collecting oidfs."
self.ofs.gc()
elif cmd == "save":
# Save the root in oidfs
if len(args):
oidname = args[0]
else:
oidname = "examplePtrie"
self.ofs.store(self.root, oidname)
elif cmd == "ls":
for orec in self.ofs.oriter():
print orec
elif cmd == "rm":
if len(args):
oidname = args[0]
else:
oidname = "examplePtrie"
self.ofs.delete(oidname)
elif cmd == "inspect":
if "pdscache" in sys.modules:
pdscache.write_coid(self.root)
self.ptrieObj.inspect(self.root)
self.pstor.print_stats()
elif cmd == "stats":
self.pstor.print_stats()
else:
print "Bad command: type 'quit' to quit"