def test03_rnopen(self):
data = string.split("The quick brown fox jumped over the lazy dog.")
if verbose:
print "\nTesting: rnopen"
f = rnopen(self.filename, 'c')
for x in range(len(data)):
f[x + 1] = data[x]
getTest = (f[1], f[2], f[3])
if verbose:
print '%s %s %s' % getTest
assert getTest[1] == 'quick', 'data mismatch!'
rv = f.set_location(3)
if rv != (3, 'brown'):
self.fail('recno database set_location failed: ' + repr(rv))
f[25] = 'twenty-five'
f.close()
del f
f = rnopen(self.filename, 'w')
f[20] = 'twenty'
def noRec(f):
rec = f[15]
self.assertRaises(KeyError, noRec, f)
def badKey(f):
rec = f['a string']
self.assertRaises(TypeError, badKey, f)
del f[3]
rec = f.first()
while rec:
if verbose:
print rec
try:
rec = f.next()
except KeyError:
break
f.close()
def Restore(self, timestamp):
logging.debug("TagLogger.Restore(%d)", timestamp)
dirname, db_filename = os.path.split(self.db_file)
def get_filenames():
result = []
for filename in os.listdir(dirname):
if filename.startswith(db_filename) and filename != db_filename:
file_time = int(filename.split("-")[-1])
if file_time > timestamp:
result.append(filename)
result = sorted(result)
if os.path.isfile(self.db_file):
result += [db_filename]
return result
with self.lock:
self.restoring_mode = True
for filename in get_filenames():
f = bsddb3.rnopen(os.path.join(dirname, filename), "r")
for k, v in f.items():
try:
obj = cPickle.loads(v)
obj.Redo(self)
except Exception, e:
logging.exception("occurred in TagLogger while restoring from a journal : %s", e)
f.close()
self.restoring_mode = False
def Restore(self):
logging.debug("TagLogger.Restore")
dirname, db_filename = os.path.split(self.db_file)
def get_filenames():
result = []
for filename in os.listdir(dirname):
if filename.startswith(db_filename):
result.append(filename)
result = sorted(result)
if result and result[0] == db_filename:
result = result[1:] + result[:1]
return result
with self.lock:
self.restoring_mode = True
for filename in get_filenames():
f = bsddb3.rnopen(os.path.join(dirname, filename), "r")
for k, v in f.items():
try:
obj = cPickle.loads(v)
obj.Redo(self)
except Exception, e:
logging.exception("occurred in TagLogger while restoring from a journal : %s", e)
f.close()
self.restoring_mode = False
def Restore(self, timestamp, tagRef, cloud_requester_state):
logging.debug("TagLogger.Restore(%d)", timestamp)
dirname, db_filename = os.path.split(self.db_filename)
def get_filenames():
result = []
for filename in os.listdir(dirname):
if filename.startswith(db_filename) and filename != db_filename:
file_time = int(filename.split("-")[-1])
if file_time > timestamp:
result.append(filename)
result = sorted(result)
if os.path.isfile(self.db_filename):
result += [db_filename]
return result
with self._db_lock:
self._restoring_mode = True
for filename in get_filenames():
f = bsddb3.rnopen(os.path.join(dirname, filename), "r")
for k, v in f.items():
try:
obj = cPickle.loads(v)
if isinstance(obj, CloudRequestStart):
cloud_requester_state.start_request(obj.id, obj.update)
elif isinstance(obj, CloudRequestFinish):
cloud_requester_state.finish_request(obj.id)
else:
obj.Redo(tagRef)
except Exception, e:
logging.exception("occurred in TagLogger while restoring from a journal : %s", e)
f.close()
self._restoring_mode = False
def __init__(self, path):
self.corpus_path = path
self.chunks = {}
self.idx = rnopen(
os.path.join(self.corpus_path, Corpus.IDX_FILE)
) # FIXME: issue #2 this should be implemented with bsddb3 Queue
self.ridx = hashopen(os.path.join(self.corpus_path, Corpus.RIDX_FILE))
self.properties = yaml.load(file(os.path.join(self.corpus_path, Corpus.CONFIG_FILE), "r"))
def test03_rnopen(self):
data = string.split("The quick brown fox jumped over the lazy dog.")
if verbose:
print "\nTesting: rnopen"
f = rnopen(self.filename, 'c')
for x in range(len(data)):
f[x+1] = data[x]
getTest = (f[1], f[2], f[3])
if verbose:
print '%s %s %s' % getTest
assert getTest[1] == 'quick', 'data mismatch!'
rv = f.set_location(3)
if rv != (3, 'brown'):
self.fail('recno database set_location failed: '+repr(rv))
f[25] = 'twenty-five'
f.close()
del f
f = rnopen(self.filename, 'w')
f[20] = 'twenty'
def noRec(f):
rec = f[15]
self.assertRaises(KeyError, noRec, f)
def badKey(f):
rec = f['a string']
self.assertRaises(TypeError, badKey, f)
del f[3]
rec = f.first()
while rec:
if verbose:
print rec
try:
rec = f.next()
except KeyError:
break
f.close()
def make_graph(dbdir):
childdb = bsddb.btopen(dbdir + "/" + CHILD_DB)
parentdb = bsddb.btopen(dbdir + "/" + PARENT_DB)
provdb = bsddb.btopen(dbdir + "/" + PROV_DB)
tnum2tokdb = bsddb.rnopen(dbdir + "/" + TNUM2TOK_DB)
tokens = load_token_map(tnum2tokdb)
digraph, nodes = build_graph(parentdb)
parse_prov(provdb, tokens, nodes)
graph_join(digraph, nodes)
return digraph, nodes
def make_graph(dbdir):
childdb = bsddb.btopen(dbdir + "/" + CHILD_DB)
parentdb = bsddb.btopen(dbdir + "/" + PARENT_DB)
provdb = bsddb.btopen(dbdir + "/" + PROV_DB)
tnum2tokdb = bsddb.rnopen(dbdir + "/" + TNUM2TOK_DB)
tokens = load_token_map(tnum2tokdb)
digraph, nodes = build_graph(parentdb)
parse_prov(provdb, tokens, nodes)
graph_join(digraph, nodes)
return digraph, nodes
def __init__(self, path):
self.corpus_path = path
self.chunks = {}
self.idx = rnopen(os.path.join(self.corpus_path, Corpus.IDX_FILE)) #FIXME: issue #2 this should be implemented with bsddb3 Queue
self.ridx = hashopen(os.path.join(self.corpus_path, Corpus.RIDX_FILE))
self.properties = yaml.load(file(os.path.join(self.corpus_path, Corpus.CONFIG_FILE), 'r'))
def Open(self, filename):
self.file = bsddb3.rnopen(filename, "c")
self.file_opened = True
def Open(self, filename):
self.file = bsddb3.rnopen(filename, "c")
def __init__(self, filename):
self._impl = bsddb3.rnopen(filename, "c")
