Beispiel #1
0
    def test03_rnopen(self):
        data = string.split("The quick brown fox jumped over the lazy dog.")
        if verbose:
            print "\nTesting: rnopen"

        f = rnopen(self.filename, 'c')
        for x in range(len(data)):
            f[x + 1] = data[x]

        getTest = (f[1], f[2], f[3])
        if verbose:
            print '%s %s %s' % getTest

        assert getTest[1] == 'quick', 'data mismatch!'

        rv = f.set_location(3)
        if rv != (3, 'brown'):
            self.fail('recno database set_location failed: ' + repr(rv))

        f[25] = 'twenty-five'
        f.close()
        del f

        f = rnopen(self.filename, 'w')
        f[20] = 'twenty'

        def noRec(f):
            rec = f[15]

        self.assertRaises(KeyError, noRec, f)

        def badKey(f):
            rec = f['a string']

        self.assertRaises(TypeError, badKey, f)

        del f[3]

        rec = f.first()
        while rec:
            if verbose:
                print rec
            try:
                rec = f.next()
            except KeyError:
                break

        f.close()
Beispiel #2
0
    def Restore(self, timestamp):
        logging.debug("TagLogger.Restore(%d)", timestamp)
        dirname, db_filename = os.path.split(self.db_file)

        def get_filenames():
            result = []
            for filename in os.listdir(dirname):
                if filename.startswith(db_filename) and filename != db_filename:
                    file_time = int(filename.split("-")[-1])
                    if file_time > timestamp:
                        result.append(filename)
            result = sorted(result)
            if os.path.isfile(self.db_file):
                result += [db_filename]
            return result

        with self.lock:
            self.restoring_mode = True
            for filename in get_filenames():
                f = bsddb3.rnopen(os.path.join(dirname, filename), "r")
                for k, v in f.items():
                    try:
                        obj = cPickle.loads(v)
                        obj.Redo(self)
                    except Exception, e:
                        logging.exception("occurred in TagLogger while restoring from a journal : %s", e)
                f.close()
            self.restoring_mode = False
Beispiel #3
0
    def Restore(self):
        logging.debug("TagLogger.Restore")
        dirname, db_filename = os.path.split(self.db_file)

        def get_filenames():
            result = []
            for filename in os.listdir(dirname):
                if filename.startswith(db_filename):
                    result.append(filename)
            result = sorted(result)
            if result and result[0] == db_filename:
                result = result[1:] + result[:1]
            return result

        with self.lock:
            self.restoring_mode = True
            for filename in get_filenames():
                f = bsddb3.rnopen(os.path.join(dirname, filename), "r")
                for k, v in f.items():
                    try:
                        obj = cPickle.loads(v)
                        obj.Redo(self)
                    except Exception, e:
                        logging.exception("occurred in TagLogger while restoring from a journal : %s", e)
                f.close()
            self.restoring_mode = False
Beispiel #4
0
    def Restore(self, timestamp, tagRef, cloud_requester_state):
        logging.debug("TagLogger.Restore(%d)", timestamp)
        dirname, db_filename = os.path.split(self.db_filename)

        def get_filenames():
            result = []
            for filename in os.listdir(dirname):
                if filename.startswith(db_filename) and filename != db_filename:
                    file_time = int(filename.split("-")[-1])
                    if file_time > timestamp:
                        result.append(filename)
            result = sorted(result)
            if os.path.isfile(self.db_filename):
                result += [db_filename]
            return result

        with self._db_lock:
            self._restoring_mode = True
            for filename in get_filenames():
                f = bsddb3.rnopen(os.path.join(dirname, filename), "r")
                for k, v in f.items():
                    try:
                        obj = cPickle.loads(v)

                        if isinstance(obj, CloudRequestStart):
                            cloud_requester_state.start_request(obj.id, obj.update)
                        elif isinstance(obj, CloudRequestFinish):
                            cloud_requester_state.finish_request(obj.id)
                        else:
                            obj.Redo(tagRef)

                    except Exception, e:
                        logging.exception("occurred in TagLogger while restoring from a journal : %s", e)
                f.close()
            self._restoring_mode = False
Beispiel #5
0
 def __init__(self, path):
     self.corpus_path = path
     self.chunks = {}
     self.idx = rnopen(
         os.path.join(self.corpus_path, Corpus.IDX_FILE)
     )  # FIXME: issue #2 this should be implemented with bsddb3 Queue
     self.ridx = hashopen(os.path.join(self.corpus_path, Corpus.RIDX_FILE))
     self.properties = yaml.load(file(os.path.join(self.corpus_path, Corpus.CONFIG_FILE), "r"))
Beispiel #6
0
    def test03_rnopen(self):
        data = string.split("The quick brown fox jumped over the lazy dog.")
        if verbose:
            print "\nTesting: rnopen"

        f = rnopen(self.filename, 'c')
        for x in range(len(data)):
            f[x+1] = data[x]

        getTest = (f[1], f[2], f[3])
        if verbose:
            print '%s %s %s' % getTest

        assert getTest[1] == 'quick', 'data mismatch!'

        rv = f.set_location(3)
        if rv != (3, 'brown'):
            self.fail('recno database set_location failed: '+repr(rv))

        f[25] = 'twenty-five'
        f.close()
        del f

        f = rnopen(self.filename, 'w')
        f[20] = 'twenty'

        def noRec(f):
            rec = f[15]
        self.assertRaises(KeyError, noRec, f)

        def badKey(f):
            rec = f['a string']
        self.assertRaises(TypeError, badKey, f)

        del f[3]

        rec = f.first()
        while rec:
            if verbose:
                print rec
            try:
                rec = f.next()
            except KeyError:
                break

        f.close()
Beispiel #7
0
def make_graph(dbdir):
    childdb = bsddb.btopen(dbdir + "/" + CHILD_DB)
    parentdb = bsddb.btopen(dbdir + "/" + PARENT_DB)
    provdb = bsddb.btopen(dbdir + "/" + PROV_DB)
    tnum2tokdb = bsddb.rnopen(dbdir + "/" + TNUM2TOK_DB)

    tokens = load_token_map(tnum2tokdb)
    digraph, nodes = build_graph(parentdb)
    parse_prov(provdb, tokens, nodes)
    graph_join(digraph, nodes)
    return digraph, nodes
Beispiel #8
0
def make_graph(dbdir):
    childdb = bsddb.btopen(dbdir + "/" + CHILD_DB)
    parentdb = bsddb.btopen(dbdir + "/" + PARENT_DB)
    provdb = bsddb.btopen(dbdir + "/" + PROV_DB)
    tnum2tokdb = bsddb.rnopen(dbdir + "/" + TNUM2TOK_DB)

    tokens = load_token_map(tnum2tokdb)
    digraph, nodes = build_graph(parentdb)
    parse_prov(provdb, tokens, nodes)
    graph_join(digraph, nodes)
    return digraph, nodes
Beispiel #9
0
 def __init__(self, path):
     self.corpus_path = path
     self.chunks = {}
     self.idx = rnopen(os.path.join(self.corpus_path, Corpus.IDX_FILE))  #FIXME: issue #2 this should be implemented with bsddb3 Queue
     self.ridx = hashopen(os.path.join(self.corpus_path, Corpus.RIDX_FILE))
     self.properties = yaml.load(file(os.path.join(self.corpus_path, Corpus.CONFIG_FILE), 'r'))
Beispiel #10
0
 def Open(self, filename):
     self.file = bsddb3.rnopen(filename, "c")
     self.file_opened = True
Beispiel #11
0
 def Open(self, filename):
     self.file = bsddb3.rnopen(filename, "c")
Beispiel #12
0
 def __init__(self, filename):
     self._impl = bsddb3.rnopen(filename, "c")