def run(self):
            """Dequeue requests until we're done working"""
            if self.host == None:
                db = rdb4.db_open()
            else:
                db = rdb4.db_open(host=self.host[0], port=self.host[1])
            while True:
                try:
                    request = self.parent.requests.get_nowait()
                except Queue.Empty:
                    break
                if request.has_key("substream"):
                    rdb4.db_substream(db, request['substream'])
                else:
                    rdb4.db_substream(db, 0)

                first = True
                result = []
                last = []

                while first or len(last) == 10000:
                    last = rdb4.db_query(db, int(request['streamid']),
                                         int(request['starttime']),
                                         int(request['endtime']))
                    first = False
                    print len(last)
                    result.extend(last)
                    if not self.full: break

                if self.parent.as_numpy and len(result) > 0:
                    result = np.array(result)
                    result = result[:,[0,2]]                      
                self.parent.returns[request['streamid']] = result

            rdb4.db_close(db)
Beispiel #2
0
        def run(self):
            """Dequeue requests until we're done working"""
            if self.host == None:
                db = rdb4.db_open()
            else:
                db = rdb4.db_open(host=self.host[0], port=self.host[1])
            while True:
                try:
                    request = self.parent.requests.get_nowait()
                except Queue.Empty:
                    break
                if request.has_key("substream"):
                    rdb4.db_substream(db, request['substream'])
                else:
                    rdb4.db_substream(db, 0)

                first = True
                result = []
                last = []

                while first or len(last) == 10000:
                    last = rdb4.db_query(db, int(request['streamid']),
                                         int(request['starttime']),
                                         int(request['endtime']))
                    first = False
                    print len(last)
                    result.extend(last)
                    if not self.full: break

                if self.parent.as_numpy and len(result) > 0:
                    result = np.array(result)
                    result = result[:, [0, 2]]
                self.parent.returns[request['streamid']] = result

            rdb4.db_close(db)
    def tearDown(self):
        rdb.db_close(self.conn)
        self.db.terminate()
        self.db.wait()
        self.log.close()

        for f in glob.glob(os.path.join(datadir, '*')):
            os.remove(f)
        os.removedirs(datadir)
Beispiel #4
0
    def tearDown(self):
        rdb.db_close(self.conn)
        self.db.terminate()
        self.db.wait()
        self.log.close()

        for f in glob.glob(os.path.join(datadir, '*')):
            os.remove(f)
        os.removedirs(datadir)
Beispiel #5
0
    for f in sys.argv[2:]:
        with open(f, 'r') as fp:
            add_vec = []
            streamid = int(f[f.rindex('.')+1:])
            for line in fp.readlines():
                parts = line.strip().split(',')
                print parts
                assert len(parts) == 2
                tuple = (int(parts[0]), 0, float(parts[1]))
                add_vec.append(tuple)
                if len(add_vec) == 100:
                    rdb.db_add(db, streamid, add_vec)
                    add_vec = []
elif sys.argv[1] == '-m':
    d = rdb.db_query([1, 2], 0, 100000000000, limit=-1)
    d1, d2 = d
    assert np.shape(d1) == (1e5, 2)
    assert np.shape(d2) == (1e4, 2)
    assert np.sum(d1[:, 0] - np.arange(0, 1e5)) == 0
    assert np.sum(d1[:, 1] - np.arange(0, 1e5)) == 0
    assert np.sum(d2[:, 0] - np.arange(0, 3600 * 1e4, 3600)) == 0
    assert np.sum(d2[:, 1] - np.arange(0, 3600 * 1e4, 3600)) == 0
else:
    print "invalid argument"

# for i in xrange(0, 1000):
#     x = rdb.db_next(db, 1, i)
#     assert x[0][0] == i+1

rdb.db_close(db)
#!/usr/bin/python

import readingdb as rdb
import sys
import time

rdb.db_setup("localhost", 4242)
a = rdb.db_open("localhost")

b = rdb.db_prev(1, 100000000000, conn=a)
lasttime = b[0][0][0]

rdb.db_close(a)

print(lasttime)

# ltime = file('tempfiles/lasttime', 'w')
# ltime.write(str(lasttime))
# ltime.close()

sys.exit()
Beispiel #7
0
import readingdb as rdb
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')

rdb.db_add(a, 1, [(x, 0, x) for x in xrange(0, 100)])
print rdb.db_query(1, 0, 100, conn=a)
rdb.db_close(a)
Beispiel #8
0
        user=settings.DATABASE_USER,
    )
    rdb = readingdb.db_open()
    loader = TsdbLoader()
    loader.start()

    for s in Stream.objects.all().order_by("id"):
        if s.id <= 1:
            next
        first = True
        data = []
        startts = 0
        extra = "source=%s path=%s" % (s.subscription, s.path())
        print extra
        while first or len(data) == 10000:
            first = False
            data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)

            if len(data) > 0:
                startts = data[-1][0]
                loader.q.put({"method": "put", "id": s.id, "data": data, "extra": extra})

            print startts
            # print data
        break
    loader.q.put({"method": "quit"})
    loader.join()

    readingdb.db_close(rdb)
    db.close()
Beispiel #9
0
                    import_map.append((int(ids[0]), int(id)))

    for to_stream, from_stream in import_map:
        print "starting %i <- %i" % (to_stream, from_stream)
        first = True
        vec = [(IMPORT_START,)]
        t = tic()
        data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0)
        if not len(data):
            continue
        data = data[0]
        print "received", data.shape
        toc(t)

        t = tic()
        if opts.noop:
            continue
        bound = (int(data.shape[0]) / 100) + 1
        for i in xrange(0, bound):
            vec = (data[(i * 100) : (i * 100) + 100, :]).tolist()
            # print time.ctime(vec[0][0])
            rdb4.db_add(db1, to_stream, map(tuple, vec))
            if len(vec) < 100:
                break
        print "inserted", to_stream
        toc(t)
        time.sleep(float(opts.delay))
        # rdb4.db_sync(db4)
    # rdb4.db_close(db0)
    rdb4.db_close(db1)
Beispiel #10
0
    loader.start()
    
    for s in Stream.objects.all().order_by('id'):
        if s.id <= 1:
            next
        first = True
        data = []
        startts = 0
        extra = 'source=%s path=%s' % (s.subscription, s.path())
        print extra
        while first or len(data) == 10000:
            first = False
            data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)

            if len(data) > 0:
                startts = data[-1][0]
                loader.q.put({'method': 'put',
                              'id': s.id,
                              'data': data,
                              'extra': extra})
                
            print startts
            # print data
        break
    loader.q.put({'method': 'quit'})
    loader.join()
    
    readingdb.db_close(rdb)
    db.close()

Beispiel #11
0
                line = re.sub('\#.*$', '', line)
                ids = re.split('[ \t]+', line)
                for id in ids[1:]:
                    import_map.append((int(ids[0]), int(id)))
        
    for to_stream, from_stream in import_map:
        print "starting %i <- %i" % (to_stream, from_stream) 
        first = True
        vec = [(IMPORT_START,)]
        t = tic()
        data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0)
        if not len(data): continue
        data = data[0]
        print "received", data.shape
        toc(t)

        t = tic()
        if opts.noop: continue
        bound = (int(data.shape[0]) / 100) + 1
        for i in xrange(0, bound):
            vec = (data[(i*100):(i*100) + 100, :]).tolist()
            # print time.ctime(vec[0][0])
            rdb4.db_add(db1, to_stream, map(tuple, vec))
            if len(vec) < 100: break
        print "inserted", to_stream
        toc(t)
        time.sleep(float(opts.delay))
        # rdb4.db_sync(db4)
    #rdb4.db_close(db0)
    rdb4.db_close(db1)
Beispiel #12
0
    for f in sys.argv[2:]:
        with open(f, 'r') as fp:
            add_vec = []
            streamid = int(f[f.rindex('.') + 1:])
            for line in fp.readlines():
                parts = line.strip().split(',')
                print parts
                assert len(parts) == 2
                tuple = (int(parts[0]), 0, float(parts[1]))
                add_vec.append(tuple)
                if len(add_vec) == 100:
                    rdb.db_add(db, streamid, add_vec)
                    add_vec = []
elif sys.argv[1] == '-m':
    d = rdb.db_query([1, 2], 0, 100000000000, limit=-1)
    d1, d2 = d
    assert np.shape(d1) == (1e5, 2)
    assert np.shape(d2) == (1e4, 2)
    assert np.sum(d1[:, 0] - np.arange(0, 1e5)) == 0
    assert np.sum(d1[:, 1] - np.arange(0, 1e5)) == 0
    assert np.sum(d2[:, 0] - np.arange(0, 3600 * 1e4, 3600)) == 0
    assert np.sum(d2[:, 1] - np.arange(0, 3600 * 1e4, 3600)) == 0
else:
    print "invalid argument"

# for i in xrange(0, 1000):
#     x = rdb.db_next(db, 1, i)
#     assert x[0][0] == i+1

rdb.db_close(db)