Beispiel #1
0
        def run(self):
            """Dequeue requests until we're done working"""
            if self.host == None:
                db = rdb4.db_open()
            else:
                db = rdb4.db_open(host=self.host[0], port=self.host[1])
            while True:
                try:
                    request = self.parent.requests.get_nowait()
                except Queue.Empty:
                    break
                if request.has_key("substream"):
                    rdb4.db_substream(db, request['substream'])
                else:
                    rdb4.db_substream(db, 0)

                first = True
                result = []
                last = []

                while first or len(last) == 10000:
                    last = rdb4.db_query(db, int(request['streamid']),
                                         int(request['starttime']),
                                         int(request['endtime']))
                    first = False
                    print len(last)
                    result.extend(last)
                    if not self.full: break

                if self.parent.as_numpy and len(result) > 0:
                    result = np.array(result)
                    result = result[:, [0, 2]]
                self.parent.returns[request['streamid']] = result

            rdb4.db_close(db)
        def run(self):
            """Dequeue requests until we're done working"""
            if self.host == None:
                db = rdb4.db_open()
            else:
                db = rdb4.db_open(host=self.host[0], port=self.host[1])
            while True:
                try:
                    request = self.parent.requests.get_nowait()
                except Queue.Empty:
                    break
                if request.has_key("substream"):
                    rdb4.db_substream(db, request['substream'])
                else:
                    rdb4.db_substream(db, 0)

                first = True
                result = []
                last = []

                while first or len(last) == 10000:
                    last = rdb4.db_query(db, int(request['streamid']),
                                         int(request['starttime']),
                                         int(request['endtime']))
                    first = False
                    print len(last)
                    result.extend(last)
                    if not self.full: break

                if self.parent.as_numpy and len(result) > 0:
                    result = np.array(result)
                    result = result[:,[0,2]]                      
                self.parent.returns[request['streamid']] = result

            rdb4.db_close(db)
Beispiel #3
0
class TestIface(unittest.TestCase):
    def setUp(self):
        try:
            os.makedirs(datadir)
        except OSError, e:
            if e.errno != os.errno.EEXIST:
                raise
        cmd = [readingdb, '-p', str(port), '-d', datadir, '-c', '1']
        self.log = open(log, 'a')
        self.db = subprocess.Popen(cmd,
                                   stderr=subprocess.PIPE,
                                   stdout=self.log)

        # wait for startup or a fatal message
        for x in xrange(0, 20):
            l = self.db.stderr.readline()
            if 'FATAL' in l:
                raise Exception(l)
            elif 'listening' in l:
                break

        self.conn = rdb.db_open('localhost', port)
Beispiel #4
0
import sys
import time
import readingdb as rdb
import _readingdb

import numpy as np

print "using readingdb", rdb.__file__
print _readingdb.__file__

end = 1304102690

rdb.db_setup('localhost', 4242)
db = rdb.db_open(host='localhost', port=4242)
# db = rdb.db_open()

def next(id, ref, n=1): 
    return rdb.db_next(id, ref, n=n, conn=db)[0].tolist()

def prev(id, ref, n=1, conn=db): 
    return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist()

S1MAX = 1000 * 100
if len(sys.argv) == 1:
    print "%s [-a | -r | -n | -d | -c]" % sys.argv[0]
elif sys.argv[1] == '-a':
    # substream 1 has every bucket filled
    for i in range(0, 1000):
        data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)]
        rdb.db_add(db, 1, data)
Beispiel #5
0
import sys
import time
import readingdb as rdb
import _readingdb

import numpy as np

print "using readingdb", rdb.__file__
print _readingdb.__file__

end = 1304102690
PORT = 4242

rdb.db_setup("localhost", PORT)
db = rdb.db_open(host="localhost", port=PORT)
# db = rdb.db_open()


def next(id, ref, n=1):
    return rdb.db_next(id, ref, n=n, conn=db)[0].tolist()


def prev(id, ref, n=1, conn=db):
    return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist()


S1MAX = 1000 * 100
if len(sys.argv) == 1:
    print "%s [-a | -r | -n | -d | -c]" % sys.argv[0]
elif sys.argv[1] == "-a":
    # substream 1 has every bucket filled
#!/usr/bin/python

import readingdb as rdb
import sys
import time

rdb.db_setup("localhost", 4242)
a = rdb.db_open("localhost")

b = rdb.db_prev(1, 100000000000, conn=a)
lasttime = b[0][0][0]

rdb.db_close(a)

print(lasttime)

# ltime = file('tempfiles/lasttime', 'w')
# ltime.write(str(lasttime))
# ltime.close()

sys.exit()
Beispiel #7
0
#!/usr/bin/python

import readingdb as rdb
import sys
import time

rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')

#print(sys.argv[1])
debug = eval(sys.argv[1])

starttime = time.time()
temp = rdb.db_query(list(range(1, 10001)), 0, 1000000000000)
endtime = time.time()
completiontime = endtime-starttime

if debug:
    removeempty = lambda x: x != []
    debugout = file('tempfiles/debugout', 'w')
    processed = list(temp)
    processed = list(map(list, processed))
    for x in range(len(processed)):
        processed[x] = list(map(lambda z: [x+1, int(z[0])], processed[x]))
    processed = filter(removeempty, processed)
    for line in processed:
        debugout.write(str(line) + "\n")
    debugout.close()

rdb.db_close(a)
Beispiel #8
0
import readingdb as rdb
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')

rdb.db_add(a, 1, [(x, 0, x) for x in xrange(0, 100)])
print rdb.db_query(1, 0, 100, conn=a)
rdb.db_close(a)
Beispiel #9
0
                for i in xrange(0, len(add["data"])):
                    a = "put %i %i %f %s\n" % (add["id"], add["data"][i][0], add["data"][i][2], add["extra"])
                    s.send(a)
            elif add["method"] == "quit":
                break
        s.close()


if __name__ == "__main__":
    db = mq.connect(
        passwd=settings.DATABASE_PASSWORD,
        db=settings.DATABASE_NAME,
        host=settings.DATABASE_HOST,
        user=settings.DATABASE_USER,
    )
    rdb = readingdb.db_open()
    loader = TsdbLoader()
    loader.start()

    for s in Stream.objects.all().order_by("id"):
        if s.id <= 1:
            next
        first = True
        data = []
        startts = 0
        extra = "source=%s path=%s" % (s.subscription, s.path())
        print extra
        while first or len(data) == 10000:
            first = False
            data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)
Beispiel #10
0
        "-n", "--no-action", dest="noop", default=False, action="store_true", help="don't actually insert the data"
    )
    parser.add_option("-f", "--map-file", dest="mapfile", default=False, help="import using a map file")
    opts, hosts = parser.parse_args()
    if len(hosts) != 2:
        parser.print_help()
        sys.exit(1)

    old_db = parse_netloc(hosts[0])
    new_db = parse_netloc(hosts[1])

    print "Importing data from %s:%i to %s:%i" % (old_db + new_db)
    print "substream: %i" % int(opts.substream)

    rdb4.db_setup(old_db[0], old_db[1])
    db0 = rdb4.db_open(host=old_db[0], port=old_db[1])
    db1 = rdb4.db_open(host=new_db[0], port=new_db[1])

    # rdb4.db_substream(db0, int(opts.substream))
    # rdb4.db_substream(db1, int(opts.substream))

    if not opts.zero:
        IMPORT_START = int(time.time()) - (int(opts.ago) * 3600)
        IMPORT_STOP = 2 ** 32 - 10
    else:
        IMPORT_START = 1
        IMPORT_STOP = int(time.time()) - (int(opts.ago) * 3600)
    print "Importing from %i to %i" % (IMPORT_START, IMPORT_STOP)

    if not opts.mapfile:
        start = int(opts.startid)
Beispiel #11
0
                for i in xrange(0, len(add['data'])):
                    a = "put %i %i %f %s\n" % (add['id'],
                                               add['data'][i][0],
                                               add['data'][i][2],
                                               add['extra'])
                    s.send(a)
            elif add['method'] == 'quit':
                break
        s.close()

if __name__ == '__main__':
    db = mq.connect(passwd=settings.DATABASE_PASSWORD,
                    db=settings.DATABASE_NAME,
                    host=settings.DATABASE_HOST,
                    user=settings.DATABASE_USER)
    rdb = readingdb.db_open()
    loader = TsdbLoader()
    loader.start()
    
    for s in Stream.objects.all().order_by('id'):
        if s.id <= 1:
            next
        first = True
        data = []
        startts = 0
        extra = 'source=%s path=%s' % (s.subscription, s.path())
        print extra
        while first or len(data) == 10000:
            first = False
            data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)
Beispiel #12
0
                      help='don\'t actually insert the data')
    parser.add_option('-f', '--map-file', dest='mapfile', default=False,
                      help='import using a map file')
    opts, hosts = parser.parse_args()
    if len(hosts) != 2:
        parser.print_help()
        sys.exit(1)

    old_db = parse_netloc(hosts[0])
    new_db = parse_netloc(hosts[1])

    print "Importing data from %s:%i to %s:%i" % (old_db + new_db)
    print "substream: %i" % int(opts.substream)
    
    rdb4.db_setup(old_db[0], old_db[1])
    db0 = rdb4.db_open(host=old_db[0], port=old_db[1])
    db1 = rdb4.db_open(host=new_db[0], port=new_db[1])

    # rdb4.db_substream(db0, int(opts.substream))
    # rdb4.db_substream(db1, int(opts.substream))

    if not opts.zero:
        IMPORT_START = int(time.time()) - (int(opts.ago) * 3600)
        IMPORT_STOP = 2**32 - 10
    else:
        IMPORT_START = 1
        IMPORT_STOP = int(time.time()) - (int(opts.ago) * 3600)
    print "Importing from %i to %i" % (IMPORT_START, IMPORT_STOP)

    if not opts.mapfile:
        start = int(opts.startid)
Beispiel #13
0
import sys
import time
import readingdb as rdb
import _readingdb

import numpy as np

print "using readingdb", rdb.__file__
print _readingdb.__file__

end = 1304102690

rdb.db_setup('localhost', 4242)
db = rdb.db_open(host='localhost', port=4242)
# db = rdb.db_open()


def next(id, ref, n=1):
    return rdb.db_next(id, ref, n=n, conn=db)[0].tolist()


def prev(id, ref, n=1, conn=db):
    return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist()


S1MAX = 1000 * 100
if len(sys.argv) == 1:
    print "%s [-a | -r | -n | -d | -c]" % sys.argv[0]
elif sys.argv[1] == '-a':
    # substream 1 has every bucket filled
    for i in range(0, 1000):
Beispiel #14
0
import sys
import time
import readingdb as rdb
import _readingdb

import numpy as np

print "using readingdb", rdb.__file__
print _readingdb.__file__

end = 1304102690
PORT = 4242

rdb.db_setup('localhost', PORT)
db = rdb.db_open(host='localhost', port=PORT)
# db = rdb.db_open()


def next(id, ref, n=1):
    return rdb.db_next(id, ref, n=n, conn=db)[0].tolist()


def prev(id, ref, n=1, conn=db):
    return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist()


S1MAX = 1000 * 100
if len(sys.argv) == 1:
    print "%s [-a | -r | -n | -d | -c]" % sys.argv[0]
elif sys.argv[1] == '-a':
    # substream 1 has every bucket filled