def run(self): """Dequeue requests until we're done working""" if self.host == None: db = rdb4.db_open() else: db = rdb4.db_open(host=self.host[0], port=self.host[1]) while True: try: request = self.parent.requests.get_nowait() except Queue.Empty: break if request.has_key("substream"): rdb4.db_substream(db, request['substream']) else: rdb4.db_substream(db, 0) first = True result = [] last = [] while first or len(last) == 10000: last = rdb4.db_query(db, int(request['streamid']), int(request['starttime']), int(request['endtime'])) first = False print len(last) result.extend(last) if not self.full: break if self.parent.as_numpy and len(result) > 0: result = np.array(result) result = result[:, [0, 2]] self.parent.returns[request['streamid']] = result rdb4.db_close(db)
def run(self): """Dequeue requests until we're done working""" if self.host == None: db = rdb4.db_open() else: db = rdb4.db_open(host=self.host[0], port=self.host[1]) while True: try: request = self.parent.requests.get_nowait() except Queue.Empty: break if request.has_key("substream"): rdb4.db_substream(db, request['substream']) else: rdb4.db_substream(db, 0) first = True result = [] last = [] while first or len(last) == 10000: last = rdb4.db_query(db, int(request['streamid']), int(request['starttime']), int(request['endtime'])) first = False print len(last) result.extend(last) if not self.full: break if self.parent.as_numpy and len(result) > 0: result = np.array(result) result = result[:,[0,2]] self.parent.returns[request['streamid']] = result rdb4.db_close(db)
class TestIface(unittest.TestCase): def setUp(self): try: os.makedirs(datadir) except OSError, e: if e.errno != os.errno.EEXIST: raise cmd = [readingdb, '-p', str(port), '-d', datadir, '-c', '1'] self.log = open(log, 'a') self.db = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=self.log) # wait for startup or a fatal message for x in xrange(0, 20): l = self.db.stderr.readline() if 'FATAL' in l: raise Exception(l) elif 'listening' in l: break self.conn = rdb.db_open('localhost', port)
import sys import time import readingdb as rdb import _readingdb import numpy as np print "using readingdb", rdb.__file__ print _readingdb.__file__ end = 1304102690 rdb.db_setup('localhost', 4242) db = rdb.db_open(host='localhost', port=4242) # db = rdb.db_open() def next(id, ref, n=1): return rdb.db_next(id, ref, n=n, conn=db)[0].tolist() def prev(id, ref, n=1, conn=db): return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist() S1MAX = 1000 * 100 if len(sys.argv) == 1: print "%s [-a | -r | -n | -d | -c]" % sys.argv[0] elif sys.argv[1] == '-a': # substream 1 has every bucket filled for i in range(0, 1000): data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)] rdb.db_add(db, 1, data)
import sys import time import readingdb as rdb import _readingdb import numpy as np print "using readingdb", rdb.__file__ print _readingdb.__file__ end = 1304102690 PORT = 4242 rdb.db_setup("localhost", PORT) db = rdb.db_open(host="localhost", port=PORT) # db = rdb.db_open() def next(id, ref, n=1): return rdb.db_next(id, ref, n=n, conn=db)[0].tolist() def prev(id, ref, n=1, conn=db): return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist() S1MAX = 1000 * 100 if len(sys.argv) == 1: print "%s [-a | -r | -n | -d | -c]" % sys.argv[0] elif sys.argv[1] == "-a": # substream 1 has every bucket filled
#!/usr/bin/python import readingdb as rdb import sys import time rdb.db_setup("localhost", 4242) a = rdb.db_open("localhost") b = rdb.db_prev(1, 100000000000, conn=a) lasttime = b[0][0][0] rdb.db_close(a) print(lasttime) # ltime = file('tempfiles/lasttime', 'w') # ltime.write(str(lasttime)) # ltime.close() sys.exit()
#!/usr/bin/python import readingdb as rdb import sys import time rdb.db_setup('localhost', 4242) a = rdb.db_open('localhost') #print(sys.argv[1]) debug = eval(sys.argv[1]) starttime = time.time() temp = rdb.db_query(list(range(1, 10001)), 0, 1000000000000) endtime = time.time() completiontime = endtime-starttime if debug: removeempty = lambda x: x != [] debugout = file('tempfiles/debugout', 'w') processed = list(temp) processed = list(map(list, processed)) for x in range(len(processed)): processed[x] = list(map(lambda z: [x+1, int(z[0])], processed[x])) processed = filter(removeempty, processed) for line in processed: debugout.write(str(line) + "\n") debugout.close() rdb.db_close(a)
import readingdb as rdb rdb.db_setup('localhost', 4242) a = rdb.db_open('localhost') rdb.db_add(a, 1, [(x, 0, x) for x in xrange(0, 100)]) print rdb.db_query(1, 0, 100, conn=a) rdb.db_close(a)
for i in xrange(0, len(add["data"])): a = "put %i %i %f %s\n" % (add["id"], add["data"][i][0], add["data"][i][2], add["extra"]) s.send(a) elif add["method"] == "quit": break s.close() if __name__ == "__main__": db = mq.connect( passwd=settings.DATABASE_PASSWORD, db=settings.DATABASE_NAME, host=settings.DATABASE_HOST, user=settings.DATABASE_USER, ) rdb = readingdb.db_open() loader = TsdbLoader() loader.start() for s in Stream.objects.all().order_by("id"): if s.id <= 1: next first = True data = [] startts = 0 extra = "source=%s path=%s" % (s.subscription, s.path()) print extra while first or len(data) == 10000: first = False data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)
"-n", "--no-action", dest="noop", default=False, action="store_true", help="don't actually insert the data" ) parser.add_option("-f", "--map-file", dest="mapfile", default=False, help="import using a map file") opts, hosts = parser.parse_args() if len(hosts) != 2: parser.print_help() sys.exit(1) old_db = parse_netloc(hosts[0]) new_db = parse_netloc(hosts[1]) print "Importing data from %s:%i to %s:%i" % (old_db + new_db) print "substream: %i" % int(opts.substream) rdb4.db_setup(old_db[0], old_db[1]) db0 = rdb4.db_open(host=old_db[0], port=old_db[1]) db1 = rdb4.db_open(host=new_db[0], port=new_db[1]) # rdb4.db_substream(db0, int(opts.substream)) # rdb4.db_substream(db1, int(opts.substream)) if not opts.zero: IMPORT_START = int(time.time()) - (int(opts.ago) * 3600) IMPORT_STOP = 2 ** 32 - 10 else: IMPORT_START = 1 IMPORT_STOP = int(time.time()) - (int(opts.ago) * 3600) print "Importing from %i to %i" % (IMPORT_START, IMPORT_STOP) if not opts.mapfile: start = int(opts.startid)
for i in xrange(0, len(add['data'])): a = "put %i %i %f %s\n" % (add['id'], add['data'][i][0], add['data'][i][2], add['extra']) s.send(a) elif add['method'] == 'quit': break s.close() if __name__ == '__main__': db = mq.connect(passwd=settings.DATABASE_PASSWORD, db=settings.DATABASE_NAME, host=settings.DATABASE_HOST, user=settings.DATABASE_USER) rdb = readingdb.db_open() loader = TsdbLoader() loader.start() for s in Stream.objects.all().order_by('id'): if s.id <= 1: next first = True data = [] startts = 0 extra = 'source=%s path=%s' % (s.subscription, s.path()) print extra while first or len(data) == 10000: first = False data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)
help='don\'t actually insert the data') parser.add_option('-f', '--map-file', dest='mapfile', default=False, help='import using a map file') opts, hosts = parser.parse_args() if len(hosts) != 2: parser.print_help() sys.exit(1) old_db = parse_netloc(hosts[0]) new_db = parse_netloc(hosts[1]) print "Importing data from %s:%i to %s:%i" % (old_db + new_db) print "substream: %i" % int(opts.substream) rdb4.db_setup(old_db[0], old_db[1]) db0 = rdb4.db_open(host=old_db[0], port=old_db[1]) db1 = rdb4.db_open(host=new_db[0], port=new_db[1]) # rdb4.db_substream(db0, int(opts.substream)) # rdb4.db_substream(db1, int(opts.substream)) if not opts.zero: IMPORT_START = int(time.time()) - (int(opts.ago) * 3600) IMPORT_STOP = 2**32 - 10 else: IMPORT_START = 1 IMPORT_STOP = int(time.time()) - (int(opts.ago) * 3600) print "Importing from %i to %i" % (IMPORT_START, IMPORT_STOP) if not opts.mapfile: start = int(opts.startid)
import sys import time import readingdb as rdb import _readingdb import numpy as np print "using readingdb", rdb.__file__ print _readingdb.__file__ end = 1304102690 rdb.db_setup('localhost', 4242) db = rdb.db_open(host='localhost', port=4242) # db = rdb.db_open() def next(id, ref, n=1): return rdb.db_next(id, ref, n=n, conn=db)[0].tolist() def prev(id, ref, n=1, conn=db): return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist() S1MAX = 1000 * 100 if len(sys.argv) == 1: print "%s [-a | -r | -n | -d | -c]" % sys.argv[0] elif sys.argv[1] == '-a': # substream 1 has every bucket filled for i in range(0, 1000):
import sys import time import readingdb as rdb import _readingdb import numpy as np print "using readingdb", rdb.__file__ print _readingdb.__file__ end = 1304102690 PORT = 4242 rdb.db_setup('localhost', PORT) db = rdb.db_open(host='localhost', port=PORT) # db = rdb.db_open() def next(id, ref, n=1): return rdb.db_next(id, ref, n=n, conn=db)[0].tolist() def prev(id, ref, n=1, conn=db): return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist() S1MAX = 1000 * 100 if len(sys.argv) == 1: print "%s [-a | -r | -n | -d | -c]" % sys.argv[0] elif sys.argv[1] == '-a': # substream 1 has every bucket filled