def run(self):
"""Dequeue requests until we're done working"""
if self.host == None:
db = rdb4.db_open()
else:
db = rdb4.db_open(host=self.host[0], port=self.host[1])
while True:
try:
request = self.parent.requests.get_nowait()
except Queue.Empty:
break
if request.has_key("substream"):
rdb4.db_substream(db, request['substream'])
else:
rdb4.db_substream(db, 0)
first = True
result = []
last = []
while first or len(last) == 10000:
last = rdb4.db_query(db, int(request['streamid']),
int(request['starttime']),
int(request['endtime']))
first = False
print len(last)
result.extend(last)
if not self.full: break
if self.parent.as_numpy and len(result) > 0:
result = np.array(result)
result = result[:,[0,2]]
self.parent.returns[request['streamid']] = result
rdb4.db_close(db)
def run(self):
"""Dequeue requests until we're done working"""
if self.host == None:
db = rdb4.db_open()
else:
db = rdb4.db_open(host=self.host[0], port=self.host[1])
while True:
try:
request = self.parent.requests.get_nowait()
except Queue.Empty:
break
if request.has_key("substream"):
rdb4.db_substream(db, request['substream'])
else:
rdb4.db_substream(db, 0)
first = True
result = []
last = []
while first or len(last) == 10000:
last = rdb4.db_query(db, int(request['streamid']),
int(request['starttime']),
int(request['endtime']))
first = False
print len(last)
result.extend(last)
if not self.full: break
if self.parent.as_numpy and len(result) > 0:
result = np.array(result)
result = result[:, [0, 2]]
self.parent.returns[request['streamid']] = result
rdb4.db_close(db)
def test_simple(self):
self.infill_stream(1)
d = rdb.db_query(self.conn, 1, 0, 10000)
self.assertEqual(len(d), 10000)
for i in xrange(0, 10000):
self.assertEqual(d[i][0], i)
self.assertEqual(d[i][1], i)
def test_simple(self):
self.infill_stream(1)
d = rdb.db_query(self.conn, 1, 0, 10000)
self.assertEqual(len(d), 10000)
for i in xrange(0, 10000):
self.assertEqual(d[i][0], i)
self.assertEqual(d[i][1], i)
S1MAX = 1000 * 100
if len(sys.argv) == 1:
print "%s [-a | -r | -n | -d | -c]" % sys.argv[0]
elif sys.argv[1] == '-a':
# substream 1 has every bucket filled
for i in range(0, 1000):
data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)]
rdb.db_add(db, 1, data)
# substream 2 has points one hour apart
for i in range(0, 10000):
rdb.db_add(db, 2, [(i * 3600, 0, i * 3600)])
elif sys.argv[1] == '-r':
# test that we read back what we wrote
d = rdb.db_query(1, 0, 10000)
assert len(d) == 1
d = d[0].tolist()
assert len(d) == 10000
for i in xrange(0, 10000):
assert d[i][0] == i
assert d[i][1] == i
d = rdb.db_query(2, 0, 3600 * 10000)
assert len(d) == 1
d = d[0].tolist()
print d[0:10]
for i in xrange(0, 10000):
assert d[i][0] == i * 3600
elif sys.argv[1] == '-d':
#!/usr/bin/python
import readingdb as rdb
import sys
import time
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')
#print(sys.argv[1])
debug = eval(sys.argv[1])
starttime = time.time()
temp = rdb.db_query(list(range(1, 10001)), 0, 1000000000000)
endtime = time.time()
completiontime = endtime-starttime
if debug:
removeempty = lambda x: x != []
debugout = file('tempfiles/debugout', 'w')
processed = list(temp)
processed = list(map(list, processed))
for x in range(len(processed)):
processed[x] = list(map(lambda z: [x+1, int(z[0])], processed[x]))
processed = filter(removeempty, processed)
for line in processed:
debugout.write(str(line) + "\n")
debugout.close()
rdb.db_close(a)
import readingdb as rdb
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')
rdb.db_add(a, 1, [(x, 0, x) for x in xrange(0, 100)])
print rdb.db_query(1, 0, 100, conn=a)
rdb.db_close(a)
import readingdb as rdb
# specify default host/port
rdb.db_setup('localhost', 4242)
# create a connection
db = rdb.db_open('localhost')
# add data. the tuples are (timestamp, seqno, value)
rdb.db_add(db, 1, [(x, 0, x) for x in xrange(0, 100)])
# read back the data we just wrote using the existing connection
# the args are streamid, start_timestamp, end_timestamp
print rdb.db_query(1, 0, 100, conn=db)
# close
rdb.db_close(db)
# read back the data again using a connection pool this time. You can
# specify a list of streamids to range-query multiple streams at once.
rdb.db_query([1], 0, 100)
user=settings.DATABASE_USER,
)
rdb = readingdb.db_open()
loader = TsdbLoader()
loader.start()
for s in Stream.objects.all().order_by("id"):
if s.id <= 1:
next
first = True
data = []
startts = 0
extra = "source=%s path=%s" % (s.subscription, s.path())
print extra
while first or len(data) == 10000:
first = False
data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)
if len(data) > 0:
startts = data[-1][0]
loader.q.put({"method": "put", "id": s.id, "data": data, "extra": extra})
print startts
# print data
break
loader.q.put({"method": "quit"})
loader.join()
readingdb.db_close(rdb)
db.close()
import readingdb as rdb
import sys
import time
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')
debug = eval(sys.argv[2])
params = eval(sys.argv[1])
streams = params[0]
qstarttime = params[1]
qendtime = params[2]
starttime = time.time()
temp = rdb.db_query(list(range(1, 1+streams)), qstarttime, qendtime)
endtime = time.time()
completiontime = endtime-starttime
if debug:
removeempty = lambda x: x != []
debugout = file('tempfiles/debugout', 'w')
processed = list(temp)
processed = list(map(list, processed))
for x in range(len(processed)):
processed[x] = list(map(lambda z: [x+1, int(z[0])], processed[x]))
processed = filter(removeempty, processed)
for line in processed:
debugout.write(str(line) + "\n")
debugout.close()
import_map = [(x, x) for x in xrange(start, int(opts.maxid))]
else:
import_map = []
with open(opts.mapfile, "r") as fp:
for line in fp.readlines():
line = re.sub("\#.*$", "", line)
ids = re.split("[ \t]+", line)
for id in ids[1:]:
import_map.append((int(ids[0]), int(id)))
for to_stream, from_stream in import_map:
print "starting %i <- %i" % (to_stream, from_stream)
first = True
vec = [(IMPORT_START,)]
t = tic()
data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0)
if not len(data):
continue
data = data[0]
print "received", data.shape
toc(t)
t = tic()
if opts.noop:
continue
bound = (int(data.shape[0]) / 100) + 1
for i in xrange(0, bound):
vec = (data[(i * 100) : (i * 100) + 100, :]).tolist()
# print time.ctime(vec[0][0])
rdb4.db_add(db1, to_stream, map(tuple, vec))
if len(vec) < 100:
user=settings.DATABASE_USER)
rdb = readingdb.db_open()
loader = TsdbLoader()
loader.start()
for s in Stream.objects.all().order_by('id'):
if s.id <= 1:
next
first = True
data = []
startts = 0
extra = 'source=%s path=%s' % (s.subscription, s.path())
print extra
while first or len(data) == 10000:
first = False
data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)
if len(data) > 0:
startts = data[-1][0]
loader.q.put({'method': 'put',
'id': s.id,
'data': data,
'extra': extra})
print startts
# print data
break
loader.q.put({'method': 'quit'})
loader.join()
readingdb.db_close(rdb)
import_map = [(x, x) for x in xrange(start, int(opts.maxid))]
else:
import_map = []
with open(opts.mapfile, "r") as fp:
for line in fp.readlines():
line = re.sub('\#.*$', '', line)
ids = re.split('[ \t]+', line)
for id in ids[1:]:
import_map.append((int(ids[0]), int(id)))
for to_stream, from_stream in import_map:
print "starting %i <- %i" % (to_stream, from_stream)
first = True
vec = [(IMPORT_START,)]
t = tic()
data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0)
if not len(data): continue
data = data[0]
print "received", data.shape
toc(t)
t = tic()
if opts.noop: continue
bound = (int(data.shape[0]) / 100) + 1
for i in xrange(0, bound):
vec = (data[(i*100):(i*100) + 100, :]).tolist()
# print time.ctime(vec[0][0])
rdb4.db_add(db1, to_stream, map(tuple, vec))
if len(vec) < 100: break
print "inserted", to_stream
toc(t)
S1MAX = 1000 * 100
if len(sys.argv) == 1:
print "%s [-a | -r | -n | -d | -c]" % sys.argv[0]
elif sys.argv[1] == '-a':
# substream 1 has every bucket filled
for i in range(0, 1000):
data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)]
rdb.db_add(db, 1, data)
# substream 2 has points one hour apart
for i in range(0, 10000):
rdb.db_add(db, 2, [(i * 3600, 0, i * 3600)])
elif sys.argv[1] == '-r':
# test that we read back what we wrote
d = rdb.db_query(1, 0, 10000)
assert len(d) == 1
d = d[0].tolist()
assert len(d) == 10000
for i in xrange(0, 10000):
assert d[i][0] == i
assert d[i][1] == i
d = rdb.db_query(2, 0, 3600 * 10000)
assert len(d) == 1
d = d[0].tolist()
print d[0:10]
for i in xrange(0, 10000):
assert d[i][0] == i * 3600
elif sys.argv[1] == '-d':
