def infill_stream(self, stream):
for i in range(0, 1000):
data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)]
self.assertEqual(rdb.db_add(self.conn, stream, data), 1)
# db = rdb.db_open()
def next(id, ref, n=1):
return rdb.db_next(id, ref, n=n, conn=db)[0].tolist()
def prev(id, ref, n=1, conn=db):
return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist()
S1MAX = 1000 * 100
if len(sys.argv) == 1:
print "%s [-a | -r | -n | -d | -c]" % sys.argv[0]
elif sys.argv[1] == '-a':
# substream 1 has every bucket filled
for i in range(0, 1000):
data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)]
rdb.db_add(db, 1, data)
# substream 2 has points one hour apart
for i in range(0, 10000):
rdb.db_add(db, 2, [(i * 3600, 0, i * 3600)])
elif sys.argv[1] == '-r':
# test that we read back what we wrote
d = rdb.db_query(1, 0, 10000)
assert len(d) == 1
d = d[0].tolist()
assert len(d) == 10000
for i in xrange(0, 10000):
assert d[i][0] == i
assert d[i][1] == i
import time
from framework import SubGenerator_w
props = eval(sys.argv[1])
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')
rangestep = props.pop()
rangemax = props.pop()
rangemin = props.pop()
rebuildrange = range(rangemin, rangemax+1, rangestep)
roundgen = SubGenerator_w(*props, valid_values=rebuildrange)
completiontime = 0
overallstart = time.time()
for roundvals in roundgen: #for each list in subgen
for val in roundvals: #for each element in the list returned by subgen.next()
starttime = time.time()
rdb.db_add(a, val[0], [(val[1], 0, val[2])])
endtime = time.time()
completiontime += endtime - starttime
rdb.db_close(a)
print([overallstart, endtime, completiontime]) #put list on stdout for t_readingdb
sys.exit()
import readingdb as rdb
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')
rdb.db_add(a, 1, [(x, 0, x) for x in xrange(0, 100)])
print rdb.db_query(1, 0, 100, conn=a)
rdb.db_close(a)
import_map.append((int(ids[0]), int(id)))
for to_stream, from_stream in import_map:
print "starting %i <- %i" % (to_stream, from_stream)
first = True
vec = [(IMPORT_START,)]
t = tic()
data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0)
if not len(data):
continue
data = data[0]
print "received", data.shape
toc(t)
t = tic()
if opts.noop:
continue
bound = (int(data.shape[0]) / 100) + 1
for i in xrange(0, bound):
vec = (data[(i * 100) : (i * 100) + 100, :]).tolist()
# print time.ctime(vec[0][0])
rdb4.db_add(db1, to_stream, map(tuple, vec))
if len(vec) < 100:
break
print "inserted", to_stream
toc(t)
time.sleep(float(opts.delay))
# rdb4.db_sync(db4)
# rdb4.db_close(db0)
rdb4.db_close(db1)
line = re.sub('\#.*$', '', line)
ids = re.split('[ \t]+', line)
for id in ids[1:]:
import_map.append((int(ids[0]), int(id)))
for to_stream, from_stream in import_map:
print "starting %i <- %i" % (to_stream, from_stream)
first = True
vec = [(IMPORT_START,)]
t = tic()
data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0)
if not len(data): continue
data = data[0]
print "received", data.shape
toc(t)
t = tic()
if opts.noop: continue
bound = (int(data.shape[0]) / 100) + 1
for i in xrange(0, bound):
vec = (data[(i*100):(i*100) + 100, :]).tolist()
# print time.ctime(vec[0][0])
rdb4.db_add(db1, to_stream, map(tuple, vec))
if len(vec) < 100: break
print "inserted", to_stream
toc(t)
time.sleep(float(opts.delay))
# rdb4.db_sync(db4)
#rdb4.db_close(db0)
rdb4.db_close(db1)
def next(id, ref, n=1):
return rdb.db_next(id, ref, n=n, conn=db)[0].tolist()
def prev(id, ref, n=1, conn=db):
return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist()
S1MAX = 1000 * 100
if len(sys.argv) == 1:
print "%s [-a | -r | -n | -d | -c]" % sys.argv[0]
elif sys.argv[1] == '-a':
# substream 1 has every bucket filled
for i in range(0, 1000):
data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)]
rdb.db_add(db, 1, data)
# substream 2 has points one hour apart
for i in range(0, 10000):
rdb.db_add(db, 2, [(i * 3600, 0, i * 3600)])
elif sys.argv[1] == '-r':
# test that we read back what we wrote
d = rdb.db_query(1, 0, 10000)
assert len(d) == 1
d = d[0].tolist()
assert len(d) == 10000
for i in xrange(0, 10000):
assert d[i][0] == i
assert d[i][1] == i
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')
genprops = eval(sys.argv[1])
rangesetup = genprops.pop(2)
rangesetup[1] = rangesetup[1] + 1 #fix off by one
roundgen = SubGenerator_h(genprops[0], genprops[1], range(*rangesetup), genprops[2])
completiontime = 0
overallstart = time.time()
for roundvals in roundgen:
newvals = list(map(lambda x: (x[1], 0, x[2]), roundvals))
streamid = roundvals[0][0]
starttime = time.time()
rdb.db_add(a, streamid, newvals)
endtime = time.time()
completiontime += (endtime - starttime)
overallfinish = time.time()
timings = [overallstart, overallfinish, completiontime]
rdb.db_close(a)
print(timings)
sys.exit()
def infill_stream(self, stream):
for i in range(0, 1000):
data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)]
self.assertEqual(rdb.db_add(self.conn, stream, data), 1)
