def run(self):
"""Dequeue requests until we're done working"""
if self.host == None:
db = rdb4.db_open()
else:
db = rdb4.db_open(host=self.host[0], port=self.host[1])
while True:
try:
request = self.parent.requests.get_nowait()
except Queue.Empty:
break
if request.has_key("substream"):
rdb4.db_substream(db, request['substream'])
else:
rdb4.db_substream(db, 0)
first = True
result = []
last = []
while first or len(last) == 10000:
last = rdb4.db_query(db, int(request['streamid']),
int(request['starttime']),
int(request['endtime']))
first = False
print len(last)
result.extend(last)
if not self.full: break
if self.parent.as_numpy and len(result) > 0:
result = np.array(result)
result = result[:,[0,2]]
self.parent.returns[request['streamid']] = result
rdb4.db_close(db)
def run(self):
"""Dequeue requests until we're done working"""
if self.host == None:
db = rdb4.db_open()
else:
db = rdb4.db_open(host=self.host[0], port=self.host[1])
while True:
try:
request = self.parent.requests.get_nowait()
except Queue.Empty:
break
if request.has_key("substream"):
rdb4.db_substream(db, request['substream'])
else:
rdb4.db_substream(db, 0)
first = True
result = []
last = []
while first or len(last) == 10000:
last = rdb4.db_query(db, int(request['streamid']),
int(request['starttime']),
int(request['endtime']))
first = False
print len(last)
result.extend(last)
if not self.full: break
if self.parent.as_numpy and len(result) > 0:
result = np.array(result)
result = result[:, [0, 2]]
self.parent.returns[request['streamid']] = result
rdb4.db_close(db)
def tearDown(self):
rdb.db_close(self.conn)
self.db.terminate()
self.db.wait()
self.log.close()
for f in glob.glob(os.path.join(datadir, '*')):
os.remove(f)
os.removedirs(datadir)
def tearDown(self):
rdb.db_close(self.conn)
self.db.terminate()
self.db.wait()
self.log.close()
for f in glob.glob(os.path.join(datadir, '*')):
os.remove(f)
os.removedirs(datadir)
for f in sys.argv[2:]:
with open(f, 'r') as fp:
add_vec = []
streamid = int(f[f.rindex('.')+1:])
for line in fp.readlines():
parts = line.strip().split(',')
print parts
assert len(parts) == 2
tuple = (int(parts[0]), 0, float(parts[1]))
add_vec.append(tuple)
if len(add_vec) == 100:
rdb.db_add(db, streamid, add_vec)
add_vec = []
elif sys.argv[1] == '-m':
d = rdb.db_query([1, 2], 0, 100000000000, limit=-1)
d1, d2 = d
assert np.shape(d1) == (1e5, 2)
assert np.shape(d2) == (1e4, 2)
assert np.sum(d1[:, 0] - np.arange(0, 1e5)) == 0
assert np.sum(d1[:, 1] - np.arange(0, 1e5)) == 0
assert np.sum(d2[:, 0] - np.arange(0, 3600 * 1e4, 3600)) == 0
assert np.sum(d2[:, 1] - np.arange(0, 3600 * 1e4, 3600)) == 0
else:
print "invalid argument"
# for i in xrange(0, 1000):
# x = rdb.db_next(db, 1, i)
# assert x[0][0] == i+1
rdb.db_close(db)
#!/usr/bin/python
import readingdb as rdb
import sys
import time
rdb.db_setup("localhost", 4242)
a = rdb.db_open("localhost")
b = rdb.db_prev(1, 100000000000, conn=a)
lasttime = b[0][0][0]
rdb.db_close(a)
print(lasttime)
# ltime = file('tempfiles/lasttime', 'w')
# ltime.write(str(lasttime))
# ltime.close()
sys.exit()
import readingdb as rdb
rdb.db_setup('localhost', 4242)
a = rdb.db_open('localhost')
rdb.db_add(a, 1, [(x, 0, x) for x in xrange(0, 100)])
print rdb.db_query(1, 0, 100, conn=a)
rdb.db_close(a)
user=settings.DATABASE_USER,
)
rdb = readingdb.db_open()
loader = TsdbLoader()
loader.start()
for s in Stream.objects.all().order_by("id"):
if s.id <= 1:
next
first = True
data = []
startts = 0
extra = "source=%s path=%s" % (s.subscription, s.path())
print extra
while first or len(data) == 10000:
first = False
data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)
if len(data) > 0:
startts = data[-1][0]
loader.q.put({"method": "put", "id": s.id, "data": data, "extra": extra})
print startts
# print data
break
loader.q.put({"method": "quit"})
loader.join()
readingdb.db_close(rdb)
db.close()
import_map.append((int(ids[0]), int(id)))
for to_stream, from_stream in import_map:
print "starting %i <- %i" % (to_stream, from_stream)
first = True
vec = [(IMPORT_START,)]
t = tic()
data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0)
if not len(data):
continue
data = data[0]
print "received", data.shape
toc(t)
t = tic()
if opts.noop:
continue
bound = (int(data.shape[0]) / 100) + 1
for i in xrange(0, bound):
vec = (data[(i * 100) : (i * 100) + 100, :]).tolist()
# print time.ctime(vec[0][0])
rdb4.db_add(db1, to_stream, map(tuple, vec))
if len(vec) < 100:
break
print "inserted", to_stream
toc(t)
time.sleep(float(opts.delay))
# rdb4.db_sync(db4)
# rdb4.db_close(db0)
rdb4.db_close(db1)
loader.start()
for s in Stream.objects.all().order_by('id'):
if s.id <= 1:
next
first = True
data = []
startts = 0
extra = 'source=%s path=%s' % (s.subscription, s.path())
print extra
while first or len(data) == 10000:
first = False
data = readingdb.db_query(rdb, s.id, startts, 2 ** 31)
if len(data) > 0:
startts = data[-1][0]
loader.q.put({'method': 'put',
'id': s.id,
'data': data,
'extra': extra})
print startts
# print data
break
loader.q.put({'method': 'quit'})
loader.join()
readingdb.db_close(rdb)
db.close()
line = re.sub('\#.*$', '', line)
ids = re.split('[ \t]+', line)
for id in ids[1:]:
import_map.append((int(ids[0]), int(id)))
for to_stream, from_stream in import_map:
print "starting %i <- %i" % (to_stream, from_stream)
first = True
vec = [(IMPORT_START,)]
t = tic()
data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0)
if not len(data): continue
data = data[0]
print "received", data.shape
toc(t)
t = tic()
if opts.noop: continue
bound = (int(data.shape[0]) / 100) + 1
for i in xrange(0, bound):
vec = (data[(i*100):(i*100) + 100, :]).tolist()
# print time.ctime(vec[0][0])
rdb4.db_add(db1, to_stream, map(tuple, vec))
if len(vec) < 100: break
print "inserted", to_stream
toc(t)
time.sleep(float(opts.delay))
# rdb4.db_sync(db4)
#rdb4.db_close(db0)
rdb4.db_close(db1)
for f in sys.argv[2:]:
with open(f, 'r') as fp:
add_vec = []
streamid = int(f[f.rindex('.') + 1:])
for line in fp.readlines():
parts = line.strip().split(',')
print parts
assert len(parts) == 2
tuple = (int(parts[0]), 0, float(parts[1]))
add_vec.append(tuple)
if len(add_vec) == 100:
rdb.db_add(db, streamid, add_vec)
add_vec = []
elif sys.argv[1] == '-m':
d = rdb.db_query([1, 2], 0, 100000000000, limit=-1)
d1, d2 = d
assert np.shape(d1) == (1e5, 2)
assert np.shape(d2) == (1e4, 2)
assert np.sum(d1[:, 0] - np.arange(0, 1e5)) == 0
assert np.sum(d1[:, 1] - np.arange(0, 1e5)) == 0
assert np.sum(d2[:, 0] - np.arange(0, 3600 * 1e4, 3600)) == 0
assert np.sum(d2[:, 1] - np.arange(0, 3600 * 1e4, 3600)) == 0
else:
print "invalid argument"
# for i in xrange(0, 1000):
# x = rdb.db_next(db, 1, i)
# assert x[0][0] == i+1
rdb.db_close(db)