def __get(filenames):
for n in filenames:
with open(n, "rb") as f:
try:
for o, txn in read_structs(f.read()):
yield txn[0].orgId
except error:
pass
def run(kanso_filenames, events):
d_print("Update server is started.")
log.info("start")
# DANGEROUS: Could crash updater
updater = IndexServerUpdater(config.IDX_FILENAME)
ks = kstream(config.KANSO_FILENAME)
ks2 = kstream(config.KANSO_FILENAME2)
offset = updater.offset
offset2 = updater.offset2
snapshot_manager = Snapshot(".", config.IDX_FILENAME, config.IDX_SNAPSHOT_DIR)
while not events.stop.isSet():
start = datetime.now()
events.endupdate.clear()
try:
# Important note:
# ---------------
# We guess that records (transactions in our case) stored in KANSO does not cross
# KANSO chunk border and it is because of KANSO atomic writes.
for b in ks.read(offset):
trydb(updater.begin, "db-begin:%s")
try:
for inner_offset, txn in read_structs(b):
trydb(updater.insert_record, "db-insert:%s", (offset + inner_offset, txn))
if events.stop.isSet():
break
except error, e:
log.warning("read_structs:%s" % e)
trydb(updater.commit, "db-commit:%s")
offset = updater.offset
for b in ks2.read(offset2):
trydb(updater.begin, "db-begin:%s")
try:
for inner_offset, txn in read_structs(b):
trydb(updater.insert_record2, "db-insert:%s", (offset2 + inner_offset, txn))
if events.stop.isSet():
break
except error, e:
log.warning("read_structs:%s" % e)
trydb(updater.commit, "db-commit:%s")
offset2 = updater.offset2
import fake_kanso
from kstream import *
from IndexServer import *
from read_structs import read_structs
if __name__ == '__main__':
fake_kanso.run()
import datetime
start = datetime.datetime.now()
updater = IndexServerUpdater("index.sqlite")
offset = 0
ks = kstream("kanso://127.0.0.1/pfr/test/transactions")
while fake_kanso.append():
inner_offset = 0
try:
for txn in read_structs(ks.read(offset)):
updater.insert_record(txn)
inner_offset = txn[0]
except: pass
offset += inner_offset
inner_offset = 0
updater.commit()
print str(datetime.datetime.now() - start)
l = len(bytes)
for i in xrange(0, l-3, 4):
h ^= unpack_from("=i", bytes, i)[0]
if l % 4 == 1:
h ^= unpack_from("=i", bytes+"\x00\x00\x00", l-1)[0]
if l % 4 == 2:
h ^= unpack_from("=i", bytes+"\x00\x00", l-2)[0]
if l % 4 == 3:
h ^= unpack_from("=i", bytes+"\x00", l-3)[0]
return h
def write_struct(head, doc, sign):
b = __write_head(head) + __write_doc(doc) + __write_sign(sign)
b = b + pack("=i", __hash(b))
return b
# for testing, remove this
if __name__ == "__main__":
txnHead = TxnHead(uuid4(), uuid4(), 0, 1, "1234567", 2000, 2, 3)
doc = Doc(uuid4(), 1, 2, 3, "preved", 10101010, 777)
sign = Sign(uuid4(), uuid4(), 10, 20202020, 888)
bin = open("bin", "ab")
b = write_struct(txnHead, doc, sign)
bin.write(b)
bin.close()
bin = open("bin", "rb")
for t in read_structs(bin.read()):
print t