def __get(filenames): for n in filenames: with open(n, "rb") as f: try: for o, txn in read_structs(f.read()): yield txn[0].orgId except error: pass
def run(kanso_filenames, events): d_print("Update server is started.") log.info("start") # DANGEROUS: Could crash updater updater = IndexServerUpdater(config.IDX_FILENAME) ks = kstream(config.KANSO_FILENAME) ks2 = kstream(config.KANSO_FILENAME2) offset = updater.offset offset2 = updater.offset2 snapshot_manager = Snapshot(".", config.IDX_FILENAME, config.IDX_SNAPSHOT_DIR) while not events.stop.isSet(): start = datetime.now() events.endupdate.clear() try: # Important note: # --------------- # We guess that records (transactions in our case) stored in KANSO does not cross # KANSO chunk border and it is because of KANSO atomic writes. for b in ks.read(offset): trydb(updater.begin, "db-begin:%s") try: for inner_offset, txn in read_structs(b): trydb(updater.insert_record, "db-insert:%s", (offset + inner_offset, txn)) if events.stop.isSet(): break except error, e: log.warning("read_structs:%s" % e) trydb(updater.commit, "db-commit:%s") offset = updater.offset for b in ks2.read(offset2): trydb(updater.begin, "db-begin:%s") try: for inner_offset, txn in read_structs(b): trydb(updater.insert_record2, "db-insert:%s", (offset2 + inner_offset, txn)) if events.stop.isSet(): break except error, e: log.warning("read_structs:%s" % e) trydb(updater.commit, "db-commit:%s") offset2 = updater.offset2
import fake_kanso from kstream import * from IndexServer import * from read_structs import read_structs if __name__ == '__main__': fake_kanso.run() import datetime start = datetime.datetime.now() updater = IndexServerUpdater("index.sqlite") offset = 0 ks = kstream("kanso://127.0.0.1/pfr/test/transactions") while fake_kanso.append(): inner_offset = 0 try: for txn in read_structs(ks.read(offset)): updater.insert_record(txn) inner_offset = txn[0] except: pass offset += inner_offset inner_offset = 0 updater.commit() print str(datetime.datetime.now() - start)
l = len(bytes) for i in xrange(0, l-3, 4): h ^= unpack_from("=i", bytes, i)[0] if l % 4 == 1: h ^= unpack_from("=i", bytes+"\x00\x00\x00", l-1)[0] if l % 4 == 2: h ^= unpack_from("=i", bytes+"\x00\x00", l-2)[0] if l % 4 == 3: h ^= unpack_from("=i", bytes+"\x00", l-3)[0] return h def write_struct(head, doc, sign): b = __write_head(head) + __write_doc(doc) + __write_sign(sign) b = b + pack("=i", __hash(b)) return b # for testing, remove this if __name__ == "__main__": txnHead = TxnHead(uuid4(), uuid4(), 0, 1, "1234567", 2000, 2, 3) doc = Doc(uuid4(), 1, 2, 3, "preved", 10101010, 777) sign = Sign(uuid4(), uuid4(), 10, 20202020, 888) bin = open("bin", "ab") b = write_struct(txnHead, doc, sign) bin.write(b) bin.close() bin = open("bin", "rb") for t in read_structs(bin.read()): print t