def persist_questions(self, question_list_per_topic): insert_sql = "INSERT IGNORE INTO ZHIHU_QUESTION " \ "(QUESTION_ID, QUESTION_TITLE, ANSWER, IS_TOP_QUESTION, CREATED_TIME) " \ "VALUES (%s, %s, %s, %s, %s)" tm = TransactionManager() tm.execute_many_sql(insert_sql, question_list_per_topic) tm.close_connection()
def generate_question_id_list(self, last_visit): question_id_list = [] if self.is_develop_mode(): return get_question_id_list() available_ids = generate_available_ids(MAX_QUESTION_TABLE_ID, QUESTION_ID_STEP) available_id_list = available_ids.split(',') import math loop = int(math.ceil(float(len(available_id_list))/AVAIL_ID_SIZE_THRESHOLD)) print "......loop:%s" % loop i = 0 pre_sql = None tm = TransactionManager() while i < loop: begin_index = i * AVAIL_ID_SIZE_THRESHOLD end_index = (i + 1) * AVAIL_ID_SIZE_THRESHOLD sql = "SELECT QUESTION_ID FROM (select @index:=@index+1 as ID, QUESTION_ID, LAST_VISIT from ZHIHU_QUESTION_ID) AS q WHERE timestamp(q.LAST_VISIT) < timestamp('%s')" % last_visit sql += " AND ID IN (%s) " % ",".join(available_id_list[begin_index:end_index]) if i == 1: pre_sql = "SET @index=0;" results = tm.execute_sql(sql, pre_sql) for row in results: question_id_list.append(str(row[0])) i += 1 tm.close_connection() return question_id_list
def worker_thread2(ftf, connection): tm = TransactionManager() tm.resolve_missing_transactions(tm2.get_top_hash(), sm.get_transactions(ftf, connection)) self.assertEqual(tm.get_transaction(2)['data'], 'foo5') self.assertEqual(tm.get_transaction(3)['data'], 'foo2') self.assertEqual(tm.get_transaction(4)['data'], 'foo3') self.assertEqual(tm.get_transaction(5)['data'], 'foo4') passes.append(True)
def get_proxy_from_db(): select_sql = "SELECT PROXY_IP FROM ZHIHU_PROXY ORDER BY RAND() LIMIT 1" tm = TransactionManager() results = tm.execute_sql(select_sql) tm.close_connection() if len(results) == 0: return "" for row in results: return str(row[0])
def persist_topics(topic_list): """ Persist topics into mysql :param topic_list: all the topics including level 1 and level 2. :return: None """ insert_sql = "INSERT IGNORE INTO ZHIHU_TOPIC (TOPIC_ID, NAME, PARENT_ID) \ VALUES (%s, %s, %s)" print "insert sql:%s" % insert_sql tm = TransactionManager() tm.execute_many_sql(insert_sql, topic_list) tm.close_connection()
def main(): """ Main entry point for the application """ if validate_args(): # Confirm time with professor transaction_manager = TransactionManager(LOGGER) parser = Parser(sys.argv[1], LOGGER) instruction = parser.get_instruction() while instruction: transaction_manager.execute(instruction) instruction = parser.get_instruction()
def main(): txs = get_transactions() tx_manager = TransactionManager(txs) sup: int = int(input('Enter min support: ')) freq_items = apriori(sup, tx_manager) print_freq_items(freq_items)
def apriori(min_support: int, transactionManager: TransactionManager, freq_items: List[FrequentItems] = []) -> List[FrequentItems]: if freq_items == []: unique_items = transactionManager.get_unique_items() first_set_items = [frozenset([frozenset([i]) for i in unique_items])] return apriori(min_support, transactionManager, first_set_items) sup_validator = lambda tx: transactionManager.get_support_count( tx) >= min_support next_freq_items = make_next_freq_items(freq_items[-1], sup_validator) if next_freq_items == frozenset(): return freq_items new_freq_items = freq_items + [next_freq_items] return apriori(min_support, transactionManager, new_freq_items)
def test_logic(): tm1 = TransactionManager("test_tm.pickle") tm1._clear() tm1.add_transaction("foo1", 1, "bubba", "gump") tm1.add_transaction("foo2", 2, "bubba", "gump") sm1 = StatusManager(self.reactor, self._get_fake_multicast_ping(), tm1, 32825) tm2 = TransactionManager("test_tm2.pickle") tm2._clear() sm2 = StatusManager(self.reactor, self._get_fake_multicast_ping(), tm2, 32826) message = "offer_state %s %s" % (tm1.get_top_hash(), sm1.comm_port) sm2.check_state_update(message, "localhost") self._wait_on_check_state_update_to_finish(sm2) self.assertEqual(tm2.get_current_id(), 2) self.reactor.callFromThread(self.reactor.stop)
def test_transaction_resolution_same_ts_diff_data(self): self.tm.add_transaction("foo0", 1, "bubba", "gump") tm2 = TransactionManager("test_tm.pickle") tm2._clear() tm2.add_transaction("foo1", 1, "bubba", "gump") tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old()) self.assertEqual(tm2.current_id, 2)
def get_level2_topic_id_list(last_visit_date, is_develop=False): level2_topic_id_list = [] sql = "SELECT TOPIC_ID FROM ZHIHU_TOPIC WHERE TOPIC_ID != PARENT_ID AND LAST_VISIT < '%s'" \ % last_visit_date available_topic_ids = generate_available_topic_ids(MAX_TOPIC_TABLE_ID, TOPIC_ID_STEP) sql += " AND ID IN (%s) " % available_topic_ids if is_develop: sql += " LIMIT 2" print "......execute sql:%s" % sql tm = TransactionManager() results = tm.execute_sql(sql) tm.close_connection() for row in results: level2_topic_id_list.append(str(row[0])) return level2_topic_id_list
def test_transaction_resolution_multiple_same_timestamp(self): self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1") self.tm.add_transaction("foo1", 1, "bubba", "gump", uuid="2") self.tm.add_transaction("foo2", 1, "bubba", "gump", uuid="3") tm2 = TransactionManager("test_tm.pickle") tm2._clear() tm2.add_transaction("foo1", 1, "bubba", "gump", uuid="2") tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old()) self.assertEqual(tm2.current_id, 3) self.assertEqual(self.tm.top_hash, tm2.top_hash)
class TestTransactionManager(TestCase): def setUp(self): self.tm = TransactionManager() self.tm._clear() def test_init(self): tm = self.tm self.assertEqual(tm.current_id, 0) self.assertEqual(tm.top_hash, None) def test_bubble_up(self): self.tm.set_val(0, "hi") self.tm.set_val(1, "there") self.tm.set_val(2, "buddy") self.tm.current_id = 2 self.tm._bubble_up(1) self.assertEqual(self.tm.get_val(3), "buddy") self.assertEqual(self.tm.get_val(2), "there") self.assertEqual(self.tm.get_val(0), "hi") self.tm.current_id = 3 self.tm._bubble_up(3) self.assertEqual(self.tm.get_val(4), "buddy") def test_add_transaction(self): self.tm.add_transaction("foo", 12345, "bubba", "gump") self.assertEqual(self.tm.current_id, 1) item = self.tm.get_transaction(1) self.assertEqual(item["data"], "foo") self.assertEqual(item["ts"], 12345) self.assertEqual(item["user"], "bubba") self.assertEqual(item["signature"], "gump") self.assertIsNotNone(item["hash"]) self.tm.add_transaction("fooz", 12349, "bubbas", "gumps") old_top_hash = self.tm.top_hash self.tm.add_transaction("middle", 12346, "blubba", "gumple") bottom = self.tm.get_transaction(1) self.assertEqual(bottom["data"], "foo") middle = self.tm.get_transaction(2) self.assertEqual(middle["data"], "middle") top = self.tm.get_transaction(3) self.assertEqual(top["data"], "fooz") self.assertNotEqual(self.tm.top_hash, old_top_hash) def test_transaction_resolution(self): self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1") self.tm.add_transaction("foo1", 2, "bubba", "gump", uuid="2") self.tm.add_transaction("foo2", 3, "bubba", "gump", uuid="3") self.tm.add_transaction("foo3", 4, "bubba", "gump", uuid="4") self.tm.add_transaction("foo4", 5, "bubba", "gump", uuid="5") tm2 = TransactionManager("test_tm.pickle") tm2._clear() tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old()) self.assertEqual(tm2.current_id, 5) for i in xrange(0, 5): self.assertEqual(tm2.get_transaction(i + 1)["data"], "foo%s" % i) tm2 = TransactionManager("test_tm.pickle") tm2._clear() tm2.add_transaction("foo0", 1, "bubba", "gump", uuid="1") tm2.add_transaction("foo1", 2, "bubba", "gump", uuid="2") tm2.add_transaction("foo3", 4, "bubba", "gump", uuid="4") tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old()) self.assertEqual(tm2.current_id, 5) for i in xrange(0, 5): self.assertEqual(tm2.get_transaction(i + 1)["data"], "foo%s" % i) def test_transaction_resolution_multiple_same_timestamp(self): self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1") self.tm.add_transaction("foo1", 1, "bubba", "gump", uuid="2") self.tm.add_transaction("foo2", 1, "bubba", "gump", uuid="3") tm2 = TransactionManager("test_tm.pickle") tm2._clear() tm2.add_transaction("foo1", 1, "bubba", "gump", uuid="2") tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old()) self.assertEqual(tm2.current_id, 3) self.assertEqual(self.tm.top_hash, tm2.top_hash) def test_transaction_resolution_same_ts_diff_data(self): self.tm.add_transaction("foo0", 1, "bubba", "gump") tm2 = TransactionManager("test_tm.pickle") tm2._clear() tm2.add_transaction("foo1", 1, "bubba", "gump") tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old()) self.assertEqual(tm2.current_id, 2) def test_current_id_loads(self): self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1") tm = TransactionManager() self.assertEqual(tm.current_id, 1)
def test_get_transactions(self): class FakeTransactionFetcher(object): def request_transactions(self): pass class FakeConnection(object): def addCallback(self, func): pass ftf = FakeTransactionFetcher() sm = StatusManager(None, self._get_fake_multicast_ping()) tm2 = TransactionManager('test_tm.pickle') tm2._clear() tm2.add_transaction("foo1", 1, "bubba", "gump") tm2.add_transaction("foo2", 2, "bubba", "gump") passes = [] def worker_thread(ftf, connection): tm = TransactionManager() tm._clear() tm.resolve_missing_transactions(tm2.get_top_hash(), sm.get_transactions(ftf, connection)) self.assertEqual(tm.get_transaction(1)['data'], 'foo1') self.assertEqual(tm.get_transaction(2)['data'], 'foo2') passes.append(True) t = threading.Thread(target=worker_thread, args=[ftf, FakeConnection()]) t.daemon = True t.start() self._wait_for_callback_to_be_set(ftf) for data in tm2.get_transactions_recent_to_old(2): ftf.call_back(data) ftf.call_back("done") t.join() self.assertEqual(len(passes), 1) tm2.add_transaction("foo3", 3, "bubba", "gump") tm2.add_transaction("foo4", 4, "bubba", "gump") tm2.add_transaction("foo5", 1.5, "bubba", "gump") def worker_thread2(ftf, connection): tm = TransactionManager() tm.resolve_missing_transactions(tm2.get_top_hash(), sm.get_transactions(ftf, connection)) self.assertEqual(tm.get_transaction(2)['data'], 'foo5') self.assertEqual(tm.get_transaction(3)['data'], 'foo2') self.assertEqual(tm.get_transaction(4)['data'], 'foo3') self.assertEqual(tm.get_transaction(5)['data'], 'foo4') passes.append(True) t = threading.Thread(target=worker_thread2, args=[ftf, FakeConnection()]) t.daemon = True del ftf.call_back t.start() self._wait_for_callback_to_be_set(ftf) for data in tm2.get_transactions_recent_to_old(2): ftf.call_back(data) time.sleep(0.05) ftf.call_back("done") t.join() self.assertEqual(len(passes), 2)
def setUp(self): self.tm = TransactionManager() self.tm._clear()
from paxos_manager import PaxosManager from transaction_manager import TransactionManager from util import safe_print, ConnectGraph if len(sys.argv) < 3: raise Exception('Wrong arguments. Correct Usage: python server.py <config_file_path> <server_index> <optional_dump_path>') else: file = io.open(sys.argv[1]) configJson = json.load(file) serverI = int(sys.argv[2]) % len(configJson) localConfig = configJson[serverI] serverIdList = map(lambda x: x['id'], configJson) val = serverI connectGraph = ConnectGraph(configJson) transactionManager = TransactionManager(localConfig['id']) paxosManager = PaxosManager(configJson, serverI, transactionManager, connectGraph) serverSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) serverSocket.bind((localConfig['local_ip_addr'], int(localConfig['port']))) print "Server {0} listening on port {1}".format(str(localConfig['id']), str(localConfig['port'])) if len(sys.argv) == 4: # load paxosManager from last saved paxosManager.initializeFromJSON(json.load(open(sys.argv[3]))) if len(transactionManager.getQueue()) > 0: # if there was a transaction before the crash, start the save attempt thread paxosManager.attempt_save_timeout_refresh() def listenRequests(): while True: data, addr = serverSocket.recvfrom(1024)
import argparse from block import Block from block_chain import BlockChain from serializer import Serializer from transaction_manager import TransactionManager # flask init app = Flask(__name__) app.secret_key = 'super secret key' # objects init mutex = Lock() s = Serializer() block_chain = BlockChain() transaction_manager = TransactionManager() # create genesis block genesis_block = Block(index=0, timestamp=timer(), transactions=transaction_manager.get_transactions(), previous_hash="genesis_block", difficulty=block_chain.get_difficulty()) block_chain.set_working_block(genesis_block) @app.route('/', methods=['GET']) def main(): return block_chain.to_html()
import psycopg2 from transaction_manager import TransactionManager tm = TransactionManager(user="******", password="******", host="localhost", port="5432", verbose=False) tm.add_connection("fly_booking", database="fly_booking_db") tm.add_connection("hotel_booking", database="hotel_booking_db") tm.add_connection("account", database="account_db") def decrease_money(conn, money): account_cursor = conn.cursor() account_query = "UPDATE account SET amount = amount - %s WHERE client_name = 'Roman Vey';" account_data = (money, ) account_cursor.execute(account_query, account_data) def book_hotel(conn): hotel_cursor = conn.cursor() hotel_query = "INSERT INTO hotel_booking (client_name, hotel_name, arrival, departure) VALUES (%s, %s, NOW(), NOW());" hotel_data = ("Roman Vey", "Nice Hotel") hotel_cursor.execute(hotel_query, hotel_data) tm.create_transaction("book_hotel", ["hotel_booking", "account"]) tm.add_to_transaction("book_hotel", "hotel_booking", book_hotel,
app.secret_key = os.urandom(24) login_manager = LoginManager() login_manager.init_app(app) # Generate a globally unique address for this node node_identifier = str(uuid4()).replace('-', '') # Instantiate the Blockchain blockchain = Blockchain() # Create dummy users with ids 1 to 5 users = [User(id, node_identifier) for id in range(1, 6)] # Instantiate the transaction manager transaction_manager = TransactionManager(users) @app.route('/mine', methods=['GET']) def mine(): # We run the proof of work algorithm to get the next proof... last_block = blockchain.last_block proof = blockchain.proof_of_work(last_block) # We must receive a reward for finding the proof. # The sender is "0" to signify that this node has mined a new coin. blockchain.new_transaction( sender="0", recipient=node_identifier, amount=1, )
from transaction_manager import TransactionManager import piecash GNUCASH_FILE = "example_file.gnucash" # book = piecash.open_book(GNUCASH_FILE, readonly=True) # print(book.default_currency) book = TransactionManager(GNUCASH_FILE) a1 = book.get_account("Assets") a2 = book.get_account("Expence") book.make_transaction(a1, a2, 20, description="Test transaction") print(a1)
def __init__(self, filename): self._fp = open(filename, 'r') self._tm = TransactionManager() self._run()
class Neo: def __init__(self, filename): self._fp = open(filename, 'r') self._tm = TransactionManager() self._run() def _run(self): for line in self._fp.readlines(): line = line.strip() if line: self._tm.inc_ts() commands = line.split(';') for c in commands: clist = c.strip().strip(')') clist = clist.split('(') method = clist[0] args = clist[1].split(',') try: _method = getattr(self, method) _method(*args) except Exception as e: print 'Unknown method, exiting Neo...', traceback.format_exc() sys.exit(1) def begin(self, txn_id): print '---------------------------------------------------------\n' print 'begin: ', txn_id try: self._tm.begin(txn_id.strip()) except Exception as e: print traceback.format_exc() def beginRO(self, txn_id): print '---------------------------------------------------------\n' print 'beginRO: ', txn_id try: self._tm.beginRO(txn_id.strip()) except Exception as e: print traceback.format_exc() def W(self, txn_id, var, value): print '---------------------------------------------------------\n' print 'W: ', txn_id, var, value try: self._tm.write(txn_id.strip(), var.strip(), value.strip()) except Exception as e: print traceback.format_exc() def R(self, txn_id, var): print '---------------------------------------------------------\n' print 'R: ', txn_id, var try: self._tm.read(txn_id.strip(), var.strip()) except Exception as e: print traceback.format_exc() def end(self, txn_id): print '---------------------------------------------------------\n' print 'end: ', txn_id self._tm.end(txn_id.strip()) def dump(self, arg): print '---------------------------------------------------------\n' print 'dump: ', arg if arg: try: site_id = int(arg.strip()) pprint(self._tm.dump(site_id=site_id)) except ValueError: pprint(self._tm.dump(var_id=arg.strip())) else: pprint(self._tm.dump()) def fail(self, site_id): print '---------------------------------------------------------\n' print 'fail: ', site_id self._tm.fail(int(site_id.strip())) def recover(self, site_id): print '---------------------------------------------------------\n' print 'recover: ', site_id self._tm.recover(int(site_id.strip()))
from transaction_manager import TransactionManager tx_manager = TransactionManager()
def test_transaction_resolution(self): self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1") self.tm.add_transaction("foo1", 2, "bubba", "gump", uuid="2") self.tm.add_transaction("foo2", 3, "bubba", "gump", uuid="3") self.tm.add_transaction("foo3", 4, "bubba", "gump", uuid="4") self.tm.add_transaction("foo4", 5, "bubba", "gump", uuid="5") tm2 = TransactionManager("test_tm.pickle") tm2._clear() tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old()) self.assertEqual(tm2.current_id, 5) for i in xrange(0, 5): self.assertEqual(tm2.get_transaction(i + 1)["data"], "foo%s" % i) tm2 = TransactionManager("test_tm.pickle") tm2._clear() tm2.add_transaction("foo0", 1, "bubba", "gump", uuid="1") tm2.add_transaction("foo1", 2, "bubba", "gump", uuid="2") tm2.add_transaction("foo3", 4, "bubba", "gump", uuid="4") tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old()) self.assertEqual(tm2.current_id, 5) for i in xrange(0, 5): self.assertEqual(tm2.get_transaction(i + 1)["data"], "foo%s" % i)
def update_level2_topic_timestamp(level2_topic_id): sql = "UPDATE ZHIHU_TOPIC SET LAST_VISIT = '%s' WHERE TOPIC_ID = %s" % \ (get_current_timestamp(), level2_topic_id) tm = TransactionManager() tm.execute_sql(sql) tm.close_connection()