def persist_questions(self, question_list_per_topic):
insert_sql = "INSERT IGNORE INTO ZHIHU_QUESTION " \
"(QUESTION_ID, QUESTION_TITLE, ANSWER, IS_TOP_QUESTION, CREATED_TIME) " \
"VALUES (%s, %s, %s, %s, %s)"
tm = TransactionManager()
tm.execute_many_sql(insert_sql, question_list_per_topic)
tm.close_connection()
def generate_question_id_list(self, last_visit):
question_id_list = []
if self.is_develop_mode():
return get_question_id_list()
available_ids = generate_available_ids(MAX_QUESTION_TABLE_ID, QUESTION_ID_STEP)
available_id_list = available_ids.split(',')
import math
loop = int(math.ceil(float(len(available_id_list))/AVAIL_ID_SIZE_THRESHOLD))
print "......loop:%s" % loop
i = 0
pre_sql = None
tm = TransactionManager()
while i < loop:
begin_index = i * AVAIL_ID_SIZE_THRESHOLD
end_index = (i + 1) * AVAIL_ID_SIZE_THRESHOLD
sql = "SELECT QUESTION_ID FROM (select @index:=@index+1 as ID, QUESTION_ID, LAST_VISIT from ZHIHU_QUESTION_ID) AS q WHERE timestamp(q.LAST_VISIT) < timestamp('%s')" % last_visit
sql += " AND ID IN (%s) " % ",".join(available_id_list[begin_index:end_index])
if i == 1:
pre_sql = "SET @index=0;"
results = tm.execute_sql(sql, pre_sql)
for row in results:
question_id_list.append(str(row[0]))
i += 1
tm.close_connection()
return question_id_list
def worker_thread2(ftf, connection):
tm = TransactionManager()
tm.resolve_missing_transactions(tm2.get_top_hash(), sm.get_transactions(ftf, connection))
self.assertEqual(tm.get_transaction(2)['data'], 'foo5')
self.assertEqual(tm.get_transaction(3)['data'], 'foo2')
self.assertEqual(tm.get_transaction(4)['data'], 'foo3')
self.assertEqual(tm.get_transaction(5)['data'], 'foo4')
passes.append(True)
def get_proxy_from_db():
select_sql = "SELECT PROXY_IP FROM ZHIHU_PROXY ORDER BY RAND() LIMIT 1"
tm = TransactionManager()
results = tm.execute_sql(select_sql)
tm.close_connection()
if len(results) == 0:
return ""
for row in results:
return str(row[0])
def persist_topics(topic_list):
"""
Persist topics into mysql
:param topic_list: all the topics including level 1 and level 2.
:return: None
"""
insert_sql = "INSERT IGNORE INTO ZHIHU_TOPIC (TOPIC_ID, NAME, PARENT_ID) \
VALUES (%s, %s, %s)"
print "insert sql:%s" % insert_sql
tm = TransactionManager()
tm.execute_many_sql(insert_sql, topic_list)
tm.close_connection()
def main():
""" Main entry point for the application """
if validate_args():
# Confirm time with professor
transaction_manager = TransactionManager(LOGGER)
parser = Parser(sys.argv[1], LOGGER)
instruction = parser.get_instruction()
while instruction:
transaction_manager.execute(instruction)
instruction = parser.get_instruction()
def main():
txs = get_transactions()
tx_manager = TransactionManager(txs)
sup: int = int(input('Enter min support: '))
freq_items = apriori(sup, tx_manager)
print_freq_items(freq_items)
def apriori(min_support: int,
transactionManager: TransactionManager,
freq_items: List[FrequentItems] = []) -> List[FrequentItems]:
if freq_items == []:
unique_items = transactionManager.get_unique_items()
first_set_items = [frozenset([frozenset([i]) for i in unique_items])]
return apriori(min_support, transactionManager, first_set_items)
sup_validator = lambda tx: transactionManager.get_support_count(
tx) >= min_support
next_freq_items = make_next_freq_items(freq_items[-1], sup_validator)
if next_freq_items == frozenset():
return freq_items
new_freq_items = freq_items + [next_freq_items]
return apriori(min_support, transactionManager, new_freq_items)
def test_logic():
tm1 = TransactionManager("test_tm.pickle")
tm1._clear()
tm1.add_transaction("foo1", 1, "bubba", "gump")
tm1.add_transaction("foo2", 2, "bubba", "gump")
sm1 = StatusManager(self.reactor, self._get_fake_multicast_ping(), tm1, 32825)
tm2 = TransactionManager("test_tm2.pickle")
tm2._clear()
sm2 = StatusManager(self.reactor, self._get_fake_multicast_ping(), tm2, 32826)
message = "offer_state %s %s" % (tm1.get_top_hash(), sm1.comm_port)
sm2.check_state_update(message, "localhost")
self._wait_on_check_state_update_to_finish(sm2)
self.assertEqual(tm2.get_current_id(), 2)
self.reactor.callFromThread(self.reactor.stop)
def test_transaction_resolution_same_ts_diff_data(self):
self.tm.add_transaction("foo0", 1, "bubba", "gump")
tm2 = TransactionManager("test_tm.pickle")
tm2._clear()
tm2.add_transaction("foo1", 1, "bubba", "gump")
tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old())
self.assertEqual(tm2.current_id, 2)
def get_level2_topic_id_list(last_visit_date, is_develop=False):
level2_topic_id_list = []
sql = "SELECT TOPIC_ID FROM ZHIHU_TOPIC WHERE TOPIC_ID != PARENT_ID AND LAST_VISIT < '%s'" \
% last_visit_date
available_topic_ids = generate_available_topic_ids(MAX_TOPIC_TABLE_ID, TOPIC_ID_STEP)
sql += " AND ID IN (%s) " % available_topic_ids
if is_develop:
sql += " LIMIT 2"
print "......execute sql:%s" % sql
tm = TransactionManager()
results = tm.execute_sql(sql)
tm.close_connection()
for row in results:
level2_topic_id_list.append(str(row[0]))
return level2_topic_id_list
def test_transaction_resolution_multiple_same_timestamp(self):
self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1")
self.tm.add_transaction("foo1", 1, "bubba", "gump", uuid="2")
self.tm.add_transaction("foo2", 1, "bubba", "gump", uuid="3")
tm2 = TransactionManager("test_tm.pickle")
tm2._clear()
tm2.add_transaction("foo1", 1, "bubba", "gump", uuid="2")
tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old())
self.assertEqual(tm2.current_id, 3)
self.assertEqual(self.tm.top_hash, tm2.top_hash)
class TestTransactionManager(TestCase):
def setUp(self):
self.tm = TransactionManager()
self.tm._clear()
def test_init(self):
tm = self.tm
self.assertEqual(tm.current_id, 0)
self.assertEqual(tm.top_hash, None)
def test_bubble_up(self):
self.tm.set_val(0, "hi")
self.tm.set_val(1, "there")
self.tm.set_val(2, "buddy")
self.tm.current_id = 2
self.tm._bubble_up(1)
self.assertEqual(self.tm.get_val(3), "buddy")
self.assertEqual(self.tm.get_val(2), "there")
self.assertEqual(self.tm.get_val(0), "hi")
self.tm.current_id = 3
self.tm._bubble_up(3)
self.assertEqual(self.tm.get_val(4), "buddy")
def test_add_transaction(self):
self.tm.add_transaction("foo", 12345, "bubba", "gump")
self.assertEqual(self.tm.current_id, 1)
item = self.tm.get_transaction(1)
self.assertEqual(item["data"], "foo")
self.assertEqual(item["ts"], 12345)
self.assertEqual(item["user"], "bubba")
self.assertEqual(item["signature"], "gump")
self.assertIsNotNone(item["hash"])
self.tm.add_transaction("fooz", 12349, "bubbas", "gumps")
old_top_hash = self.tm.top_hash
self.tm.add_transaction("middle", 12346, "blubba", "gumple")
bottom = self.tm.get_transaction(1)
self.assertEqual(bottom["data"], "foo")
middle = self.tm.get_transaction(2)
self.assertEqual(middle["data"], "middle")
top = self.tm.get_transaction(3)
self.assertEqual(top["data"], "fooz")
self.assertNotEqual(self.tm.top_hash, old_top_hash)
def test_transaction_resolution(self):
self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1")
self.tm.add_transaction("foo1", 2, "bubba", "gump", uuid="2")
self.tm.add_transaction("foo2", 3, "bubba", "gump", uuid="3")
self.tm.add_transaction("foo3", 4, "bubba", "gump", uuid="4")
self.tm.add_transaction("foo4", 5, "bubba", "gump", uuid="5")
tm2 = TransactionManager("test_tm.pickle")
tm2._clear()
tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old())
self.assertEqual(tm2.current_id, 5)
for i in xrange(0, 5):
self.assertEqual(tm2.get_transaction(i + 1)["data"], "foo%s" % i)
tm2 = TransactionManager("test_tm.pickle")
tm2._clear()
tm2.add_transaction("foo0", 1, "bubba", "gump", uuid="1")
tm2.add_transaction("foo1", 2, "bubba", "gump", uuid="2")
tm2.add_transaction("foo3", 4, "bubba", "gump", uuid="4")
tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old())
self.assertEqual(tm2.current_id, 5)
for i in xrange(0, 5):
self.assertEqual(tm2.get_transaction(i + 1)["data"], "foo%s" % i)
def test_transaction_resolution_multiple_same_timestamp(self):
self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1")
self.tm.add_transaction("foo1", 1, "bubba", "gump", uuid="2")
self.tm.add_transaction("foo2", 1, "bubba", "gump", uuid="3")
tm2 = TransactionManager("test_tm.pickle")
tm2._clear()
tm2.add_transaction("foo1", 1, "bubba", "gump", uuid="2")
tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old())
self.assertEqual(tm2.current_id, 3)
self.assertEqual(self.tm.top_hash, tm2.top_hash)
def test_transaction_resolution_same_ts_diff_data(self):
self.tm.add_transaction("foo0", 1, "bubba", "gump")
tm2 = TransactionManager("test_tm.pickle")
tm2._clear()
tm2.add_transaction("foo1", 1, "bubba", "gump")
tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old())
self.assertEqual(tm2.current_id, 2)
def test_current_id_loads(self):
self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1")
tm = TransactionManager()
self.assertEqual(tm.current_id, 1)
def test_get_transactions(self):
class FakeTransactionFetcher(object):
def request_transactions(self):
pass
class FakeConnection(object):
def addCallback(self, func):
pass
ftf = FakeTransactionFetcher()
sm = StatusManager(None, self._get_fake_multicast_ping())
tm2 = TransactionManager('test_tm.pickle')
tm2._clear()
tm2.add_transaction("foo1", 1, "bubba", "gump")
tm2.add_transaction("foo2", 2, "bubba", "gump")
passes = []
def worker_thread(ftf, connection):
tm = TransactionManager()
tm._clear()
tm.resolve_missing_transactions(tm2.get_top_hash(), sm.get_transactions(ftf, connection))
self.assertEqual(tm.get_transaction(1)['data'], 'foo1')
self.assertEqual(tm.get_transaction(2)['data'], 'foo2')
passes.append(True)
t = threading.Thread(target=worker_thread, args=[ftf, FakeConnection()])
t.daemon = True
t.start()
self._wait_for_callback_to_be_set(ftf)
for data in tm2.get_transactions_recent_to_old(2):
ftf.call_back(data)
ftf.call_back("done")
t.join()
self.assertEqual(len(passes), 1)
tm2.add_transaction("foo3", 3, "bubba", "gump")
tm2.add_transaction("foo4", 4, "bubba", "gump")
tm2.add_transaction("foo5", 1.5, "bubba", "gump")
def worker_thread2(ftf, connection):
tm = TransactionManager()
tm.resolve_missing_transactions(tm2.get_top_hash(), sm.get_transactions(ftf, connection))
self.assertEqual(tm.get_transaction(2)['data'], 'foo5')
self.assertEqual(tm.get_transaction(3)['data'], 'foo2')
self.assertEqual(tm.get_transaction(4)['data'], 'foo3')
self.assertEqual(tm.get_transaction(5)['data'], 'foo4')
passes.append(True)
t = threading.Thread(target=worker_thread2, args=[ftf, FakeConnection()])
t.daemon = True
del ftf.call_back
t.start()
self._wait_for_callback_to_be_set(ftf)
for data in tm2.get_transactions_recent_to_old(2):
ftf.call_back(data)
time.sleep(0.05)
ftf.call_back("done")
t.join()
self.assertEqual(len(passes), 2)
def setUp(self):
self.tm = TransactionManager()
self.tm._clear()
from paxos_manager import PaxosManager
from transaction_manager import TransactionManager
from util import safe_print, ConnectGraph
if len(sys.argv) < 3:
raise Exception('Wrong arguments. Correct Usage: python server.py <config_file_path> <server_index> <optional_dump_path>')
else:
file = io.open(sys.argv[1])
configJson = json.load(file)
serverI = int(sys.argv[2]) % len(configJson)
localConfig = configJson[serverI]
serverIdList = map(lambda x: x['id'], configJson)
val = serverI
connectGraph = ConnectGraph(configJson)
transactionManager = TransactionManager(localConfig['id'])
paxosManager = PaxosManager(configJson, serverI, transactionManager, connectGraph)
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
serverSocket.bind((localConfig['local_ip_addr'], int(localConfig['port'])))
print "Server {0} listening on port {1}".format(str(localConfig['id']), str(localConfig['port']))
if len(sys.argv) == 4:
# load paxosManager from last saved
paxosManager.initializeFromJSON(json.load(open(sys.argv[3])))
if len(transactionManager.getQueue()) > 0: # if there was a transaction before the crash, start the save attempt thread
paxosManager.attempt_save_timeout_refresh()
def listenRequests():
while True:
data, addr = serverSocket.recvfrom(1024)
import argparse
from block import Block
from block_chain import BlockChain
from serializer import Serializer
from transaction_manager import TransactionManager
# flask init
app = Flask(__name__)
app.secret_key = 'super secret key'
# objects init
mutex = Lock()
s = Serializer()
block_chain = BlockChain()
transaction_manager = TransactionManager()
# create genesis block
genesis_block = Block(index=0,
timestamp=timer(),
transactions=transaction_manager.get_transactions(),
previous_hash="genesis_block",
difficulty=block_chain.get_difficulty())
block_chain.set_working_block(genesis_block)
@app.route('/', methods=['GET'])
def main():
return block_chain.to_html()
import psycopg2
from transaction_manager import TransactionManager
tm = TransactionManager(user="******",
password="******",
host="localhost",
port="5432",
verbose=False)
tm.add_connection("fly_booking", database="fly_booking_db")
tm.add_connection("hotel_booking", database="hotel_booking_db")
tm.add_connection("account", database="account_db")
def decrease_money(conn, money):
account_cursor = conn.cursor()
account_query = "UPDATE account SET amount = amount - %s WHERE client_name = 'Roman Vey';"
account_data = (money, )
account_cursor.execute(account_query, account_data)
def book_hotel(conn):
hotel_cursor = conn.cursor()
hotel_query = "INSERT INTO hotel_booking (client_name, hotel_name, arrival, departure) VALUES (%s, %s, NOW(), NOW());"
hotel_data = ("Roman Vey", "Nice Hotel")
hotel_cursor.execute(hotel_query, hotel_data)
tm.create_transaction("book_hotel", ["hotel_booking", "account"])
tm.add_to_transaction("book_hotel",
"hotel_booking",
book_hotel,
app.secret_key = os.urandom(24)
login_manager = LoginManager()
login_manager.init_app(app)
# Generate a globally unique address for this node
node_identifier = str(uuid4()).replace('-', '')
# Instantiate the Blockchain
blockchain = Blockchain()
# Create dummy users with ids 1 to 5
users = [User(id, node_identifier) for id in range(1, 6)]
# Instantiate the transaction manager
transaction_manager = TransactionManager(users)
@app.route('/mine', methods=['GET'])
def mine():
# We run the proof of work algorithm to get the next proof...
last_block = blockchain.last_block
proof = blockchain.proof_of_work(last_block)
# We must receive a reward for finding the proof.
# The sender is "0" to signify that this node has mined a new coin.
blockchain.new_transaction(
sender="0",
recipient=node_identifier,
amount=1,
)
from transaction_manager import TransactionManager
import piecash
GNUCASH_FILE = "example_file.gnucash"
# book = piecash.open_book(GNUCASH_FILE, readonly=True)
# print(book.default_currency)
book = TransactionManager(GNUCASH_FILE)
a1 = book.get_account("Assets")
a2 = book.get_account("Expence")
book.make_transaction(a1, a2, 20, description="Test transaction")
print(a1)
def __init__(self, filename): self._fp = open(filename, 'r') self._tm = TransactionManager() self._run()
class Neo:
def __init__(self, filename):
self._fp = open(filename, 'r')
self._tm = TransactionManager()
self._run()
def _run(self):
for line in self._fp.readlines():
line = line.strip()
if line:
self._tm.inc_ts()
commands = line.split(';')
for c in commands:
clist = c.strip().strip(')')
clist = clist.split('(')
method = clist[0]
args = clist[1].split(',')
try:
_method = getattr(self, method)
_method(*args)
except Exception as e:
print 'Unknown method, exiting Neo...', traceback.format_exc()
sys.exit(1)
def begin(self, txn_id):
print '---------------------------------------------------------\n'
print 'begin: ', txn_id
try:
self._tm.begin(txn_id.strip())
except Exception as e:
print traceback.format_exc()
def beginRO(self, txn_id):
print '---------------------------------------------------------\n'
print 'beginRO: ', txn_id
try:
self._tm.beginRO(txn_id.strip())
except Exception as e:
print traceback.format_exc()
def W(self, txn_id, var, value):
print '---------------------------------------------------------\n'
print 'W: ', txn_id, var, value
try:
self._tm.write(txn_id.strip(), var.strip(), value.strip())
except Exception as e:
print traceback.format_exc()
def R(self, txn_id, var):
print '---------------------------------------------------------\n'
print 'R: ', txn_id, var
try:
self._tm.read(txn_id.strip(), var.strip())
except Exception as e:
print traceback.format_exc()
def end(self, txn_id):
print '---------------------------------------------------------\n'
print 'end: ', txn_id
self._tm.end(txn_id.strip())
def dump(self, arg):
print '---------------------------------------------------------\n'
print 'dump: ', arg
if arg:
try:
site_id = int(arg.strip())
pprint(self._tm.dump(site_id=site_id))
except ValueError:
pprint(self._tm.dump(var_id=arg.strip()))
else:
pprint(self._tm.dump())
def fail(self, site_id):
print '---------------------------------------------------------\n'
print 'fail: ', site_id
self._tm.fail(int(site_id.strip()))
def recover(self, site_id):
print '---------------------------------------------------------\n'
print 'recover: ', site_id
self._tm.recover(int(site_id.strip()))
from transaction_manager import TransactionManager tx_manager = TransactionManager()
def test_transaction_resolution(self):
self.tm.add_transaction("foo0", 1, "bubba", "gump", uuid="1")
self.tm.add_transaction("foo1", 2, "bubba", "gump", uuid="2")
self.tm.add_transaction("foo2", 3, "bubba", "gump", uuid="3")
self.tm.add_transaction("foo3", 4, "bubba", "gump", uuid="4")
self.tm.add_transaction("foo4", 5, "bubba", "gump", uuid="5")
tm2 = TransactionManager("test_tm.pickle")
tm2._clear()
tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old())
self.assertEqual(tm2.current_id, 5)
for i in xrange(0, 5):
self.assertEqual(tm2.get_transaction(i + 1)["data"], "foo%s" % i)
tm2 = TransactionManager("test_tm.pickle")
tm2._clear()
tm2.add_transaction("foo0", 1, "bubba", "gump", uuid="1")
tm2.add_transaction("foo1", 2, "bubba", "gump", uuid="2")
tm2.add_transaction("foo3", 4, "bubba", "gump", uuid="4")
tm2.resolve_missing_transactions(self.tm.top_hash, self.tm.get_transactions_recent_to_old())
self.assertEqual(tm2.current_id, 5)
for i in xrange(0, 5):
self.assertEqual(tm2.get_transaction(i + 1)["data"], "foo%s" % i)
def update_level2_topic_timestamp(level2_topic_id):
sql = "UPDATE ZHIHU_TOPIC SET LAST_VISIT = '%s' WHERE TOPIC_ID = %s" % \
(get_current_timestamp(), level2_topic_id)
tm = TransactionManager()
tm.execute_sql(sql)
tm.close_connection()
