def parse_block(self, item):
url = item.get('href')
# Selecting block with Name
title_block = item.select_one('span[data-ftid=bull_title]')
title = title_block.string.strip()
# engine transmission power block
cap = item.select('span.css-xyj9u2.e162wx9x0')
s = cap[0].get_text().split(' ')
engcapacity = s[0] + s[1]
transmission = cap[2].get_text().replace(',', '')
power = s[2].replace('(', '') + " " + s[3].replace('),', '')
# Block with price
price = item.select_one('span[data-ftid=bull_price]').get_text(
'\n').strip()
currency = '₽'
# Datetime block
date = None
date_block = item.select_one('div[data-ftid=bull_date]').get_text()
date = self.parse_date(item=date_block)
return Block(url=url,
title=title,
price=price,
currency=currency,
date=date,
engcapacity=engcapacity,
transmission=transmission,
power=power)
def __init__(self, id, server_address, RequestHandlerClass):
# Basic components of the server #
super().__init__(server_address, RequestHandlerClass)
self.id = id # the server's id
# incoming message listener
# self.server_ = ThreadedTCPServer((SERVERS[id]['ip'], SERVERS[id]['port']), ThreadedTCPHandler)
self.tcp_listener = threading.Thread(target=self.serve_forever)
self.tcp_listener.daemon = True
# the path to the block chain
self.chain_path = './chains/blockchain%d.txt' % id
# local block chain
self.chain = Blockchain(self.id)
# initial balance
self.balance = 0
# local block
self.lblock = Block()
self.delay = 5. + self.id * 0.1
# Paxos related #
# the latest known ballot, used to generate now ballot number
self.latest_ballot = 0
# the ballot number of the ballot that the server currently belongs to
self.ballot_num = None
# the block it accepted
self.accept_block = None
# the ballot number of the accepted block
self.accept_ballot = None
# the quorum for election (also collects the accepted blocks)
self.quorum = Quorum(args.num_server, self.accept_ballot,
self.accept_block, self.ballot_num)
# the quorum counter for accept
self.accept_counter = 1
# Step 1: load chain from file
self.load_chain()
# Step 2: start TCP listener
self.start_listener()
# the quorum counter for blockchain sync
self.sync_quorum = SyncQuorum(args.num_server, self.chain.tail.seq)
self.sync_block()
# get balance
self.update_balance()
def parse_block(self, item):
# Block with url
url_block = item.select_one('a.snippet-link')
# print(url_block)
href = url_block.get('href')
if href:
url = 'https://www.avito.ru' + href
else:
url = None
# Selecting block with Name
title_block = item.select_one('h3.snippet-title span')
title = title_block.string.strip()
# engine capacity block
try:
cap = item.select_one('div.specific-params.specific-params_block')
s = cap.string.strip().split(', ')[1].split(' ')
engcapacity = s[0]
transmission = s[1]
power = s[2].replace('(', '') + " " + s[3].replace(')', '')
except Exception as ex:
print("Описание движка не дописано в данном объявлении", url)
if len(s) != 0:
transmission = s[0]
power = 'Неясно'
engcapacity = 'Неясно'
else:
power = 'Неясно'
engcapacity = 'Неясно'
power = 'Неясно'
# Block with name and currency
price_block = item.select_one('span.snippet-price').get_text(
'\n').strip()
price_block = list(price_block.rsplit(' ', maxsplit=1))
if len(price_block) == 2:
price, currency = price_block
else:
price, currency = None, None
print('Проблема со считыванием цены и валюты: ', price_block)
# Datetime block
date = None
date_block = item.select_one('div.snippet-date-info')
absolute_date = date_block.get('data-tooltip')
if (absolute_date == ''):
absolute_date = date_block.get_text().strip()
date = self.parse_date(item=absolute_date)
return Block(url=url,
title=title,
price=price,
currency=currency,
date=date,
engcapacity=engcapacity,
transmission=transmission,
power=power)
def add_block():
""" add a new block to the chain """
body = request.get_json()
try:
block = Block(len(CHAIN), body["data"], CHAIN.head().hash)
CHAIN.add_block(block)
except HashMismatchError:
return "new block has incorrect previous hash", 200
except IndexMismatchError:
return "new block has incorrect index", 200
except KeyError:
return "expected data attribute in post body", 200
return "added block at index {}".format(len(CHAIN) - 1)
def parse_block(self, item):
urlblock = item.select_one('a[data-target="serp-snippet-title"]')
url = urlblock.get('href')
# Selecting block with Name
title = urlblock.get('title').strip()
# engine transmission power block
transmission = item.select_one(
'div[data-target-id="serp-snippet-gear-type"]').get_text()
power = item.select_one(
'div[data-target-id="serp-snippet-engine-power"]').get_text()
engcapacity = item.select_one(
'div[data-target-id="serp-snippet-engine-vol-type"]').get_text(
).split(', ')[0].replace(',', '')
# Block with price
price = item.select_one(
'div[data-target="serp-snippet-price"]').get_text('\n').strip()
currency = '₽'
# Datetime block
date = None
date_block = item.select_one(
'div[data-target="serp-snippet-actual-date"]').get_text()
date = self.parse_date(item=date_block)
return Block(url=url,
title=title,
price=price,
currency=currency,
date=date,
engcapacity=engcapacity,
transmission=transmission,
power=power)
def parse_block(self, data):
url = 'https://auto.ru/cars/used/sale/' + data['vehicle_info'][
'mark_info']['code'].lower() + '/' + data['vehicle_info'][
'model_info']['code'].lower() + '/' + data['saleId']
# title of car
title = data['vehicle_info']['mark_info']['name'] + \
' ' + data['vehicle_info']['model_info']['name']
if 'name' in data['vehicle_info']['super_gen']:
title = title + ' ' + data['vehicle_info']['super_gen']['name']
if 'notice' in data['vehicle_info']['configuration']:
title = title + ' ' + data['vehicle_info']['configuration'][
'notice']
# engine capacity transmission
cap = data['lk_summary'].replace('\xa0', ' ').split(' ')
engcapacity = cap[0]
power = cap[2].replace('(', '') + ' ' + cap[3].replace('),', '')
transmission = cap[1]
# getting price
price, currency = data['price_info']['RUR'], '₽'
# data of publicity
date = data['additional_info']['hot_info']['start_time'].replace(
'T', ' ').replace('Z', '')
return Block(url=url,
title=title,
price=price,
currency=currency,
date=date,
engcapacity=engcapacity,
transmission=transmission,
power=power)
def paxos(self, tr):
time.sleep(0.5)
# If the balance on the client account is at least amt
# the server logs the request, and executes the transfer locally
if tr.amt <= self.balance:
self.lblock.add_trans(tr)
self.update_balance()
print("[SUCCESS] Transaction (local) SUCCESS, current balance %d" %
self.balance)
# Otherwise, the server runs a modified Paxos protocal among all servers
# to get the most up-to-date transactions from other servers and then
# updates its blockchain copy
else:
# Part I: Leader Election
fail = self.leader_election()
if fail:
print("[FAIL] Failed in leader election, try again")
self.paxos(tr)
else:
# Part II: Normal Operation
# if some followers have previously accepted blocks
# process the previously accepted block first
if self.quorum.accept_block is not None:
print("[PRE] commit previously accepted block first")
self.quorum.accept_block.seq = self.lblock.seq
self.lblock.seq += 1
fail = self.normal(self.quorum.accept_block,
self.quorum.accept_ballot)
if fail:
# if found out it is not the leader any more
# redo paxos to see if there is enough money
# after the update
print(
"[FAIL] Failed to commit previous block, try again"
)
# anyway, a new block is committed
# so check the new balance before continuing
self.paxos(tr)
else:
# If there is no accepted block returned
# merge the block to commit
merge_block = Block()
merge_block.merge(self.lblock)
for b in self.quorum.local_blocks:
merge_block.merge(b)
merge_block.seq = self.lblock.seq
# accept? and commit the merged block
fail = self.normal(merge_block, self.ballot_num)
if fail:
# if the server is not leader anymore
# cause a new block is committed
# check the balance and redo paxos
print(
"[FAIL] Failed to commit merged block, try again")
self.paxos(tr)
else:
# else, the merged block is committed
# check whether the balance is enough for the transaction
if tr.amt <= self.balance:
self.lblock.add_trans(tr)
self.update_balance()
print(
"[SUCCESS] Transaction (local) SUCCESS, current balance %d"
% self.balance)
else:
print(
"[ERROR] Not enough money, current balance %d"
% self.balance)
class Server(ThreadedTCPServer):
def __init__(self, id, server_address, RequestHandlerClass):
# Basic components of the server #
super().__init__(server_address, RequestHandlerClass)
self.id = id # the server's id
# incoming message listener
# self.server_ = ThreadedTCPServer((SERVERS[id]['ip'], SERVERS[id]['port']), ThreadedTCPHandler)
self.tcp_listener = threading.Thread(target=self.serve_forever)
self.tcp_listener.daemon = True
# the path to the block chain
self.chain_path = './chains/blockchain%d.txt' % id
# local block chain
self.chain = Blockchain(self.id)
# initial balance
self.balance = 0
# local block
self.lblock = Block()
self.delay = 5. + self.id * 0.1
# Paxos related #
# the latest known ballot, used to generate now ballot number
self.latest_ballot = 0
# the ballot number of the ballot that the server currently belongs to
self.ballot_num = None
# the block it accepted
self.accept_block = None
# the ballot number of the accepted block
self.accept_ballot = None
# the quorum for election (also collects the accepted blocks)
self.quorum = Quorum(args.num_server, self.accept_ballot,
self.accept_block, self.ballot_num)
# the quorum counter for accept
self.accept_counter = 1
# Step 1: load chain from file
self.load_chain()
# Step 2: start TCP listener
self.start_listener()
# the quorum counter for blockchain sync
self.sync_quorum = SyncQuorum(args.num_server, self.chain.tail.seq)
self.sync_block()
# get balance
self.update_balance()
# load the previously saved blockchain from file
def load_chain(self):
if os.path.exists(self.chain_path):
self.chain = pickle.load(open(self.chain_path, "rb"))
self.lblock.seq = self.chain.tail.seq + 1
else:
print("No blockchain is found")
def update_balance(self):
self.balance = self.chain.get_balance() + self.lblock.balance_change(
self.id)
# write the chain to file
def save_chain(self):
pickle.dump(self.chain, open(self.chain_path, "wb"))
# Part I: Leader Election
def leader_election(self):
# step 1: generate the ballot <ballotNum, Id>
self.ballot_num = Ballot(self.latest_ballot, self.id)
# increment ballot number by 1
self.latest_ballot += 1
self.ballot_num.increment()
self.latest_ballot += 1
# broad cast leader prepare message to all
mesg = ("prepare", self.ballot_num)
self.broadcast(mesg)
self.quorum.reset(self.accept_ballot, self.accept_block,
self.ballot_num)
self.lblock.seq = self.chain.tail.seq + 1
counter = 0
# step 3: collect acknowledgement
while self.quorum.num_ack < (args.num_server - 1) / 2 + 1:
counter += 1
time.sleep(0.1)
# If the seq number of the block to commit is smaller/equal to the chain's tail
# (other process commit a block)
if self.lblock.seq <= self.chain.tail.seq:
return True
if counter > 1200:
return True
# once getting enough messages, the server becomes the leader
print("[LEADER] Get enough ack to become the leader"
) # the server has become the leader
return False
def commit(self, block2commit):
# commit the new block to chain
incomplete = self.chain.add_block(block2commit)
if incomplete:
self.sync_block()
# reset the accepted block and ballot
if self.accept_block is not None:
if self.accept_block.seq <= block2commit.seq:
self.accept_block = None
self.accept_ballot = None
# save the chain to local file
self.save_chain()
# update the balance and remove committed logs in local log
self.lblock.wash(block2commit)
self.update_balance()
# self.lblock.seq = self.chain.tail.seq + 1
# Part II: Normal Operations
def normal(self, block2commit, ballot):
self.accept_block = block2commit
self.accept_ballot = ballot
# broadcast message <accept?, ballotNum, block to commit>
msg = ("accept?", self.ballot_num, block2commit)
self.broadcast(msg)
# collect accept from quorum
self.accept_counter = 1
counter = 0
while self.accept_counter < (args.num_server - 1) / 2 + 1:
counter += 1
time.sleep(0.1)
if block2commit.seq <= self.chain.tail.seq:
self.lblock.seq += 1
return True
if counter > 1200:
return True
print("[COMMIT] the block is accepted by the majority")
msg = ("commit", block2commit)
self.broadcast(msg)
self.commit(block2commit)
return False
def trans_perform(self, todo_list):
for tr in todo_list:
if tr.s != self.id:
print("[Error] Can only send money from your own account")
else:
if tr.r == self.id:
print("[Ignored] You are sending money to yourself")
else:
self.paxos(tr)
def paxos(self, tr):
time.sleep(0.5)
# If the balance on the client account is at least amt
# the server logs the request, and executes the transfer locally
if tr.amt <= self.balance:
self.lblock.add_trans(tr)
self.update_balance()
print("[SUCCESS] Transaction (local) SUCCESS, current balance %d" %
self.balance)
# Otherwise, the server runs a modified Paxos protocal among all servers
# to get the most up-to-date transactions from other servers and then
# updates its blockchain copy
else:
# Part I: Leader Election
fail = self.leader_election()
if fail:
print("[FAIL] Failed in leader election, try again")
self.paxos(tr)
else:
# Part II: Normal Operation
# if some followers have previously accepted blocks
# process the previously accepted block first
if self.quorum.accept_block is not None:
print("[PRE] commit previously accepted block first")
self.quorum.accept_block.seq = self.lblock.seq
self.lblock.seq += 1
fail = self.normal(self.quorum.accept_block,
self.quorum.accept_ballot)
if fail:
# if found out it is not the leader any more
# redo paxos to see if there is enough money
# after the update
print(
"[FAIL] Failed to commit previous block, try again"
)
# anyway, a new block is committed
# so check the new balance before continuing
self.paxos(tr)
else:
# If there is no accepted block returned
# merge the block to commit
merge_block = Block()
merge_block.merge(self.lblock)
for b in self.quorum.local_blocks:
merge_block.merge(b)
merge_block.seq = self.lblock.seq
# accept? and commit the merged block
fail = self.normal(merge_block, self.ballot_num)
if fail:
# if the server is not leader anymore
# cause a new block is committed
# check the balance and redo paxos
print(
"[FAIL] Failed to commit merged block, try again")
self.paxos(tr)
else:
# else, the merged block is committed
# check whether the balance is enough for the transaction
if tr.amt <= self.balance:
self.lblock.add_trans(tr)
self.update_balance()
print(
"[SUCCESS] Transaction (local) SUCCESS, current balance %d"
% self.balance)
else:
print(
"[ERROR] Not enough money, current balance %d"
% self.balance)
def sync_block(self):
print("Start syncing block chain")
# when the server starts (recovers), it contacts other servers for the latest blockchain
# step 1: broad cast the request with <"sync", local largest seq number, id>
msg = ("sync", self.chain.tail.seq, self.id)
self.broadcast(msg)
timeout = False
counter = 0
while self.sync_quorum.num_syn < self.sync_quorum.quorum_size:
time.sleep(0.1)
counter += 1
if counter > 200:
timeout = True
self.sync_block()
if not timeout:
# get enough chains
if self.sync_quorum.chain is None:
print("[SYNC] No need to update")
else:
self.chain.merge(self.sync_quorum.chain)
print("[SYNC] Local Chain is updated")
# start the TCP listener
def start_listener(self):
try:
self.tcp_listener.start()
except KeyboardInterrupt:
self.shutdown()
sys.exit()
@staticmethod
def send(mesg, target):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((SERVERS[target]['ip'], SERVERS[target]['port']))
mesg = pickle.dumps(mesg)
sock.sendall(mesg)
sock.close()
except ConnectionError:
pass
def broadcast(self, mesg):
for i in range(args.num_server):
if i != self.id:
Server.send(mesg, i)
# Skip to where pallet_town is stored on the ROM
### NEED TO DOUBLE CHECK THE ADDING 1 HERE
all_data = gold_rom.read()
relevant_hex_data = all_data[current_map.start:current_map.end + 1]
#i, j = 550, 350
i, j = 0, 0
i_ = 0
for h in relevant_hex_data:
offset = hex(current_map.end - bytes_to_read)
bytes_to_read -= 1
FSP_repr = hex_to_FSP[current_map.bank].get(h)
b = Block(FSP_repr, i, j)
current_map.add_block(b)
i += 8
i_ += 1
if (i_ % current_map.width == 0):
i_ = 0
i -= 8 * current_map.width
j += 8
print(current_map.FSP_map())
#print(pallet_town.name, pallet_town.start, pallet_town.end)
# Skip to where pallet_town is stored on the ROM
### NEED TO DOUBLE CHECK THE ADDING 1 HERE
all_data = gold_rom.read()
relevant_hex_data = all_data[current_map.start:current_map.end + 1]
i = i_ = j = block_id = 0
for h in relevant_hex_data:
offset = hex(current_map.end - bytes_to_read)
d = (int(offset, 0))
bytes_to_read -= 1
contents = all_data[d]
b = Block(i, j, contents, width=SQUARE_SIZE, block_id=block_id)
current_map.add_block(b)
#print(current_map.generate_block_html(b))
#print(current_map.generate_block_css(b))
i += SQUARE_SIZE
i_ += 1
block_id += 1
if (i_ % current_map.width == 0):
i_ = i = 0
j += SQUARE_SIZE
'''
FSP_repr = hex_to_FSP[current_map.bank].get(h)
b = Block(FSP_repr, i, j)