def parse_block(self, item): url = item.get('href') # Selecting block with Name title_block = item.select_one('span[data-ftid=bull_title]') title = title_block.string.strip() # engine transmission power block cap = item.select('span.css-xyj9u2.e162wx9x0') s = cap[0].get_text().split(' ') engcapacity = s[0] + s[1] transmission = cap[2].get_text().replace(',', '') power = s[2].replace('(', '') + " " + s[3].replace('),', '') # Block with price price = item.select_one('span[data-ftid=bull_price]').get_text( '\n').strip() currency = '₽' # Datetime block date = None date_block = item.select_one('div[data-ftid=bull_date]').get_text() date = self.parse_date(item=date_block) return Block(url=url, title=title, price=price, currency=currency, date=date, engcapacity=engcapacity, transmission=transmission, power=power)
def __init__(self, id, server_address, RequestHandlerClass): # Basic components of the server # super().__init__(server_address, RequestHandlerClass) self.id = id # the server's id # incoming message listener # self.server_ = ThreadedTCPServer((SERVERS[id]['ip'], SERVERS[id]['port']), ThreadedTCPHandler) self.tcp_listener = threading.Thread(target=self.serve_forever) self.tcp_listener.daemon = True # the path to the block chain self.chain_path = './chains/blockchain%d.txt' % id # local block chain self.chain = Blockchain(self.id) # initial balance self.balance = 0 # local block self.lblock = Block() self.delay = 5. + self.id * 0.1 # Paxos related # # the latest known ballot, used to generate now ballot number self.latest_ballot = 0 # the ballot number of the ballot that the server currently belongs to self.ballot_num = None # the block it accepted self.accept_block = None # the ballot number of the accepted block self.accept_ballot = None # the quorum for election (also collects the accepted blocks) self.quorum = Quorum(args.num_server, self.accept_ballot, self.accept_block, self.ballot_num) # the quorum counter for accept self.accept_counter = 1 # Step 1: load chain from file self.load_chain() # Step 2: start TCP listener self.start_listener() # the quorum counter for blockchain sync self.sync_quorum = SyncQuorum(args.num_server, self.chain.tail.seq) self.sync_block() # get balance self.update_balance()
def parse_block(self, item): # Block with url url_block = item.select_one('a.snippet-link') # print(url_block) href = url_block.get('href') if href: url = 'https://www.avito.ru' + href else: url = None # Selecting block with Name title_block = item.select_one('h3.snippet-title span') title = title_block.string.strip() # engine capacity block try: cap = item.select_one('div.specific-params.specific-params_block') s = cap.string.strip().split(', ')[1].split(' ') engcapacity = s[0] transmission = s[1] power = s[2].replace('(', '') + " " + s[3].replace(')', '') except Exception as ex: print("Описание движка не дописано в данном объявлении", url) if len(s) != 0: transmission = s[0] power = 'Неясно' engcapacity = 'Неясно' else: power = 'Неясно' engcapacity = 'Неясно' power = 'Неясно' # Block with name and currency price_block = item.select_one('span.snippet-price').get_text( '\n').strip() price_block = list(price_block.rsplit(' ', maxsplit=1)) if len(price_block) == 2: price, currency = price_block else: price, currency = None, None print('Проблема со считыванием цены и валюты: ', price_block) # Datetime block date = None date_block = item.select_one('div.snippet-date-info') absolute_date = date_block.get('data-tooltip') if (absolute_date == ''): absolute_date = date_block.get_text().strip() date = self.parse_date(item=absolute_date) return Block(url=url, title=title, price=price, currency=currency, date=date, engcapacity=engcapacity, transmission=transmission, power=power)
def add_block(): """ add a new block to the chain """ body = request.get_json() try: block = Block(len(CHAIN), body["data"], CHAIN.head().hash) CHAIN.add_block(block) except HashMismatchError: return "new block has incorrect previous hash", 200 except IndexMismatchError: return "new block has incorrect index", 200 except KeyError: return "expected data attribute in post body", 200 return "added block at index {}".format(len(CHAIN) - 1)
def parse_block(self, item): urlblock = item.select_one('a[data-target="serp-snippet-title"]') url = urlblock.get('href') # Selecting block with Name title = urlblock.get('title').strip() # engine transmission power block transmission = item.select_one( 'div[data-target-id="serp-snippet-gear-type"]').get_text() power = item.select_one( 'div[data-target-id="serp-snippet-engine-power"]').get_text() engcapacity = item.select_one( 'div[data-target-id="serp-snippet-engine-vol-type"]').get_text( ).split(', ')[0].replace(',', '') # Block with price price = item.select_one( 'div[data-target="serp-snippet-price"]').get_text('\n').strip() currency = '₽' # Datetime block date = None date_block = item.select_one( 'div[data-target="serp-snippet-actual-date"]').get_text() date = self.parse_date(item=date_block) return Block(url=url, title=title, price=price, currency=currency, date=date, engcapacity=engcapacity, transmission=transmission, power=power)
def parse_block(self, data): url = 'https://auto.ru/cars/used/sale/' + data['vehicle_info'][ 'mark_info']['code'].lower() + '/' + data['vehicle_info'][ 'model_info']['code'].lower() + '/' + data['saleId'] # title of car title = data['vehicle_info']['mark_info']['name'] + \ ' ' + data['vehicle_info']['model_info']['name'] if 'name' in data['vehicle_info']['super_gen']: title = title + ' ' + data['vehicle_info']['super_gen']['name'] if 'notice' in data['vehicle_info']['configuration']: title = title + ' ' + data['vehicle_info']['configuration'][ 'notice'] # engine capacity transmission cap = data['lk_summary'].replace('\xa0', ' ').split(' ') engcapacity = cap[0] power = cap[2].replace('(', '') + ' ' + cap[3].replace('),', '') transmission = cap[1] # getting price price, currency = data['price_info']['RUR'], '₽' # data of publicity date = data['additional_info']['hot_info']['start_time'].replace( 'T', ' ').replace('Z', '') return Block(url=url, title=title, price=price, currency=currency, date=date, engcapacity=engcapacity, transmission=transmission, power=power)
def paxos(self, tr): time.sleep(0.5) # If the balance on the client account is at least amt # the server logs the request, and executes the transfer locally if tr.amt <= self.balance: self.lblock.add_trans(tr) self.update_balance() print("[SUCCESS] Transaction (local) SUCCESS, current balance %d" % self.balance) # Otherwise, the server runs a modified Paxos protocal among all servers # to get the most up-to-date transactions from other servers and then # updates its blockchain copy else: # Part I: Leader Election fail = self.leader_election() if fail: print("[FAIL] Failed in leader election, try again") self.paxos(tr) else: # Part II: Normal Operation # if some followers have previously accepted blocks # process the previously accepted block first if self.quorum.accept_block is not None: print("[PRE] commit previously accepted block first") self.quorum.accept_block.seq = self.lblock.seq self.lblock.seq += 1 fail = self.normal(self.quorum.accept_block, self.quorum.accept_ballot) if fail: # if found out it is not the leader any more # redo paxos to see if there is enough money # after the update print( "[FAIL] Failed to commit previous block, try again" ) # anyway, a new block is committed # so check the new balance before continuing self.paxos(tr) else: # If there is no accepted block returned # merge the block to commit merge_block = Block() merge_block.merge(self.lblock) for b in self.quorum.local_blocks: merge_block.merge(b) merge_block.seq = self.lblock.seq # accept? and commit the merged block fail = self.normal(merge_block, self.ballot_num) if fail: # if the server is not leader anymore # cause a new block is committed # check the balance and redo paxos print( "[FAIL] Failed to commit merged block, try again") self.paxos(tr) else: # else, the merged block is committed # check whether the balance is enough for the transaction if tr.amt <= self.balance: self.lblock.add_trans(tr) self.update_balance() print( "[SUCCESS] Transaction (local) SUCCESS, current balance %d" % self.balance) else: print( "[ERROR] Not enough money, current balance %d" % self.balance)
class Server(ThreadedTCPServer): def __init__(self, id, server_address, RequestHandlerClass): # Basic components of the server # super().__init__(server_address, RequestHandlerClass) self.id = id # the server's id # incoming message listener # self.server_ = ThreadedTCPServer((SERVERS[id]['ip'], SERVERS[id]['port']), ThreadedTCPHandler) self.tcp_listener = threading.Thread(target=self.serve_forever) self.tcp_listener.daemon = True # the path to the block chain self.chain_path = './chains/blockchain%d.txt' % id # local block chain self.chain = Blockchain(self.id) # initial balance self.balance = 0 # local block self.lblock = Block() self.delay = 5. + self.id * 0.1 # Paxos related # # the latest known ballot, used to generate now ballot number self.latest_ballot = 0 # the ballot number of the ballot that the server currently belongs to self.ballot_num = None # the block it accepted self.accept_block = None # the ballot number of the accepted block self.accept_ballot = None # the quorum for election (also collects the accepted blocks) self.quorum = Quorum(args.num_server, self.accept_ballot, self.accept_block, self.ballot_num) # the quorum counter for accept self.accept_counter = 1 # Step 1: load chain from file self.load_chain() # Step 2: start TCP listener self.start_listener() # the quorum counter for blockchain sync self.sync_quorum = SyncQuorum(args.num_server, self.chain.tail.seq) self.sync_block() # get balance self.update_balance() # load the previously saved blockchain from file def load_chain(self): if os.path.exists(self.chain_path): self.chain = pickle.load(open(self.chain_path, "rb")) self.lblock.seq = self.chain.tail.seq + 1 else: print("No blockchain is found") def update_balance(self): self.balance = self.chain.get_balance() + self.lblock.balance_change( self.id) # write the chain to file def save_chain(self): pickle.dump(self.chain, open(self.chain_path, "wb")) # Part I: Leader Election def leader_election(self): # step 1: generate the ballot <ballotNum, Id> self.ballot_num = Ballot(self.latest_ballot, self.id) # increment ballot number by 1 self.latest_ballot += 1 self.ballot_num.increment() self.latest_ballot += 1 # broad cast leader prepare message to all mesg = ("prepare", self.ballot_num) self.broadcast(mesg) self.quorum.reset(self.accept_ballot, self.accept_block, self.ballot_num) self.lblock.seq = self.chain.tail.seq + 1 counter = 0 # step 3: collect acknowledgement while self.quorum.num_ack < (args.num_server - 1) / 2 + 1: counter += 1 time.sleep(0.1) # If the seq number of the block to commit is smaller/equal to the chain's tail # (other process commit a block) if self.lblock.seq <= self.chain.tail.seq: return True if counter > 1200: return True # once getting enough messages, the server becomes the leader print("[LEADER] Get enough ack to become the leader" ) # the server has become the leader return False def commit(self, block2commit): # commit the new block to chain incomplete = self.chain.add_block(block2commit) if incomplete: self.sync_block() # reset the accepted block and ballot if self.accept_block is not None: if self.accept_block.seq <= block2commit.seq: self.accept_block = None self.accept_ballot = None # save the chain to local file self.save_chain() # update the balance and remove committed logs in local log self.lblock.wash(block2commit) self.update_balance() # self.lblock.seq = self.chain.tail.seq + 1 # Part II: Normal Operations def normal(self, block2commit, ballot): self.accept_block = block2commit self.accept_ballot = ballot # broadcast message <accept?, ballotNum, block to commit> msg = ("accept?", self.ballot_num, block2commit) self.broadcast(msg) # collect accept from quorum self.accept_counter = 1 counter = 0 while self.accept_counter < (args.num_server - 1) / 2 + 1: counter += 1 time.sleep(0.1) if block2commit.seq <= self.chain.tail.seq: self.lblock.seq += 1 return True if counter > 1200: return True print("[COMMIT] the block is accepted by the majority") msg = ("commit", block2commit) self.broadcast(msg) self.commit(block2commit) return False def trans_perform(self, todo_list): for tr in todo_list: if tr.s != self.id: print("[Error] Can only send money from your own account") else: if tr.r == self.id: print("[Ignored] You are sending money to yourself") else: self.paxos(tr) def paxos(self, tr): time.sleep(0.5) # If the balance on the client account is at least amt # the server logs the request, and executes the transfer locally if tr.amt <= self.balance: self.lblock.add_trans(tr) self.update_balance() print("[SUCCESS] Transaction (local) SUCCESS, current balance %d" % self.balance) # Otherwise, the server runs a modified Paxos protocal among all servers # to get the most up-to-date transactions from other servers and then # updates its blockchain copy else: # Part I: Leader Election fail = self.leader_election() if fail: print("[FAIL] Failed in leader election, try again") self.paxos(tr) else: # Part II: Normal Operation # if some followers have previously accepted blocks # process the previously accepted block first if self.quorum.accept_block is not None: print("[PRE] commit previously accepted block first") self.quorum.accept_block.seq = self.lblock.seq self.lblock.seq += 1 fail = self.normal(self.quorum.accept_block, self.quorum.accept_ballot) if fail: # if found out it is not the leader any more # redo paxos to see if there is enough money # after the update print( "[FAIL] Failed to commit previous block, try again" ) # anyway, a new block is committed # so check the new balance before continuing self.paxos(tr) else: # If there is no accepted block returned # merge the block to commit merge_block = Block() merge_block.merge(self.lblock) for b in self.quorum.local_blocks: merge_block.merge(b) merge_block.seq = self.lblock.seq # accept? and commit the merged block fail = self.normal(merge_block, self.ballot_num) if fail: # if the server is not leader anymore # cause a new block is committed # check the balance and redo paxos print( "[FAIL] Failed to commit merged block, try again") self.paxos(tr) else: # else, the merged block is committed # check whether the balance is enough for the transaction if tr.amt <= self.balance: self.lblock.add_trans(tr) self.update_balance() print( "[SUCCESS] Transaction (local) SUCCESS, current balance %d" % self.balance) else: print( "[ERROR] Not enough money, current balance %d" % self.balance) def sync_block(self): print("Start syncing block chain") # when the server starts (recovers), it contacts other servers for the latest blockchain # step 1: broad cast the request with <"sync", local largest seq number, id> msg = ("sync", self.chain.tail.seq, self.id) self.broadcast(msg) timeout = False counter = 0 while self.sync_quorum.num_syn < self.sync_quorum.quorum_size: time.sleep(0.1) counter += 1 if counter > 200: timeout = True self.sync_block() if not timeout: # get enough chains if self.sync_quorum.chain is None: print("[SYNC] No need to update") else: self.chain.merge(self.sync_quorum.chain) print("[SYNC] Local Chain is updated") # start the TCP listener def start_listener(self): try: self.tcp_listener.start() except KeyboardInterrupt: self.shutdown() sys.exit() @staticmethod def send(mesg, target): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((SERVERS[target]['ip'], SERVERS[target]['port'])) mesg = pickle.dumps(mesg) sock.sendall(mesg) sock.close() except ConnectionError: pass def broadcast(self, mesg): for i in range(args.num_server): if i != self.id: Server.send(mesg, i)
# Skip to where pallet_town is stored on the ROM ### NEED TO DOUBLE CHECK THE ADDING 1 HERE all_data = gold_rom.read() relevant_hex_data = all_data[current_map.start:current_map.end + 1] #i, j = 550, 350 i, j = 0, 0 i_ = 0 for h in relevant_hex_data: offset = hex(current_map.end - bytes_to_read) bytes_to_read -= 1 FSP_repr = hex_to_FSP[current_map.bank].get(h) b = Block(FSP_repr, i, j) current_map.add_block(b) i += 8 i_ += 1 if (i_ % current_map.width == 0): i_ = 0 i -= 8 * current_map.width j += 8 print(current_map.FSP_map()) #print(pallet_town.name, pallet_town.start, pallet_town.end)
# Skip to where pallet_town is stored on the ROM ### NEED TO DOUBLE CHECK THE ADDING 1 HERE all_data = gold_rom.read() relevant_hex_data = all_data[current_map.start:current_map.end + 1] i = i_ = j = block_id = 0 for h in relevant_hex_data: offset = hex(current_map.end - bytes_to_read) d = (int(offset, 0)) bytes_to_read -= 1 contents = all_data[d] b = Block(i, j, contents, width=SQUARE_SIZE, block_id=block_id) current_map.add_block(b) #print(current_map.generate_block_html(b)) #print(current_map.generate_block_css(b)) i += SQUARE_SIZE i_ += 1 block_id += 1 if (i_ % current_map.width == 0): i_ = i = 0 j += SQUARE_SIZE ''' FSP_repr = hex_to_FSP[current_map.bank].get(h) b = Block(FSP_repr, i, j)