def dataReceived(self, data): for line in data.splitlines(): line = line.strip() try: envelopes = messages.read_envelope(line) for envelope in envelopes: if self.state in ["GETHELLO", "SENTHELLO"]: # Force first message to be HELLO or crash if envelope['msgtype'] == 'hello': self.handle_HELLO(json.dumps(envelope)) else: _print(" [!] Ignoring", envelope['msgtype'], "in", self.state) else: if envelope['msgtype'] == 'ping': self.handle_PING(json.dumps(envelope)) elif envelope['msgtype'] == 'pong': self.handle_PONG(json.dumps(envelope)) elif envelope['msgtype'] == 'addr': self.handle_ADDR(json.dumps(envelope)) elif envelope['msgtype'] == 'tx': self.receiveTx(json.dumps(envelope)) elif envelope['msgtype'] == 'block': self.receiveBlock(json.dumps(envelope)) except: pass
def handle_PONG(self, pong): pong = messages.read_message(pong) _print(" [<] PONG from", self.remote_nodeid, "at", self.remote_ip) # hacky addr, kind, listenPort = self.factory.peers[self.remote_nodeid][:3] self.factory.peers[self.remote_nodeid] = (addr, kind, listenPort, time())
def MiningWorker(self): while True: if self.running: newBlock = self.mine() _print(' [MINER] Mined new block:', newBlock['index']) reactor.callFromThread(self.protocol.broadcastBlock, newBlock) time.sleep(10)
def send_ADDR(self): _print(" [>] Telling " + self.remote_nodeid + " about my peers") # Shouldn't this be a list and not a dict? peers = self.factory.peers listeners = [(n, peers[n][0], peers[n][1], peers[n][2]) for n in peers] addr = messages.create_addr(self.nodeid, listeners) self.write(addr)
def submitTransaction(self, transaction): if self.verify_transaction(transaction) is True: UnTransactionDB().insert(transaction.to_dict()) _print(" [TX] Created new transaction") reactor.callFromThread(self.protocol.broadcastTx, transaction.to_dict()) return transaction.to_dict()['hash']
def broadcastBlock(self, block): _print(" [P2P] Broadcast block=" + str(block['hash']) + ", index=" + str(block['index']) + ", miner=" + str(block['miner'])) blockMsg = messages.createBlockMsg(self.nodeid, block) for client in self.factory.clients: client.transport.write(blockMsg) client.factory.knownBlocks[client.remote_ip].append(block['hash'])
def startFactory(self): self.peers = {} self.clients = [] self.knownBlocks = {} self.knownTxs = {} self.numProtocols = 0 self.nodeid = cryptotools.generate_nodeid()[:10] _print(" [P2P] NODEID:", self.nodeid)
def __init__(self, ip, port, protocol): http_server = pyjsonrpc.ThreadingHttpServer( server_address=(ip, port), RequestHandlerClass=RequestHandler) http_server.account = Account(protocol) http_server.miner = Miner(protocol) _print(" [RPC] Starting RPC server ...") _print(" [RPC] URL: http://" + ip + ":" + str(port)) http_server.serve_forever()
def evo_weekly(output, budget): try: if output == 'console': log._print(evo.weekly_report_text(budget)) if output == 'email': notification.send_notification("EVO Weekly report", evo.weekly_report_html(budget)) except: exception_handler._handle_exception("Exception in EVO weekly report")
def _handle_exception(msg="Exception in personal Automation"): """ Handles notification by looking at environment variable PA_EXCEPTIONS to decide the notification output channel. msg is the message of the exception. """ output = os.environ.get('PA_EXCEPTIONS', 'console') if output == 'console': log._print(traceback.format_exc()) if output == 'email': notification.send_notification(msg, traceback.format_exc())
def send_notification(subject, text): hotmail = passwords.get_entry('hotmail') msg = MIMEMultipart() message = text password = hotmail.password msg['From'] = hotmail.username msg['To'] = "*****@*****.**" msg['Subject'] = subject #msg.attach(MIMEText(message, 'plain')) msg.attach(MIMEText(message, 'html')) server = smtplib.SMTP('smtp.live.com: 587') server.starttls() server.login(msg['From'], password) server.sendmail(msg['From'], msg['To'], msg.as_string()) server.quit() _print("successfully sent email to %s:" % (msg['To']))
def receiveTx(self, msg): try: newTx = messages.read_message_noverify(msg)['tx'] _print(" [<] Recieve tx=" + newTx['hash'] + " from peer " + self.remote_nodeid) UnTransactionDB().insert(newTx) self.factory.knownTxs[self.remote_ip].append(newTx['hash']) # propagate to the unknown peers for client in self.factory.clients: if newTx['hash'] not in self.factory.knownTxs[ client.remote_ip]: client.transport.write(msg) except messages.InvalidSignatureError: _print(" [!] ERROR: Invalid tx sign ", self.remote_ip) self.transport.loseConnection()
def __init__(self, ip, s_ip, s_port): TUNSETIFF = 0X400454ca IFF_TUN = 0x0001 IFF_TAP = 0x0002 IFF_NO_PI = 0x1000 SERVER_IP = s_ip #"128.230.208.73" SERVER_PROT = int(s_port) #int(sys.argv[1]) tun = os.open("/dev/net/tun", os.O_RDWR) ifr = struct.pack('16sH', b'tun%d', IFF_TUN | IFF_NO_PI) ifname_bytes = fcntl.ioctl(tun, TUNSETIFF, ifr) ifname = ifname_bytes.decode('UTF-8')[:16].strip("\x00") _print(" [VPN] Interface Name: {}".format(ifname)) command = "ip addr add " + ip + "/24 dev {}" os.system(command.format(ifname)) os.system("ip link set dev {} up".format(ifname)) network_buf = ip.split(".") network_id = network_buf[0] + "." + network_buf[1] os.system("ip route add " + network_id + ".0.0/16 dev {}".format(ifname)) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((SERVER_IP, SERVER_PROT)) sock.send(ip.encode('UTF-8')) fds = [sock, tun] while True: #setup routing table #sudo ip route add origninal dst to tun0; iproute ready, _, _ = select(fds, [], []) for fd in ready: if fd is sock: data = sock.recv(4096) pkt = IP(data) #_print(" [VPN] server: {} --> {}".format(pkt.src,pkt.dst)) if len(data) > 0: #_print(" [VPN] to tun", len(data)) try: os.write(tun, data) except: pass if fd is tun: packet = os.read(tun, 4096) if len(packet) > 0: try: pkt = IP(packet) #_print(" [VPN] tun ==>: {} --> {}".format(pkt.src, pkt.dst)) sock.send(packet) except: pass
def connectionLost(self, reason): # NOTE: It looks like the NCProtocol instance will linger in memory # since ping keeps going if we don't .stop() it. try: self.lc_ping.stop() except AssertionError: pass try: self.factory.peers.pop(self.remote_nodeid) if self.nodeid != self.remote_nodeid: self.print_peers() except KeyError: if self.nodeid != self.remote_nodeid: _print(" [P2P] CLIENTS LEAVES: from", self.remote_nodeid, self.remote_ip) self.factory.clients.remove(self) self.dumpClient()
def receiveBlock(self, msg): try: newBlock = messages.read_message_noverify(msg)['block'] _print(" [<] Recieve block=" + newBlock['hash'] + " from peer " + self.remote_nodeid + ", index=" + str(newBlock['index']) + ", timestamp=" + str(newBlock['timestamp']) + ", miner=" + str(newBlock['miner'])) if BlockChainDB().verify(newBlock): self.factory.knownBlocks[self.remote_ip].append( newBlock['hash']) # only propagate valid block to peers for client in self.factory.clients: if newBlock['hash'] not in self.factory.knownBlocks[ client.remote_ip]: client.transport.write(msg) except messages.InvalidSignatureError: _print(" [!] ERROR: Invalid block sign ", self.remote_ip) except: pass
def handle_HELLO(self, hello): try: hello = messages.read_message(hello) self.remote_nodeid = hello['nodeid'] self.remote_lsnport = hello['listenPort'] if self.remote_nodeid == self.nodeid: _print(" [!] Found myself at", self.host_ip) self.transport.loseConnection() else: if self.state == "GETHELLO": my_hello = messages.create_hello_1(self.nodeid, self.lsnport, self.VERSION) self.transport.write(my_hello + "\n") self.add_peer() self.state = "READY" self.print_peers() #self.write(messages.create_ping(self.nodeid)) if self.kind == "INBOUND": # The listener pings it's audience _print(" [P2P] Starting pinger to " + self.remote_nodeid) self.lc_ping.start(PING_INTERVAL, now=False) # Tell new audience about my peers self.send_ADDR() except messages.InvalidSignatureError: _print(" [!] ERROR: Invalid hello sign ", self.remoteip) self.transport.loseConnection()
def scrap_evo(): mattr = None saldo = None driver = None try: _print(f'Start scraping') driver = browser.init_browser() _print(f'Browser starts') driver.get('https://www.evobanco.com/') time.sleep(2) _print(f'On evo.com') time.sleep(2) wait_element_css(driver, '#client_login').click() vault_evo = passwords.get_entry('evo') username = driver.find_element_by_css_selector( 'form#form1 div.form_row input[type="text"]') password = driver.find_element_by_css_selector( 'form#form1 div.form_row input[type="password"]') username.send_keys(vault_evo.username) pass_element_id = password.get_attribute('id') driver.execute_script( f'document.querySelector("#{pass_element_id}").value ="{vault_evo.password}"' ) wait_element_css(driver, '#continuar').click() _print('Logged successfully on evo') saldo = wait_element_css(driver, '#cant_saldo').text wait_element_css(driver, '#movimientosCta__0').click() next_movments_page_css_id = '#movimientosC_next' mattr = parse_movment_table(driver) while exists_element_css(driver, next_movments_page_css_id): wait_element_css(driver, next_movments_page_css_id).click() mattr.extend(parse_movment_table(driver)) time.sleep(0.5) _print(f'Has {len(mattr)} movments') finally: if driver != None: driver.close() return (mattr, saldo)
def print_peers(self): if len(self.factory.peers) == 0: _print(" [!] PEERS: No peers connected.") else: _print(" [P2P] PEERS:") for peer in self.factory.peers: addr, kind = self.factory.peers[peer][:2] _print(" [*]", peer, "at", addr, kind)
def update_urls(urls): driver = browser.init_browser() _print(f'Updating urls totals: {len(urls)}') try: for url in urls: driver.get("https://www.fotocasa.es" + url) time.sleep(3) for i in range(4): driver.execute_script(f'window.scrollTo(0,{str(500*i)})') time.sleep(.5) #html = driver.execute_script("return document.body.innerHTML") current_url = driver.execute_script("return window.location.href") ad_dir = {} ad_dir['id'] = re.findall("\d{6,}", url)[0] ad_dir['url'] = url if current_url.find('obra-nueva') > 0: scrap_obra_nueva(driver, ad_dir) else: scrap_segunda_mano(driver, ad_dir) fotocasa_db.update_ad(ad_dir) # save_page_html(html, re.findall("\d{6,}", url)[0] + ".html") finally: if driver != None: driver.close()
def periodic_scrap(): new_links = [] driver = None try: driver = browser.init_browser() existing_ad_ids = fotocasa_db.get_existing_ad_ids() found_existing_url = False now = datetime.datetime.now() now_str = now.strftime("%Y-%m-%d %H:%M:%S") _print(f'Begin periodic scrap now_str:{now_str}') j = 1 existing_counter = 0 while found_existing_url == False and existing_counter < 5: driver.get( 'https://www.fotocasa.es/es/comprar/viviendas/madrid-provincia/todas-las-zonas/l/' + str(j) + '?sortType=publicationDate') time.sleep(2) j = j + 1 no_results = driver.find_elements_by_css_selector( '.re-SearchNoResults-image') if len(no_results) == 0 or no_results[0].is_displayed() == False: for i in range(20): driver.execute_script(f'window.scrollTo(0,{str(500*i)})') time.sleep(.25) links = driver.execute_script( 'return Array.from(document.querySelectorAll("a.re-Card-link")).map(c => c.getAttribute("href"))' ) for x in links: if x.find('parking') == -1: ad_id = re.findall("\d{6,}", x)[0] if ad_id in existing_ad_ids: existing_counter = existing_counter + 1 else: fotocasa_db.insert_ad({ 'id': ad_id, 'date_created': now, 'url': x }) new_links.append(x) if existing_counter > 0: existing_counter = existing_counter - 1 else: _print('Found url that already existed in the db') found_existing_url = True _print(f'Found {len(new_links)} new links') finally: if driver != None: driver.close() update_urls(new_links)
def stopMining(self): if self.running == True: _print(' [MINER] Worker stops..') self.running = False
def send_PING(self): _print(" [>] PING to", self.remote_nodeid, "at", self.remote_ip) ping = messages.create_ping(self.nodeid) self.write(ping)
def handle_ADDR(self, addr): try: nodes = messages.read_message(addr)['nodes'] _print(" [<] Recieved addr list from peer " + self.remote_nodeid) #for node in filter(lambda n: nodes[n][1] == "SEND", nodes): for node in nodes: _print(" [*] " + node[0] + " " + node[1]) host, port = node[1].split(":") remote_lsnport = str(node[2]) if node[0] == self.nodeid: _print(" [!] Not connecting to " + node[0] + ": thats me!") continue if node[3] == "INBOUND": port = remote_lsnport _print(" [P2P] INBOUND connecting to " + host + ":" + remote_lsnport) if node[0] in self.factory.peers: _print(" [P2P] Not connecting to " + node[0] + ": already connected") continue _print(" [P2P] Trying to connect to peer " + node[0] + " " + node[1]) point = TCP4ClientEndpoint(reactor, host, int(port)) d = connectProtocol( point, NCProtocol(self.factory, "SENDHELLO", "OUTBOUND")) d.addCallback(runProtocol) except messages.InvalidSignatureError: print(addr) _print(" [!] ERROR: Invalid addr sign ", self.remote_ip) self.transport.loseConnection()
def send_HELLO(self): hello = messages.create_hello_1(self.nodeid, self.lsnport, self.VERSION) _print(" [P2P] SEND_HELLO:", self.nodeid, "to", self.remote_ip) self.transport.write(hello + "\n") self.state = "SENTHELLO"
def dumpClient(self): if len(self.factory.clients) > 0: _print(" [p2p] Connected clients") for clients in self.factory.clients: _print(" ", clients.remote_ip)
def broadcastTx(self, tx): _print(" [P2P] Broadcast tx=", tx['hash']) txMsg = messages.createTxMsg(self.nodeid, tx) for client in self.factory.clients: client.transport.write(txMsg) client.factory.knownTxs[client.remote_ip].append(tx['hash'])
time.sleep(2) if __name__ == "__main__": args = parser.parse_args() try: thread.start_new_thread(setUpVPNClient, ( args.listen, args.imAddr, args.imPort, )) time.sleep(1) endpoint = TCP4ServerEndpoint(reactor, args.port, interface=args.listen) _print(" [P2P] LISTEN ON:", args.listen, ":", args.port) ncfactory = NCFactory(args.port) endpoint.listen(ncfactory) except CannotListenError: _print("[!] Address in use") raise SystemExit # connect to bootstrap addresses _print(" [P2P] Trying to connect to bootstrap hosts:") for bootstrap in network.BOOTSTRAP_NODES + [ a + ":" + str(DEFAULT_PORT) for a in args.bootstrap ]: _print(" [*] ", bootstrap) host, port = bootstrap.split(":") point = TCP4ClientEndpoint(reactor, host, int(port))