def dataReceived(self, data):
for line in data.splitlines():
line = line.strip()
try:
envelopes = messages.read_envelope(line)
for envelope in envelopes:
if self.state in ["GETHELLO", "SENTHELLO"]:
# Force first message to be HELLO or crash
if envelope['msgtype'] == 'hello':
self.handle_HELLO(json.dumps(envelope))
else:
_print(" [!] Ignoring", envelope['msgtype'], "in",
self.state)
else:
if envelope['msgtype'] == 'ping':
self.handle_PING(json.dumps(envelope))
elif envelope['msgtype'] == 'pong':
self.handle_PONG(json.dumps(envelope))
elif envelope['msgtype'] == 'addr':
self.handle_ADDR(json.dumps(envelope))
elif envelope['msgtype'] == 'tx':
self.receiveTx(json.dumps(envelope))
elif envelope['msgtype'] == 'block':
self.receiveBlock(json.dumps(envelope))
except:
pass
def handle_PONG(self, pong):
pong = messages.read_message(pong)
_print(" [<] PONG from", self.remote_nodeid, "at", self.remote_ip)
# hacky
addr, kind, listenPort = self.factory.peers[self.remote_nodeid][:3]
self.factory.peers[self.remote_nodeid] = (addr, kind, listenPort,
time())
def MiningWorker(self):
while True:
if self.running:
newBlock = self.mine()
_print(' [MINER] Mined new block:', newBlock['index'])
reactor.callFromThread(self.protocol.broadcastBlock, newBlock)
time.sleep(10)
def send_ADDR(self):
_print(" [>] Telling " + self.remote_nodeid + " about my peers")
# Shouldn't this be a list and not a dict?
peers = self.factory.peers
listeners = [(n, peers[n][0], peers[n][1], peers[n][2]) for n in peers]
addr = messages.create_addr(self.nodeid, listeners)
self.write(addr)
def submitTransaction(self, transaction):
if self.verify_transaction(transaction) is True:
UnTransactionDB().insert(transaction.to_dict())
_print(" [TX] Created new transaction")
reactor.callFromThread(self.protocol.broadcastTx,
transaction.to_dict())
return transaction.to_dict()['hash']
def broadcastBlock(self, block):
_print(" [P2P] Broadcast block=" + str(block['hash']) + ", index=" +
str(block['index']) + ", miner=" + str(block['miner']))
blockMsg = messages.createBlockMsg(self.nodeid, block)
for client in self.factory.clients:
client.transport.write(blockMsg)
client.factory.knownBlocks[client.remote_ip].append(block['hash'])
def startFactory(self):
self.peers = {}
self.clients = []
self.knownBlocks = {}
self.knownTxs = {}
self.numProtocols = 0
self.nodeid = cryptotools.generate_nodeid()[:10]
_print(" [P2P] NODEID:", self.nodeid)
def __init__(self, ip, port, protocol):
http_server = pyjsonrpc.ThreadingHttpServer(
server_address=(ip, port), RequestHandlerClass=RequestHandler)
http_server.account = Account(protocol)
http_server.miner = Miner(protocol)
_print(" [RPC] Starting RPC server ...")
_print(" [RPC] URL: http://" + ip + ":" + str(port))
http_server.serve_forever()
def evo_weekly(output, budget):
try:
if output == 'console':
log._print(evo.weekly_report_text(budget))
if output == 'email':
notification.send_notification("EVO Weekly report",
evo.weekly_report_html(budget))
except:
exception_handler._handle_exception("Exception in EVO weekly report")
def _handle_exception(msg="Exception in personal Automation"):
"""
Handles notification by looking at environment variable PA_EXCEPTIONS
to decide the notification output channel.
msg is the message of the exception.
"""
output = os.environ.get('PA_EXCEPTIONS', 'console')
if output == 'console':
log._print(traceback.format_exc())
if output == 'email':
notification.send_notification(msg, traceback.format_exc())
def send_notification(subject, text):
hotmail = passwords.get_entry('hotmail')
msg = MIMEMultipart()
message = text
password = hotmail.password
msg['From'] = hotmail.username
msg['To'] = "*****@*****.**"
msg['Subject'] = subject
#msg.attach(MIMEText(message, 'plain'))
msg.attach(MIMEText(message, 'html'))
server = smtplib.SMTP('smtp.live.com: 587')
server.starttls()
server.login(msg['From'], password)
server.sendmail(msg['From'], msg['To'], msg.as_string())
server.quit()
_print("successfully sent email to %s:" % (msg['To']))
def receiveTx(self, msg):
try:
newTx = messages.read_message_noverify(msg)['tx']
_print(" [<] Recieve tx=" + newTx['hash'] + " from peer " +
self.remote_nodeid)
UnTransactionDB().insert(newTx)
self.factory.knownTxs[self.remote_ip].append(newTx['hash'])
# propagate to the unknown peers
for client in self.factory.clients:
if newTx['hash'] not in self.factory.knownTxs[
client.remote_ip]:
client.transport.write(msg)
except messages.InvalidSignatureError:
_print(" [!] ERROR: Invalid tx sign ", self.remote_ip)
self.transport.loseConnection()
def __init__(self, ip, s_ip, s_port):
TUNSETIFF = 0X400454ca
IFF_TUN = 0x0001
IFF_TAP = 0x0002
IFF_NO_PI = 0x1000
SERVER_IP = s_ip #"128.230.208.73"
SERVER_PROT = int(s_port) #int(sys.argv[1])
tun = os.open("/dev/net/tun", os.O_RDWR)
ifr = struct.pack('16sH', b'tun%d', IFF_TUN | IFF_NO_PI)
ifname_bytes = fcntl.ioctl(tun, TUNSETIFF, ifr)
ifname = ifname_bytes.decode('UTF-8')[:16].strip("\x00")
_print(" [VPN] Interface Name: {}".format(ifname))
command = "ip addr add " + ip + "/24 dev {}"
os.system(command.format(ifname))
os.system("ip link set dev {} up".format(ifname))
network_buf = ip.split(".")
network_id = network_buf[0] + "." + network_buf[1]
os.system("ip route add " + network_id +
".0.0/16 dev {}".format(ifname))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((SERVER_IP, SERVER_PROT))
sock.send(ip.encode('UTF-8'))
fds = [sock, tun]
while True:
#setup routing table
#sudo ip route add origninal dst to tun0; iproute
ready, _, _ = select(fds, [], [])
for fd in ready:
if fd is sock:
data = sock.recv(4096)
pkt = IP(data)
#_print(" [VPN] server: {} --> {}".format(pkt.src,pkt.dst))
if len(data) > 0:
#_print(" [VPN] to tun", len(data))
try:
os.write(tun, data)
except:
pass
if fd is tun:
packet = os.read(tun, 4096)
if len(packet) > 0:
try:
pkt = IP(packet)
#_print(" [VPN] tun ==>: {} --> {}".format(pkt.src, pkt.dst))
sock.send(packet)
except:
pass
def connectionLost(self, reason):
# NOTE: It looks like the NCProtocol instance will linger in memory
# since ping keeps going if we don't .stop() it.
try:
self.lc_ping.stop()
except AssertionError:
pass
try:
self.factory.peers.pop(self.remote_nodeid)
if self.nodeid != self.remote_nodeid:
self.print_peers()
except KeyError:
if self.nodeid != self.remote_nodeid:
_print(" [P2P] CLIENTS LEAVES: from", self.remote_nodeid,
self.remote_ip)
self.factory.clients.remove(self)
self.dumpClient()
def receiveBlock(self, msg):
try:
newBlock = messages.read_message_noverify(msg)['block']
_print(" [<] Recieve block=" + newBlock['hash'] + " from peer " +
self.remote_nodeid + ", index=" + str(newBlock['index']) +
", timestamp=" + str(newBlock['timestamp']) + ", miner=" +
str(newBlock['miner']))
if BlockChainDB().verify(newBlock):
self.factory.knownBlocks[self.remote_ip].append(
newBlock['hash'])
# only propagate valid block to peers
for client in self.factory.clients:
if newBlock['hash'] not in self.factory.knownBlocks[
client.remote_ip]:
client.transport.write(msg)
except messages.InvalidSignatureError:
_print(" [!] ERROR: Invalid block sign ", self.remote_ip)
except:
pass
def handle_HELLO(self, hello):
try:
hello = messages.read_message(hello)
self.remote_nodeid = hello['nodeid']
self.remote_lsnport = hello['listenPort']
if self.remote_nodeid == self.nodeid:
_print(" [!] Found myself at", self.host_ip)
self.transport.loseConnection()
else:
if self.state == "GETHELLO":
my_hello = messages.create_hello_1(self.nodeid,
self.lsnport,
self.VERSION)
self.transport.write(my_hello + "\n")
self.add_peer()
self.state = "READY"
self.print_peers()
#self.write(messages.create_ping(self.nodeid))
if self.kind == "INBOUND":
# The listener pings it's audience
_print(" [P2P] Starting pinger to " + self.remote_nodeid)
self.lc_ping.start(PING_INTERVAL, now=False)
# Tell new audience about my peers
self.send_ADDR()
except messages.InvalidSignatureError:
_print(" [!] ERROR: Invalid hello sign ", self.remoteip)
self.transport.loseConnection()
def scrap_evo():
mattr = None
saldo = None
driver = None
try:
_print(f'Start scraping')
driver = browser.init_browser()
_print(f'Browser starts')
driver.get('https://www.evobanco.com/')
time.sleep(2)
_print(f'On evo.com')
time.sleep(2)
wait_element_css(driver, '#client_login').click()
vault_evo = passwords.get_entry('evo')
username = driver.find_element_by_css_selector(
'form#form1 div.form_row input[type="text"]')
password = driver.find_element_by_css_selector(
'form#form1 div.form_row input[type="password"]')
username.send_keys(vault_evo.username)
pass_element_id = password.get_attribute('id')
driver.execute_script(
f'document.querySelector("#{pass_element_id}").value ="{vault_evo.password}"'
)
wait_element_css(driver, '#continuar').click()
_print('Logged successfully on evo')
saldo = wait_element_css(driver, '#cant_saldo').text
wait_element_css(driver, '#movimientosCta__0').click()
next_movments_page_css_id = '#movimientosC_next'
mattr = parse_movment_table(driver)
while exists_element_css(driver, next_movments_page_css_id):
wait_element_css(driver, next_movments_page_css_id).click()
mattr.extend(parse_movment_table(driver))
time.sleep(0.5)
_print(f'Has {len(mattr)} movments')
finally:
if driver != None:
driver.close()
return (mattr, saldo)
def print_peers(self):
if len(self.factory.peers) == 0:
_print(" [!] PEERS: No peers connected.")
else:
_print(" [P2P] PEERS:")
for peer in self.factory.peers:
addr, kind = self.factory.peers[peer][:2]
_print(" [*]", peer, "at", addr, kind)
def update_urls(urls):
driver = browser.init_browser()
_print(f'Updating urls totals: {len(urls)}')
try:
for url in urls:
driver.get("https://www.fotocasa.es" + url)
time.sleep(3)
for i in range(4):
driver.execute_script(f'window.scrollTo(0,{str(500*i)})')
time.sleep(.5)
#html = driver.execute_script("return document.body.innerHTML")
current_url = driver.execute_script("return window.location.href")
ad_dir = {}
ad_dir['id'] = re.findall("\d{6,}", url)[0]
ad_dir['url'] = url
if current_url.find('obra-nueva') > 0:
scrap_obra_nueva(driver, ad_dir)
else:
scrap_segunda_mano(driver, ad_dir)
fotocasa_db.update_ad(ad_dir)
# save_page_html(html, re.findall("\d{6,}", url)[0] + ".html")
finally:
if driver != None:
driver.close()
def periodic_scrap():
new_links = []
driver = None
try:
driver = browser.init_browser()
existing_ad_ids = fotocasa_db.get_existing_ad_ids()
found_existing_url = False
now = datetime.datetime.now()
now_str = now.strftime("%Y-%m-%d %H:%M:%S")
_print(f'Begin periodic scrap now_str:{now_str}')
j = 1
existing_counter = 0
while found_existing_url == False and existing_counter < 5:
driver.get(
'https://www.fotocasa.es/es/comprar/viviendas/madrid-provincia/todas-las-zonas/l/'
+ str(j) + '?sortType=publicationDate')
time.sleep(2)
j = j + 1
no_results = driver.find_elements_by_css_selector(
'.re-SearchNoResults-image')
if len(no_results) == 0 or no_results[0].is_displayed() == False:
for i in range(20):
driver.execute_script(f'window.scrollTo(0,{str(500*i)})')
time.sleep(.25)
links = driver.execute_script(
'return Array.from(document.querySelectorAll("a.re-Card-link")).map(c => c.getAttribute("href"))'
)
for x in links:
if x.find('parking') == -1:
ad_id = re.findall("\d{6,}", x)[0]
if ad_id in existing_ad_ids:
existing_counter = existing_counter + 1
else:
fotocasa_db.insert_ad({
'id': ad_id,
'date_created': now,
'url': x
})
new_links.append(x)
if existing_counter > 0:
existing_counter = existing_counter - 1
else:
_print('Found url that already existed in the db')
found_existing_url = True
_print(f'Found {len(new_links)} new links')
finally:
if driver != None:
driver.close()
update_urls(new_links)
def stopMining(self):
if self.running == True:
_print(' [MINER] Worker stops..')
self.running = False
def send_PING(self):
_print(" [>] PING to", self.remote_nodeid, "at", self.remote_ip)
ping = messages.create_ping(self.nodeid)
self.write(ping)
def handle_ADDR(self, addr):
try:
nodes = messages.read_message(addr)['nodes']
_print(" [<] Recieved addr list from peer " + self.remote_nodeid)
#for node in filter(lambda n: nodes[n][1] == "SEND", nodes):
for node in nodes:
_print(" [*] " + node[0] + " " + node[1])
host, port = node[1].split(":")
remote_lsnport = str(node[2])
if node[0] == self.nodeid:
_print(" [!] Not connecting to " + node[0] + ": thats me!")
continue
if node[3] == "INBOUND":
port = remote_lsnport
_print(" [P2P] INBOUND connecting to " + host + ":" +
remote_lsnport)
if node[0] in self.factory.peers:
_print(" [P2P] Not connecting to " + node[0] +
": already connected")
continue
_print(" [P2P] Trying to connect to peer " + node[0] + " " +
node[1])
point = TCP4ClientEndpoint(reactor, host, int(port))
d = connectProtocol(
point, NCProtocol(self.factory, "SENDHELLO", "OUTBOUND"))
d.addCallback(runProtocol)
except messages.InvalidSignatureError:
print(addr)
_print(" [!] ERROR: Invalid addr sign ", self.remote_ip)
self.transport.loseConnection()
def send_HELLO(self):
hello = messages.create_hello_1(self.nodeid, self.lsnport,
self.VERSION)
_print(" [P2P] SEND_HELLO:", self.nodeid, "to", self.remote_ip)
self.transport.write(hello + "\n")
self.state = "SENTHELLO"
def dumpClient(self):
if len(self.factory.clients) > 0:
_print(" [p2p] Connected clients")
for clients in self.factory.clients:
_print(" ", clients.remote_ip)
def broadcastTx(self, tx):
_print(" [P2P] Broadcast tx=", tx['hash'])
txMsg = messages.createTxMsg(self.nodeid, tx)
for client in self.factory.clients:
client.transport.write(txMsg)
client.factory.knownTxs[client.remote_ip].append(tx['hash'])
time.sleep(2)
if __name__ == "__main__":
args = parser.parse_args()
try:
thread.start_new_thread(setUpVPNClient, (
args.listen,
args.imAddr,
args.imPort,
))
time.sleep(1)
endpoint = TCP4ServerEndpoint(reactor,
args.port,
interface=args.listen)
_print(" [P2P] LISTEN ON:", args.listen, ":", args.port)
ncfactory = NCFactory(args.port)
endpoint.listen(ncfactory)
except CannotListenError:
_print("[!] Address in use")
raise SystemExit
# connect to bootstrap addresses
_print(" [P2P] Trying to connect to bootstrap hosts:")
for bootstrap in network.BOOTSTRAP_NODES + [
a + ":" + str(DEFAULT_PORT) for a in args.bootstrap
]:
_print(" [*] ", bootstrap)
host, port = bootstrap.split(":")
point = TCP4ClientEndpoint(reactor, host, int(port))
