def _scrape(self, query):
""" Make an HTTP GET request to the tracker
Note: This runs in its own thread.
"""
self.spath = "/scrape"
if not self.https:
log.warning("Warning: Will not connect to non HTTPS server")
return
try:
if self.proxy_url:
h = ProxyHTTPSConnection(self.proxy_url, \
username=self.proxy_username, \
password=self.proxy_password, \
ssl_context=self.ssl_ctx)
s = "https://%s:%d%s%s" % (self.url, self.remote_port, self.spath, query)
h.putrequest('GET', s)
else:
#No proxy url, use normal connection
h = HTTPSConnection(self.url, self.remote_port, ssl_context=self.ssl_ctx)
h.putrequest('GET', self.spath+query)
h.endheaders()
resp = h.getresponse()
data = resp.read()
resp.close()
h.close()
h = None
# urllib2 can raise various crap that doesn't have a common base
# exception class especially when proxies are used, at least
# ValueError and stuff from httplib
except Exception, g:
def f(r='Problem connecting to ' + self.url + ': ' + str(g)):
self._postrequest(errormsg=r)
def connection_completed(self):
log.info("Relay connection [%02x:%d] established" %
(int(ord(self.neighbor.id)), self.stream_id))
self.complete = True
self.flush_pre_buffer()
self.orelay.complete = True
self.orelay.flush_pre_buffer()
def socket_cb(self, sock):
""" Called by P2PConnection after connect() has completed """
if sock.connected:
log.info('Connected to %s' % str(sock.addr))
for id, v in self.incomplete.iteritems():
if v == sock.addr:
break
else:
return #loc wasn't found
AnomosNeighborInitializer(self, sock, id)
else:
#Remove nid,loc pair from incomplete
torm = []
for k, v in self.incomplete.items():
if v == sock.addr:
log.info('Failed to connect, discarding \\x%02x' % ord(k))
torm.append(k)
for j in torm:
self.rm_neighbor(j)
if sock.addr == None:
if self.incomplete.items() != []:
log.info("Remaining incomplete peers: %d" %
len(self.incomplete.items()))
else:
log.info("No remaining incomplete peers")
else:
log.info("Failed to open connection to %s\n" % str(sock.addr))
def got_have(self, message):
i = toint(message[1:])
if i >= self.torrent.numpieces:
log.error("Piece index out of range")
self.fatal_error()
return
self.download.got_have(i)
def _close(self):
if self.closed:
return
self.closed = True
# GTK Crash Hack
import time
time.sleep(.2)
self.event_handler.remove_context(self)
self._doneflag.set()
log.info("Closing connections, please wait...")
if self._announced:
self._rerequest.announce_stop()
self._rerequest.cleanup()
if self._hashcheck_thread is not None:
self._hashcheck_thread.join() # should die soon after doneflag set
if self._myfiles is not None:
self._filepool.remove_files(self._myfiles)
if self._listening:
self.neighbors.remove_torrent(self.infohash)
for port in self.reserved_ports:
self._singleport_listener.release_port(port)
if self._storage is not None:
self._storage.close()
self.schedule(0, gc.collect)
def connection_completed(self):
log.info("Relay connection [%02x:%d] established" %
(int(ord(self.neighbor.id)),self.stream_id))
self.complete = True
self.flush_pre_buffer()
self.orelay.complete = True
self.orelay.flush_pre_buffer()
def _check(self):
if self.current_started is not None:
if self.current_started <= bttime() - 58:
log.warning("Tracker announce still not complete "
"%d seconds after starting it" %
int(bttime() - self.current_started))
## Announce has been hanging for too long, retry it.
if int(bttime() - self.current_started) >= 180:
self._announce(STARTED)
return
if self.basequery is None:
self.basequery = self._makequery()
self._announce(STARTED)
return
if self.changed_port:
self._announce(STOPPED)
self.changed_port = False
self.basequery = None
return
if self.finish:
self.finish = False
self._announce(COMPLETED)
return
if self.fail_wait is not None:
if self.last_time + self.fail_wait <= bttime():
self._announce(STARTED)
return
if self.last_time > bttime() - self.config['rerequest_interval']:
return
getmore = bool(self.neighbors.failed_connections())
#TODO: also reannounce when TCs have failed
if getmore or bttime() - self.last_time > self.announce_interval:
self._announce()
def socket_cb(self, sock):
""" Called by P2PConnection after connect() has completed """
if sock.connected:
log.info('Connected to %s' %str(sock.addr))
for id,v in self.incomplete.iteritems():
if v == sock.addr:
break
else:
return #loc wasn't found
AnomosNeighborInitializer(self, sock, id)
else:
#Remove nid,loc pair from incomplete
torm = []
for k,v in self.incomplete.items():
if v == sock.addr:
log.info('Failed to connect, discarding \\x%02x' % ord(k))
torm.append(k)
for j in torm:
self.rm_neighbor(j)
if sock.addr == None:
if self.incomplete.items() != []:
log.info("Remaining incomplete peers: %d" %len(self.incomplete.items()))
else:
log.info("No remaining incomplete peers")
else:
log.info("Failed to open connection to %s\n" % str(sock.addr))
def get_ctx(self,
allow_unknown_ca=False,
req_peer_cert=True,
session=None):
ctx = SSL.Context("sslv23")
# Set certificate and private key
m2.ssl_ctx_use_x509(ctx.ctx, self.cert.x509)
m2.ssl_ctx_use_rsa_privkey(ctx.ctx, self.rsakey.rsa)
if not m2.ssl_ctx_check_privkey(ctx.ctx):
raise CryptoError('public/private key mismatch')
# Ciphers/Options
ctx.set_cipher_list(CIPHER_SET)
ctx.set_options(CTX_OPTIONS)
# CA settings
cloc = os.path.join(global_certpath, 'cacert.root.pem')
if ctx.load_verify_locations(cafile=cloc) != 1:
log.error("Problem loading CA certificates")
raise CryptoError('CA certificates not loaded')
# Verification
cb = mk_verify_cb(allow_unknown_ca=allow_unknown_ca)
CTX_V_FLAGS = SSL.verify_peer
if req_peer_cert:
CTX_V_FLAGS |= SSL.verify_fail_if_no_peer_cert
ctx.set_verify(CTX_V_FLAGS, 3, cb)
# Session
if session:
ctx.set_session_id_ctx(session)
return ctx
def handle_error(self):
t, v, tb = sys.exc_info()
if isinstance(v, KeyboardInterrupt):
raise
else:
log.info(traceback.format_exc())
self.close()
def _close(self):
if self.closed:
return
self.closed = True
# GTK Crash Hack
import time
time.sleep(0.2)
self.event_handler.remove_context(self)
self._doneflag.set()
log.info("Closing connections, please wait...")
if self._announced:
self._rerequest.announce_stop()
self._rerequest.cleanup()
if self._hashcheck_thread is not None:
self._hashcheck_thread.join() # should die soon after doneflag set
if self._myfiles is not None:
self._filepool.remove_files(self._myfiles)
if self._listening:
self.neighbors.remove_torrent(self.infohash)
for port in self.reserved_ports:
self._singleport_listener.release_port(port)
if self._storage is not None:
self._storage.close()
self.schedule(0, gc.collect)
def get_ctx(self, allow_unknown_ca=False, req_peer_cert=True, session=None):
ctx = SSL.Context("sslv23")
# Set certificate and private key
m2.ssl_ctx_use_x509(ctx.ctx, self.cert.x509)
m2.ssl_ctx_use_rsa_privkey(ctx.ctx, self.rsakey.rsa)
if not m2.ssl_ctx_check_privkey(ctx.ctx):
raise CryptoError('public/private key mismatch')
# Ciphers/Options
ctx.set_cipher_list(CIPHER_SET)
ctx.set_options(CTX_OPTIONS)
# CA settings
cloc = os.path.join(global_certpath, 'cacert.root.pem')
if ctx.load_verify_locations(cafile=cloc) != 1:
log.error("Problem loading CA certificates")
raise CryptoError('CA certificates not loaded')
# Verification
cb = mk_verify_cb(allow_unknown_ca=allow_unknown_ca)
CTX_V_FLAGS = SSL.verify_peer
if req_peer_cert:
CTX_V_FLAGS |= SSL.verify_fail_if_no_peer_cert
ctx.set_verify(CTX_V_FLAGS,3,cb)
# Session
if session:
ctx.set_session_id_ctx(session)
return ctx
def read_header(self, data):
data = data.strip()
if data == '':
# check for Accept-Encoding: header, pick a
if self.headers.has_key('accept-encoding'):
ae = self.headers['accept-encoding']
log.debug("Got Accept-Encoding: " + ae + "\n")
else:
#identity assumed if no header
ae = 'identity'
# this eventually needs to support multple acceptable types
# q-values and all that fancy HTTP crap
# for now assume we're only communicating with our own client
if ae.find('gzip') != -1:
self.encoding = 'gzip'
else:
#default to identity.
self.encoding = 'identity'
r = self.getfunc(self, self.path, self.headers)
if r is not None:
self.answer(r)
return None
try:
i = data.index(':')
except ValueError:
return None
self.headers[data[:i].strip().lower()] = data[i+1:].strip()
log.debug(data[:i].strip() + ": " + data[i+1:].strip())
return self.read_header
def got_tcode(self, message):
tcreader = TCReader(self.manager.certificate)
try:
tcdata = tcreader.parseTC(message[1:])
except Anomos.Crypto.CryptoError, e:
log.error("Decryption Error: %s" % str(e))
self.socket.close()
return
def __init__(self, *args):
apply(gtk.Window.__init__, (self,)+args)
try:
#TODO: Icon doesn't work on XP build, don't know why
if (os.name != 'nt'):
self.set_icon_from_file(os.path.join(image_root,'anomos.ico'))
except Exception, e:
log.warning(e)
def _load_fastresume(self, resumefile, typecode):
if resumefile is not None:
try:
r = array(typecode)
r.fromfile(resumefile, self.numpieces)
return r
except Exception, e:
log.warning("Couldn't read fastresume data: " + str(e))
def __init__(self, *args):
apply(gtk.Window.__init__, (self, ) + args)
try:
#TODO: Icon doesn't work on XP build, don't know why
if (os.name != 'nt'):
self.set_icon_from_file(os.path.join(image_root, 'anomos.ico'))
except Exception, e:
log.warning(e)
def found_terminator(self):
creq = self.req
self.req = ''
if not self.next_func:
log.info("Malformed request from %s:%d" % self.socket.addr)
self.handle_close()
return
self.next_func = self.next_func(creq)
def close(self):
if self.closed:
log.warning("Double close")
return
log.info("Closing %s"%self.uniq_id())
if self.complete:
self.send_break()
self.shutdown()
class AnomosNeighborProtocol(AnomosProtocol):
## NeighborProtocol is intended to be implemented by NeighborLink ##
def __init__(self):
AnomosProtocol.__init__(self)
self.msgmap.update({PARTIAL:self.got_partial,
TCODE: self.got_tcode})
def format_message(self, stream_id, message):
return tobinary(stream_id)[2:] + \
tobinary(len(message)) + message
def invalid_message(self, t):
log.warning("Invalid message of type %02x on %s. Closing neighbor."% \
(ord(t), self.uniq_id()))
self.socket.close()
def got_partial(self, message):
p_remain = toint(message[1:5])
payload = message[5:]
self.partial_recv += payload
if len(self.partial_recv) > self.config['max_message_length']:
log.error("Received message longer than max length")
return
if len(payload) == p_remain:
self.got_message(self.partial_recv)
self.partial_recv = ''
def got_tcode(self, message):
tcreader = TCReader(self.manager.certificate)
try:
tcdata = tcreader.parseTC(message[1:])
except Anomos.Crypto.CryptoError, e:
log.error("Decryption Error: %s" % str(e))
self.socket.close()
return
sid = tcdata.sessionID
if not self.manager.check_session_id(sid):
#TODO: Key mismatch is pretty serious, probably want to ban the
# user who sent this TCode
log.error("Session id mismatch")
self.socket.close()
return
if tcdata.type == chr(0): # Relayer type
nextTC = tcdata.nextLayer
nid = tcdata.neighborID
self.start_relay_stream(nid, nextTC)
elif tcdata.type == chr(1): # Terminal type
infohash = tcdata.infohash
keydata = tcdata.keydata
e2e_key = Anomos.Crypto.AESKey(keydata[:32],keydata[32:])
torrent = self.manager.get_torrent(infohash)
if not torrent:
log.error("Requested torrent not found")
self.socket.close()
return
self.start_endpoint_stream(torrent, e2e_key)
else:
log.error("Unsupported TCode Format")
self.socket.close()
def got_partial(self, message):
p_remain = toint(message[1:5])
self.partial_recv += message[5:]
if len(self.partial_recv) > self.neighbor.config['max_message_length']:
log.error("Received message longer than max length")
return
if len(message[5:]) == p_remain:
self.got_message(self.partial_recv)
self.partial_recv = ''
def send_tracking_code(self, trackcode):
#XXX: Just a test, Throw tcodes into the PMQ instead of sending them
# immediately
#self.network_ctl_msg(TCODE, trackcode)
log.info("Queuing tracking code")
self.neighbor.queue_message(self.stream_id, TCODE+trackcode)
if self.next_upload is None:
log.info("Queuing self")
self.ratelimiter.queue(self)
def get_reader(self):
if self.collector:
if self.new_collector:
self.new_collector = False
return self.collector.get_reader()
else:
log.critical("Request for a reader was made before " \
"a connection was assigned a collector")
raise RuntimeError("Unable to get data collector")
def ore_closed(self):
""" Closes the connection when a Break has been received by our
other relay (ore). Called by this object's ore during
shutdown """
if self.closed:
log.warning("Double close")
return
if not self.sent_break:
self.send_break()
def ore_closed(self):
""" Closes the connection when a Break has been received by our
other relay (ore). Called by this object's ore during
shutdown """
if self.closed:
log.warning("Double close")
return
if not self.sent_break:
self.send_break()
def close(self):
# Connection was closed locally (as opposed to
# being closed by receiving a BREAK message)
if self.closed:
log.warning("%s: Double close" % self.uniq_id())
return
log.info("Closing %s" % self.uniq_id())
if self.complete and not self.sent_break:
self.send_break()
self.shutdown()
def close(self):
# Connection was closed locally (as opposed to
# being closed by receiving a BREAK message)
if self.closed:
log.warning("%s: Double close" % self.uniq_id())
return
log.info("Closing %s"%self.uniq_id())
if self.complete and not self.sent_break:
self.send_break()
self.shutdown()
def start_circuit(self, tc, infohash, aeskey):
"""Called from Rerequester to initialize new circuits we've
just gotten TCs for from the Tracker"""
if self.count_streams() >= self.config['max_initiate']:
log.warning("Not starting circuit -- Stream count exceeds maximum")
return
tcreader = TCReader(self.certificate)
try:
tcdata = tcreader.parseTC(tc)
except Anomos.Crypto.CryptoError, e:
log.error("Decryption Error: %s" % str(e))
return
def start_circuit(self, tc, infohash, aeskey):
"""Called from Rerequester to initialize new circuits we've
just gotten TCs for from the Tracker"""
if self.count_streams() >= self.config['max_initiate']:
log.warning("Not starting circuit -- Stream count exceeds maximum")
return
tcreader = TCReader(self.certificate)
try:
tcdata = tcreader.parseTC(tc)
except Anomos.Crypto.CryptoError, e:
log.error("Decryption Error: %s" % str(e))
return
def shutdown(self):
if self.closed:
log.warning("Double close")
return
self.closed = True
if not (self.decremented_count or
(self.orelay and self.orelay.decremented_count)):
self.manager.dec_relay_count()
self.decremented_count = True
# Tell our orelay to close.
if self.orelay and not self.orelay.closed:
self.orelay.ore_closed()
self.ratelimiter.clean_closed()
def connection_completed(self):
""" Called when a CONFIRM message is received
indicating that our peer has received our
tracking code """
if self.complete:
log.warning("Double complete")
return
self.complete = True
self.upload = self.torrent.make_upload(self)
self.choker = self.upload.choker
self.choker.connection_made(self)
self.download = self.torrent.make_download(self)
self.torrent.add_active_stream(self)
def shutdown(self):
if self.closed:
log.warning("Double close")
return
self.closed = True
if not (self.decremented_count or
(self.orelay and self.orelay.decremented_count)):
self.manager.dec_relay_count()
self.decremented_count = True
# Tell our orelay to close.
if self.orelay and not self.orelay.closed:
self.orelay.ore_closed()
self.ratelimiter.clean_closed()
def _parsepeers(self, p):
peers = []
if type(p) == str:
for x in xrange(0, len(p), 6):
ip = '.'.join([str(ord(i)) for i in p[x:x+4]])
log.info("Got peer %s"%ip)
port = (ord(p[x+4]) << 8) | ord(p[x+5])
peers.append((ip, port, None))
else:
for x in p:
log.info("Got peer %s"%str(x['ip']))
peers.append((x['ip'], x['port'], x.get('nid')))
return peers
def socket_cb(self, sock):
if sock.connected:
peercert = self.socket.get_peer_cert()
recvd_pid = peercert.get_fingerprint('sha256')[-20:]
if self.peerid != recvd_pid:
# The certificate we received doesn't match the one
# given to the tracker.
# XXX: Should probably disconnect the peer rather than
# just saying the NatCheck failed.
log.warning("Peer certificate mismatch")
self.answer(False)
AnomosNeighborInitializer(self, self.socket, self.id)
else:
self.answer(False)
def connection_completed(self, socket, id):
"""Called by AnomosNeighborInitializer"""
if self.incomplete.has_key(id):
del self.incomplete[id]
if id == NAT_CHECK_ID:
log.info("NAT check ok.")
return
self.add_neighbor(socket, id)
tasks = self.waiting_tcs.get(id)
if tasks is None:
return
for task in tasks:
#TODO: Would a minimum wait between these tasks aid anonymity?
self.schedule(0, task)
del self.waiting_tcs[id]
def connection_completed(self, socket, id):
"""Called by AnomosNeighborInitializer"""
if self.incomplete.has_key(id):
del self.incomplete[id]
if id == NAT_CHECK_ID:
log.info("NAT check ok.")
return
self.add_neighbor(socket, id)
tasks = self.waiting_tcs.get(id)
if tasks is None:
return
for task in tasks:
#TODO: Would a minimum wait between these tasks aid anonymity?
self.schedule(0, task)
del self.waiting_tcs[id]
def loop(self):
try:
while not self.doneflag.isSet():
self._pop_externally_added()
period = 1e9
if len(self.tasks) > 0:
# Poll until the next task is set to execute
period = max(0, self.tasks[0][0] - bttime())
asyncore.poll(period)
self.do_tasks()
except KeyboardInterrupt:
#TODO: cleanup?
pass
except:
log.critical('\n'+traceback.format_exc())
def loop(self):
try:
while not self.doneflag.isSet():
self._pop_externally_added()
period = 1e9
if len(self.tasks) > 0:
# Poll until the next task is set to execute
period = max(0, self.tasks[0][0] - bttime())
asyncore.poll(period)
self.do_tasks()
except KeyboardInterrupt:
#TODO: cleanup?
pass
except:
log.critical('\n' + traceback.format_exc())
def socket_cb(self, sock):
if sock.connected:
peercert = self.socket.get_peer_cert()
recvd_pid = peercert.get_fingerprint('sha256')[-20:]
if self.peerid != recvd_pid:
# The certificate we received doesn't match the one
# given to the tracker.
# XXX: Should probably disconnect the peer rather than
# just saying the NatCheck failed.
log.warning("Peer certificate mismatch")
self.answer(False)
AnomosNeighborInitializer(self, self.socket, self.id)
else:
self.answer(False)
def __init__(self, url, config, schedule, neighbors, amount_left,
up, down, local_port, infohash, doneflag,
diefunc, sfunc, certificate, sessionid):
##########################
self.config = config
self.schedule = schedule
self.neighbors = neighbors
self.amount_left = amount_left
self.up = up
self.down = down
self.local_port = local_port
self.infohash = infohash
self.doneflag = doneflag
self.diefunc = diefunc
self.successfunc = sfunc
self.certificate = certificate
self.ssl_ctx = self.certificate.get_ctx(allow_unknown_ca=False)
self.sessionid = sessionid
### Tracker URL ###
self.https = True
parsed = urlparse(url) # (<scheme>,<netloc>,<path>,<params>,<query>,<fragment>)
self.url = parsed[1]
self.remote_port = 5555 # Assume port 5555 by default
if ":" in self.url: # <netloc> = <url>:<port>
i = self.url.index(":")
self.remote_port = int(self.url[i+1:])
self.url = self.url[:i]
self.path = parsed[2]
self.basequery = None
self.failed_peers = []
self.changed_port = False
self.announce_interval = 30 * 60
self.finish = False
self.current_started = None
self.fail_wait = None
self.last_time = None
self.warned = False
self.proxy_url = self.config.get('tracker_proxy', None)
self.proxy_username = None
self.proxy_password = None
if self.proxy_url:
self.parse_proxy_url()
if parsed[0] != 'https':
log.error("You are trying to make an unencrypted connection to a tracker, and this has been disabled for security reasons. Halting.")
self.https = False
def _postrequest(self, data=None, errormsg=None):
self.current_started = None
self.last_time = bttime()
if errormsg is not None:
log.warning(errormsg)
self._fail()
return
try:
# Here's where we receive/decrypt data from the tracker
r = bdecode(data)
check_peers(r)
except BTFailure, e:
if data != '':
log.error('bad data from tracker - ' + str(e))
self._fail()
return
def update_neighbor_list(self, list):
freshids = dict([(i[2],(i[0],i[1])) for i in list]) #{nid : (ip, port)}
# Remove neighbors not found in freshids
for id in self.neighbors.keys():
if not freshids.has_key(id):
self.rm_neighbor(id)
# Start connections with the new neighbors
for id, loc in freshids.iteritems():
if self.nid_collision(id, loc):
# Already had neighbor by the given id at a different location
log.warning('NID collision - x%02x' % ord(id))
# To be safe, kill connection with the neighbor we already
# had with the requested ID and add ID to the failed list
self.rm_neighbor(id)
elif (not self.has_neighbor(id)) and (id not in self.failedPeers):
self.start_connection(id, loc)
def update_neighbor_list(self, list):
freshids = dict([(i[2], (i[0], i[1]))
for i in list]) #{nid : (ip, port)}
# Remove neighbors not found in freshids
for id in self.neighbors.keys():
if not freshids.has_key(id):
self.rm_neighbor(id)
# Start connections with the new neighbors
for id, loc in freshids.iteritems():
if self.nid_collision(id, loc):
# Already had neighbor by the given id at a different location
log.warning('NID collision - x%02x' % ord(id))
# To be safe, kill connection with the neighbor we already
# had with the requested ID and add ID to the failed list
self.rm_neighbor(id)
elif (not self.has_neighbor(id)) and (id not in self.failedPeers):
self.start_connection(id, loc)
def _load(self):
"""Attempts to load the certificate and key from self.certfile and self.keyfile,
Generates the certificate and key if they don't exist"""
if not self.secure:
self.rsakey = RSA.load_key(self.keyfile, m2util.no_passphrase_callback)
else:
# Allow 3 attempts before quitting
i = 0
while i < 3:
try:
self.rsakey = RSA.load_key(self.keyfile)
break
except RSA.RSAError:
i += 1
else:
log.warning("\nInvalid password entered, exiting.")
sys.exit()
self.cert = X509.load_cert(self.certfile)