class LaunchMany: def __init__(self, config, Output): try: self.config = config self.Output = Output self.torrent_dir = config["torrent_dir"] self.torrent_cache = {} self.file_cache = {} self.blocked_files = {} self.scan_period = config["parse_dir_interval"] self.stats_period = config["display_interval"] self.torrent_list = [] self.downloads = {} self.counter = 0 self.doneflag = Event() self.hashcheck_queue = [] self.hashcheck_current = None self.rawserver = RawServer( self.doneflag, config["timeout_check_interval"], config["timeout"], ipv6_enable=config["ipv6_enabled"], failfunc=self.failed, errorfunc=self.exchandler, ) upnp_type = UPnP_test(config["upnp_nat_access"]) while 1: try: self.listen_port = self.rawserver.find_and_bind( config["minport"], config["maxport"], config["bind"], ipv6_socket_style=config["ipv6_binds_v4"], upnp=upnp_type, randomizer=config["random_port"], ) break except socketerror, e: if upnp_type and e == UPnP_ERROR: self.Output.message("WARNING: COULD NOT FORWARD VIA UPnP") upnp_type = 0 continue self.failed("Couldn't listen - " + str(e)) return self.ratelimiter = RateLimiter(self.rawserver.add_task, config["upload_unit_size"]) self.ratelimiter.set_upload_rate(config["max_upload_rate"]) self.handler = MultiHandler(self.rawserver, self.doneflag) seed(createPeerID()) self.rawserver.add_task(self.scan, 0) self.rawserver.add_task(self.stats, 0) self.start() except:
def __init__(self, config, Output): try: self.config = config self.Output = Output self.torrent_dir = config['torrent_dir'] self.torrent_cache = {} self.file_cache = {} self.blocked_files = {} self.scan_period = config['parse_dir_interval'] self.stats_period = config['display_interval'] self.torrent_list = [] self.downloads = {} self.counter = 0 self.doneflag = Event() self.hashcheck_queue = [] self.hashcheck_current = None self.rawserver = RawServer(self.doneflag, config['timeout_check_interval'], config['timeout'], ipv6_enable = config['ipv6_enabled'], failfunc = self.failed, errorfunc = self.exchandler) upnp_type = UPnP_test(config['upnp_nat_access']) while True: try: self.listen_port = self.rawserver.find_and_bind( config['minport'], config['maxport'], config['bind'], ipv6_socket_style = config['ipv6_binds_v4'], upnp = upnp_type, randomizer = config['random_port']) break except socketerror, e: if upnp_type and e == UPnP_ERROR: self.Output.message('WARNING: COULD NOT FORWARD VIA UPnP') upnp_type = 0 continue self.failed("Couldn't listen - " + str(e)) return self.ratelimiter = RateLimiter(self.rawserver.add_task, config['upload_unit_size']) self.ratelimiter.set_upload_rate(config['max_upload_rate']) self.handler = MultiHandler(self.rawserver, self.doneflag) seed(createPeerID()) self.rawserver.add_task(self.scan, 0) self.rawserver.add_task(self.stats, 0) self.handler.listen_forever() self.Output.message('shutting down') self.hashcheck_queue = [] for hash in self.torrent_list: self.Output.message('dropped "'+self.torrent_cache[hash]['path']+'"') self.downloads[hash].shutdown() self.rawserver.shutdown()
class LaunchMany: def __init__(self, config, Output): try: self.config = config self.Output = Output self.torrent_dir = config['torrent_dir'] self.torrent_cache = {} self.file_cache = {} self.blocked_files = {} self.scan_period = config['parse_dir_interval'] self.stats_period = config['display_interval'] self.torrent_list = [] self.downloads = {} self.counter = 0 self.doneflag = Event() self.hashcheck_queue = [] self.hashcheck_current = None self.rawserver = RawServer(self.doneflag, config['timeout_check_interval'], config['timeout'], ipv6_enable = config['ipv6_enabled'], failfunc = self.failed, errorfunc = self.exchandler) upnp_type = UPnP_test(config['upnp_nat_access']) while True: try: self.listen_port = self.rawserver.find_and_bind( config['minport'], config['maxport'], config['bind'], ipv6_socket_style = config['ipv6_binds_v4'], upnp = upnp_type, randomizer = config['random_port']) break except socketerror, e: if upnp_type and e == UPnP_ERROR: self.Output.message('WARNING: COULD NOT FORWARD VIA UPnP') upnp_type = 0 continue self.failed("Couldn't listen - " + str(e)) return self.ratelimiter = RateLimiter(self.rawserver.add_task, config['upload_unit_size']) self.ratelimiter.set_upload_rate(config['max_upload_rate']) self.handler = MultiHandler(self.rawserver, self.doneflag, config) seed(createPeerID()) self.rawserver.add_task(self.scan, 0) self.rawserver.add_task(self.stats, 0) self.handler.listen_forever() self.Output.message('shutting down') self.hashcheck_queue = [] for hash in self.torrent_list: self.Output.message('dropped "'+self.torrent_cache[hash]['path']+'"') self.downloads[hash].shutdown() self.rawserver.shutdown() except:
def __init__(self, host, port, dbDir, ipv6_enable = False, upnp = 0, natpmp = False): self.host = host self.port = port self.dbDir = dbDir self.store = None self.rawserver = RawServer(self, host, port, ipv6_enable, upnp, natpmp) self.tokensHandler = TokensHandler(self) self.krpc = KRPC(self) self.contacts = [] # [ip,] - contacts added by bittorrent self.announce = {} # {ip:[hash,],} - nodes that announced us
def __init__(self, config): self.socket = {} self.rs = RawServer(Event(), 100, 1000) self.nh = NetworkHandler(self) self.shutdown = Event() self.config = config self.post_commit = [] for pattern, action in self.config.items('post-commit'): try: self.post_commit.append((re.compile(pattern, re.I), action)) except re.error, msg: raise ServerError, 'Bad post-commit pattern \"%s\": %s' % \ (pattern, msg)
class LaunchMany: def __init__(self, config, Output): try: self.config = config self.Output = Output self.torrent_dir = config['torrent_dir'] self.torrent_cache = {} self.file_cache = {} self.blocked_files = {} self.scan_period = config['parse_dir_interval'] self.stats_period = config['display_interval'] self.torrent_list = [] self.downloads = {} self.counter = 0 self.doneflag = Event() self.hashcheck_queue = [] self.hashcheck_current = None self.rawserver = RawServer( self.doneflag, config['timeout_check_interval'], config['timeout'], ipv6_enable=config['ipv6_enabled'], failfunc=self.failed, errorfunc=self.exchandler) upnp_type = UPnP_test(config['upnp_nat_access']) while True: try: self.listen_port = self.rawserver.find_and_bind( config['minport'], config['maxport'], config['bind'], ipv6_socket_style=config['ipv6_binds_v4'], upnp=upnp_type, randomizer=config['random_port']) break except socket.error as e: if upnp_type and e == UPnP_ERROR: self.Output.message( 'WARNING: COULD NOT FORWARD VIA UPnP') upnp_type = 0 continue self.failed("Couldn't listen - " + str(e)) return self.ratelimiter = RateLimiter(self.rawserver.add_task, config['upload_unit_size']) self.ratelimiter.set_upload_rate(config['max_upload_rate']) self.handler = MultiHandler(self.rawserver, self.doneflag, config) seed(createPeerID()) self.rawserver.add_task(self.scan, 0) self.rawserver.add_task(self.stats, 0) self.handler.listen_forever() self.Output.message('shutting down') self.hashcheck_queue = [] for hash in self.torrent_list: self.Output.message('dropped "{}"'.format( self.torrent_cache[hash]['path'])) self.downloads[hash].shutdown() self.rawserver.shutdown() except: data = StringIO() print_exc(file=data) Output.exception(data.getvalue()) def scan(self): self.rawserver.add_task(self.scan, self.scan_period) r = parsedir(self.torrent_dir, self.torrent_cache, self.file_cache, self.blocked_files, return_metainfo=True, errfunc=self.Output.message) (self.torrent_cache, self.file_cache, self.blocked_files, added, removed) = r for hash, data in removed.iteritems(): self.Output.message('dropped "{}"'.format(data['path'])) self.remove(hash) for hash, data in added.iteritems(): self.Output.message('added "{}"'.format(data['path'])) self.add(hash, data) def stats(self): self.rawserver.add_task(self.stats, self.stats_period) data = [] for hash in self.torrent_list: cache = self.torrent_cache[hash] if self.config['display_path']: name = cache['path'] else: name = cache['name'] size = cache['length'] d = self.downloads[hash] progress = '0.0%' peers = 0 seeds = 0 seedsmsg = "S" dist = 0.0 uprate = 0.0 dnrate = 0.0 upamt = 0 dnamt = 0 t = 0 if d.is_dead(): status = 'stopped' elif d.waiting: status = 'waiting for hash check' elif d.checking: status = d.status_msg progress = '{:.1%}'.format(d.status_done) else: stats = d.statsfunc() s = stats['stats'] if d.seed: status = 'seeding' progress = '100.0%' seeds = s.numOldSeeds seedsmsg = "s" dist = s.numCopies else: if s.numSeeds + s.numPeers: t = stats['time'] if t == 0: # unlikely t = 0.01 status = fmttime(t) else: t = -1 status = 'connecting to peers' progress = '{:.1%}'.format(stats['frac']) seeds = s.numSeeds dist = s.numCopies2 dnrate = stats['down'] peers = s.numPeers uprate = stats['up'] upamt = s.upTotal dnamt = s.downTotal if d.is_dead() or d.status_errtime + 300 > clock(): msg = d.status_err[-1] else: msg = '' data.append((name, status, progress, peers, seeds, seedsmsg, dist, uprate, dnrate, upamt, dnamt, size, t, msg)) stop = self.Output.display(data) if stop: self.doneflag.set() def remove(self, hash): self.torrent_list.remove(hash) self.downloads[hash].shutdown() del self.downloads[hash] def add(self, hash, data): c = self.counter self.counter += 1 x = '' for i in xrange(3): x = mapbase64[c & 0x3F] + x c >>= 6 peer_id = createPeerID(x) d = SingleDownload(self, hash, data['metainfo'], self.config, peer_id) self.torrent_list.append(hash) self.downloads[hash] = d d.start() def saveAs(self, hash, name, saveas, isdir): x = self.torrent_cache[hash] style = self.config['saveas_style'] if style == 1 or style == 3: if saveas: saveas = os.path.join(saveas, x['file'][:-1 - len(x['type'])]) else: saveas = x['path'][:-1 - len(x['type'])] if style == 3: if not os.path.isdir(saveas): try: os.mkdir(saveas) except: raise OSError("couldn't create directory for {} ({})" ''.format(x['path'], saveas)) if not isdir: saveas = os.path.join(saveas, name) else: if saveas: saveas = os.path.join(saveas, name) else: saveas = os.path.join(os.path.split(x['path'])[0], name) if isdir and not os.path.isdir(saveas): try: os.mkdir(saveas) except: raise OSError("couldn't create directory for {} ({})".format( x['path'], saveas)) return saveas def hashchecksched(self, hash=None): if hash: self.hashcheck_queue.append(hash) if not self.hashcheck_current: self._hashcheck_start() def _hashcheck_start(self): self.hashcheck_current = self.hashcheck_queue.pop(0) self.downloads[self.hashcheck_current].hashcheck_start( self.hashcheck_callback) def hashcheck_callback(self): self.downloads[self.hashcheck_current].hashcheck_callback() if self.hashcheck_queue: self._hashcheck_start() else: self.hashcheck_current = None def died(self, hash): if hash in self.torrent_cache: self.Output.message('DIED: "{}"'.format( self.torrent_cache[hash]['path'])) def was_stopped(self, hash): try: self.hashcheck_queue.remove(hash) except: pass if self.hashcheck_current == hash: self.hashcheck_current = None if self.hashcheck_queue: self._hashcheck_start() def failed(self, s): self.Output.message('FAILURE: ' + s) def exchandler(self, s): self.Output.exception(s)
if v[0] == 172 and v[1] >= 16 and v[1] <= 31: return 1 except ValueError: return 0 def track(args): if len(args) == 0: print formatDefinitions(defaults, 80) return try: config, files = parseargs(args, defaults, 0, 0) except ValueError, e: print 'error: ' + str(e) print 'run with no arguments for parameter explanations' return r = RawServer(Event(), config['timeout_check_interval'], config['socket_timeout']) t = Tracker(config, r) r.bind(config['port'], config['bind'], True) r.listen_forever(HTTPHandler(t.get, config['min_time_between_log_flushes'])) t.save_dfile() print '# Shutting down: ' + isotime() def size_format(s): if (s < 1024): r = str(s) + 'B' elif (s < 1048576): r = str(int(s/1024)) + 'KiB' elif (s < 1073741824L): r = str(int(s/1048576)) + 'MiB' elif (s < 1099511627776L): r = str(int((s/1073741824.0)*100.0)/100.0) + 'GiB'
def __init__(self, co): self.co = co self.socket = {} self.rs = RawServer(Event(), 100, 1000) self.nh = NetworkHandler(self)
class ClientHandler: def __init__(self, co): self.co = co self.socket = {} self.rs = RawServer(Event(), 100, 1000) self.nh = NetworkHandler(self) # Functions called from lower level code def message_came_in(self, s, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'garbage data' if msg.has_key('error'): raise ServerError, msg['error'] socket = self.socket[s] srp = socket['srp'] if socket['state'] == 1: K, m = self.auth.client_key(msg['s'], msg['B'], msg['u'], srp['keys']) socket['key'], socket['m_out'] = K, m self._send_msg(s, {'m': socket['m_out'].digest()}) socket['state'] = 2 elif socket['state'] == 2: socket['m_in'] = SRP.host_authenticator(socket['key'], srp['keys'][0], socket['m_out'].digest()) if socket['m_in'].digest() != msg['auth']: raise ServerError, 'Bad host authentication' return self.nh.set_hmac(s, socket['m_in'], socket['m_out']) self.rs.doneflag.set() elif socket['state'] == 3: self.socket[s]['hash'] = msg['hash'] self.rs.doneflag.set() elif socket['state'] == 4: self.close(s) secret = crypt(msg['secret'], socket['key'])[0] self.auth.save_secret(secret) self.rs.doneflag.set() elif socket['state'] == 5: self.close(s) self.rs.doneflag.set() elif socket['state'] == 6: if len(msg['salt']) < 20: self._send_error(s, None, 'Bad salt length') self.close(s) raise NetworkError, 'Bad salt from server' salt = random_string(20) key = self.auth.session_key(salt, msg['salt']) socket['m_in'] = hmac.new(key, '', sha) key = self.auth.session_key(msg['salt'], salt) socket['m_out'] = hmac.new(key, '', sha) self._send_msg(s, {'auth': socket['m_in'].digest(), 'salt': salt}) socket['state'] = 7 elif socket['state'] == 7: if msg['auth'] != socket['m_out'].digest(): self._send_error(s, None, 'Bad auth') self.close(s) raise NetworkError, 'Bad server auth' self._req_mode(s, 1) self.nh.set_hmac(s, socket['m_in'], socket['m_out']) self.socket[s] = [{}, {}, {}, [], 1] self.rs.doneflag.set() else: self.close(s) def connection_flushed(self, s): queue = self.socket[s][Queue] socket = self.socket[s] socket[Flushed] = 1 while len(queue) and socket[Flushed] == 1: mid, msg = queue.pop(0) diff = read_diff(self.co, msg['handle'], msg['changenum'], None) socket[Flushed] = self._send_response(s, mid, {'diff': diff}) def connection_lost(self, s, msg): del self.socket[s] raise NetworkError, msg def request_came_in(self, s, mid, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'garbage request' self.request_handlers[msg['request']](self, s, mid, msg) def response_came_in(self, s, mid, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'bad data from server' rstate = self.socket[s][Request][mid] if msg.has_key('error'): # XXX: grody hack, diffs which don't belong in history if rstate['request'] == 'get diff': msg['diff'] = zlib.compress(bencode('')) else: raise ServerError, msg['error'] if self.response_handlers[rstate['request']](self, s, mid, msg, rstate): del self.socket[s][Request][mid] # request handlers def _request_get_change(self, s, mid, msg): resp = {'changeset': self.co.lcrepo.get(msg['changenum'], txn=self.txn)} self._send_response(s, mid, resp) def _request_get_diff(self, s, mid, msg): if self.socket[s][Flushed] == 1: diff = read_diff(self.co, msg['handle'], msg['changenum'], self.txn) self.socket[s][Flushed] = self._send_response(s, mid, {'diff': diff}) else: self.socket[s][Queue].append((mid, msg)) request_handlers = {'get change': _request_get_change, 'get diff': _request_get_diff} # response handlers def _response_get_head(self, s, mid, msg, rstate): return self._merge_change(s, mid, msg['head']) def _merge_change(self, s, mid, head): lstate = self.socket[s][Request][mid] lstate['head'] = head self.socket[s][UpdateInfo]['head'] = head if self.co.lcrepo.has_key(head): named, modified, added, deleted = \ handles_in_branch(self.co, lstate['heads'], [head], None) self._update_checks(s, mid, named, modified) self._update_handle_list(s, lstate, named, modified, added, deleted) self._update_finish(s, lstate) self.socket[s][UpdateInfo]['head'] = head self.rs.doneflag.set() return 1 rid = self._get_change(s, head) self.socket[s][Request][rid] = {'request': 'get change', 'changenum': head, 'ref': mid} lstate['requests'][head] = 1 lstate['count'] = 1 return 0 def _response_get_change(self, s, mid, msg, rstate): lstate = self.socket[s][Request][rstate['ref']] if sha.new(msg['changeset']).digest() != rstate['changenum']: raise ServerError, 'bad changeset' # write it out, decode and eject from memory write_changeset(self.co, rstate['changenum'], msg['changeset'], lstate['txn']) changeset = bdecode(msg['changeset']) lstate['changes'][rstate['changenum']] = changeset del msg['changeset'] # get any precursors we don't have and haven't yet requested for change in changeset['precursors']: if self.co.lcrepo.has_key(change): continue if lstate['changes'].has_key(change): continue if lstate['requests'].has_key(change): continue rid = self._get_change(s, change) self.socket[s][Request][rid] = {'request': 'get change', 'changenum': change, 'ref': rstate['ref']} lstate['requests'][change] = 1 lstate['count'] += 1 # record all the diffs we'll need to request diffs = lstate['diffs'] for handle, hinfo in changeset['handles'].items(): if not hinfo.has_key('hash'): continue if not diffs.has_key(handle): diffs[handle] = {} lstate['counts'][handle] = 0 diffs[handle][rstate['changenum']] = 1 lstate['counts'][handle] += 1 # clean up state del lstate['requests'][rstate['changenum']] lstate['count'] -= 1 if lstate['count'] == 0: sync_history(self.co, lstate['head'], lstate['txn'], cache=lstate['changes']) named, modified, added, deleted = \ handles_in_branch(self.co, lstate['heads'], [lstate['head']], lstate['txn'], cache=lstate['changes']) del lstate['changes'] self._update_checks(s, rstate['ref'], named, modified) self._update_handle_list(s, lstate, named, modified, added, deleted) handle_list = lstate['handle list'] # get all the related file diffs for i in xrange(len(handle_list)-1, -1, -1): handle = handle_list[i][1] if not diffs.has_key(handle): continue changes = diffs[handle] requested = 0 for change in changes.keys(): requested = 1 self._queue_diff(s, change, handle, rstate['ref']) lstate['count'] += requested self._get_diff(s, rstate['ref']) if lstate['count'] == 0: self._update_finish(s, lstate) del self.socket[s][Request][rstate['ref']] self.rs.doneflag.set() return 1 def _response_get_diff(self, s, mid, msg, rstate): lstate = self.socket[s][Request][rstate['ref']] # send out the next one lstate['req-outstanding'] -= 1 self._get_diff(s, rstate['ref']) diffs = lstate['diffs'] diffs[rstate['handle']][rstate['change']] = msg['diff'] lstate['counts'][rstate['handle']] -= 1 if lstate['counts'][rstate['handle']] == 0: lstate['count'] -= 1 # XXX: do better ordering WD = WriteDiff(self.co, rstate['handle'], lstate['txn']) for change, diff in diffs[rstate['handle']].items(): WD.write(diff, change) WD.close() if lstate['modified'].has_key(rstate['handle']): updates(self.co, self.socket[s][UpdateInfo], lstate, rstate['handle']) #del diffs[rstate['handle']] if lstate['count'] == 0: self._update_finish(s, lstate) self.socket[s][UpdateInfo]['head'] = lstate['head'] del self.socket[s][Request][rstate['ref']] self.rs.doneflag.set() return 1 def _response_return(self, s, mid, msg, rstate): self.rs.doneflag.set() return 1 def _response_list_repositories(self, s, mid, msg, rstate): self.socket[s][UpdateInfo] = msg['list'] self.rs.doneflag.set() return 1 response_handlers = {'get head': _response_get_head, 'get change': _response_get_change, 'get diff': _response_get_diff, 'commit': _response_return, 'create repository': _response_return, 'destroy repository': _response_return, 'list repositories': _response_list_repositories} # Internal helper functions def _update_checks(self, s, mid, named, modified): lstate = self.socket[s][Request][mid] check_modified(self.co, modified) lstate['modified'] = {} for handle in modified: lstate['modified'][handle] = 1 self.socket[s][UpdateInfo]['modified'] = [] self.socket[s][UpdateInfo]['modified conflicts'] = [] def _update_handle_list(self, s, lstate, named, modified, added, deleted): uinfo = self.socket[s][UpdateInfo] func = lstate['update function'] handles, nconflicts = func(self.co, uinfo, named, modified, added, deleted, lstate['txn']) handle_list = lstate['handle list'] for handle, show in handles: letters = (' ', ' ') if uinfo['newfiles'].has_key(handle): letters = ('A', 'A') elif uinfo['deletes'].has_key(handle): letters = ('D', 'D') elif uinfo['names'].has_key(handle): letters = (' ', 'N') if nconflicts.has_key(handle): letters = (letters[0], 'C') handle_list.append((handle_to_filename(self.co, handle, lstate['txn']), handle, show, letters)) handle_list.sort() handle_list.reverse() uinfo['name conflicts'] = nconflicts.keys() def _update_finish(self, s, lstate): updates(self.co, self.socket[s][UpdateInfo], lstate, None) def _req_mode(self, s, mode): self.nh.req_mode(s, mode) def _commit(self, s, repo, head): rid = self._send_request(s, {'request': 'commit', 'repository': repo, 'changenum': head}) self.socket[s][Request][rid] = {'request': 'commit'} def _update(self, s, repo, func, txn): heads = bdecode(self.co.linforepo.get('heads')) lstate = {'request': 'get head', 'txn': txn, 'changes': {}, 'requests': {}, 'diffs': {}, 'count': 0, 'counts': {}, 'handle list': [], 'heads': heads, 'reqq': [], 'req-outstanding': 0, 'update function': func} if repo.startswith('{') and repo.endswith('}'): head = binascii.unhexlify(repo[1:-1]) rid = self.nh.next_id(s) self.socket[s][Request][rid] = lstate if self._merge_change(s, rid, head): del self.socket[s][Request][rid] else: rid = self._send_request(s, {'request': 'get head', 'repository': repo}) self.socket[s][Request][rid] = lstate return def _create_repo(self, s, repo): rid = self._send_request(s, {'request': 'create repository', 'repository': repo}) self.socket[s][Request][rid] = {'request': 'create repository', 'repository': repo} def _remove_repo(self, s, repo): rid = self._send_request(s, {'request': 'destroy repository', 'repository': repo}) self.socket[s][Request][rid] = {'request': 'destroy repository', 'repository': repo} def _list_repos(self, s): rid = self._send_request(s, {'request': 'list repositories'}) self.socket[s][Request][rid] = {'request': 'list repositories'} def _get_change(self, s, change): req = {'request': 'get change', 'changenum': change} return self._send_request(s, req) def _queue_diff(self, s, change, handle, mid): rstate = self.socket[s][Request][mid] rstate['reqq'].append((change, handle)) def _get_diff(self, s, mid): rstate = self.socket[s][Request][mid] while len(rstate['reqq']) and rstate['req-outstanding'] <= 40: change, handle = rstate['reqq'].pop(0) req = {'request': 'get diff', 'changenum': change, 'handle': handle} state = {'request': 'get diff', 'ref': mid, 'change': change, 'handle': handle} rid = self._send_request(s, req) self.socket[s][Request][rid] = state rstate['req-outstanding'] += 1 def _send_msg(self, s, data): self.nh.send_msg(s, bencode(data)) def _send_error(self, s, mid, msg): self._send_response(s, mid, {'error': msg}) def _send_request(self, s, data): return self.nh.send_request(s, bencode(data)) def _send_response(self, s, mid, data): return self.nh.send_response(s, mid, bencode(data)) # Functions to be called from higher level code def start_connection(self, dns): s = self.nh.start_connection(dns) self.socket[s] = {'state': 0, 'srp': {}} return s def close(self, s): self.nh.close(s) del self.socket[s] def get_hash(self, s): self._send_msg(s, {'op': 'get hash', 'user': self.co.user}) self.socket[s]['state'] = 3 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() secret_hash = self.socket[s]['hash'] del self.socket[s]['hash'] return secret_hash def srp_auth(self, s): socket = self.socket[s] assert socket['state'] == 0 or socket['state'] == 3 srp = socket['srp'] = {} srp['keys'] = SRP.client_begin() self._send_msg(s, {'op': 'srp auth', 'user': self.co.user, 'A': srp['keys'][0]}) socket['state'] = 1 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def get_secret(self, s): self._send_msg(s, {'op': 'get secret'}) self.socket[s]['state'] = 4 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() return def set_password(self, s, password): socket = self.socket[s] salt, v = SRP.new_passwd(self.co.user, password) cypherv = crypt(long_to_string(v), socket['key'])[0] self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv}) socket['state'] = 5 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def secret_auth(self, s): socket = self.socket[s] self._send_msg(s, {'op': 'secret auth', 'user': self.co.user}) socket['state'] = 6 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def commit(self, s, repo, changenum, txn): self.txn = txn self._commit(s, repo, changenum) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def update(self, s, repo, func, txn): self._update(s, repo, func, txn) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() updateinfo = self.socket[s][UpdateInfo] del self.socket[s][UpdateInfo] return updateinfo def create_repo(self, s, repo): self._create_repo(s, repo) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def remove_repo(self, s, repo): self._remove_repo(s, repo) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def list_repos(self, s): self._list_repos(s) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() rlist = self.socket[s][UpdateInfo] del self.socket[s][UpdateInfo] return rlist
# To connect, run: nc localhost 8080 from threading import Event from RawServer import RawServer done_flag = Event() class Handler: def external_connection_made(self, connection): connection.write("hello there stranger!\n") def connection_flushed(self, connection): # Wrote string, so close all connections and quit. done_flag.set() timeout_check_interval = 60 * 60 timeout = timeout_check_interval server = RawServer(done_flag, timeout_check_interval, timeout) server.bind(8080) server.listen_forever(Handler())
class Factory: """ Background functionality """ def __init__(self, host, port, dbDir, ipv6_enable = False, upnp = 0, natpmp = False): self.host = host self.port = port self.dbDir = dbDir self.store = None self.rawserver = RawServer(self, host, port, ipv6_enable, upnp, natpmp) self.tokensHandler = TokensHandler(self) self.krpc = KRPC(self) self.contacts = [] # [ip,] - contacts added by bittorrent self.announce = {} # {ip:[hash,],} - nodes that announced us def Node(self): """Create a new node""" return KNode(self) def start(self): """Start Factory""" self.rawserver.add_task(self._init) self.rawserver.add_task(self._checkpoint, 60) self.rawserver.start() self.rawserver.add_task(self._cleanDataBase, KEINITIAL_DELAY) self.rawserver.add_task(self.refreshTable, 5, [True]) def _init(self): """Initialize Factory""" self._loadDB() self._loadSelfNode() self._loadRoutingTable() def _close(self): """Close Factory""" self._updateDB() self.rawserver.shutdown() #################### # Database Handler #################### def _loadDB(self): """Load the database""" if DEBUG: print("Debug: DHT - _loadDB") # connect self.store = sqlite3.connect(os.path.join(self.dbDir.encode("UTF-8"), "dht.db")) self.store.text_factory = str # create if neccacery c = self.store.cursor() statements = ["create table kv (key binary, value binary, age timestamp, primary key (key, value))", "create table nodes (id binary primary key, host text, port number)", "create table self (num number primary key, id binary, age timestamp)"] try: [c.execute(s) for s in statements] except sqlite3.OperationalError: pass else: self.store.commit() c.close() def _closeDB(self): """Close the database""" self.store.close() def _loadSelfNode(self): """Load the root node""" if DEBUG: print("Debug: DHT - loadSelfNode") # Get ID c = self.store.cursor() c.execute('select id, age from self where num = 0') data = c.fetchone() # Clean if too old if not data or time() - data[1] > 86400*5: # more than 5 days old id = newID() c.execute('delete from self') c.execute("insert into self values (0, ?, ?)", (sqlite3.Binary(id), time())) c.execute('delete from kv') c.execute('delete from nodes') self.store.commit() else: id = str(data[0]) c.close() # Load self node self.node = self.Node().init(id, self.host, self.port) def _saveSelfNode(self): """Save the root node""" if DEBUG: print("Debug: DHT - saveSelfNode") c = self.store.cursor() c.execute('delete from self') c.execute("insert into self values (0, ?, ?)", (sqlite3.Binary(self.node.id), time())) self.store.commit() c.close() def _loadRoutingTable(self): """Load routing table from the database""" self.table = KTable(self.node) c = self.store.cursor() c.execute("select id, host, port from nodes") for row in c: n = self.Node().init(str(row[0]), row[1], row[2]) self.table.insertNode(n, contacted = False) c.close() if DEBUG: print("Debug: DHT - nodes loaded:", self.stats()) def _saveRoutingTable(self): """Save routing table nodes to the database""" if DEBUG: print("Debug: DHT - saveRoutingTable") c = self.store.cursor() c.execute("delete from nodes") for bucket in self.table.buckets: for node in bucket.l: c.execute("insert into nodes values (?, ?, ?)", (sqlite3.Binary(node.id), node.host, node.port)) self.store.commit() c.close() def _updateDB(self): """Save info to database""" if DEBUG: print("Debug: DHT - updateDB") self._saveSelfNode() self._saveRoutingTable() if DEBUG: print("Debug: DHT - updateDB completed") def _flushExpired(self): """Clean old values from database""" c = self.store.cursor() c.execute("delete from kv where age > ?", (KE_AGE,)) self.store.commit() c.close() def _retrieveValue(self, key): """Returns the value found for key in local table""" values = [] c = self.store.cursor() c.execute("select value from kv where key = ?", (sqlite3.Binary(key),)) for row in c: values.append(str(row[0])) c.close() return values[:20] def _storeValue(self, key, value): """Stores <key:value> pair in the database""" c = self.store.cursor() try: c.execute("insert into kv values (?, ?, ?);", (sqlite3.Binary(key), sqlite3.Binary(value), time())) except sqlite3.IntegrityError: c.execute("update kv set age = ? where key = ? and value = ?", (time(), sqlite3.Binary(key), sqlite3.Binary(value))) self.store.commit() c.close() #################### # Automatic updates #################### def _checkpoint(self): """Make some saving and refreshing once in a while""" if DEBUG: print("Debug: DHT - checkpoint") # Save DB to disk self._updateDB() # Find close nodes self.findCloseNodes() # Refresh Table self.refreshTable() self.rawserver.add_task(self._checkpoint, randrange(int(CHECKPOINT_INTERVAL * .9), int(CHECKPOINT_INTERVAL * 1.1))) def _cleanDataBase(self): self._flushExpired() self.rawserver.add_task(self._cleanDataBase, KE_DELAY) #################### # Interface #################### def addContact(self, host, port): """ Ping this node and add the contact info to the table on pong! """ # Validation if not isinstance(port, int): port = int(port) if not isinstance(host, str): host = str(host) if host in self.contacts: return False self.contacts.append(host) # Add Contact if is_valid_ip(host): n = self.Node().init(NULL_ID, host, port) self.sendPing(n) else: Thread(target = self.addRouterContact, args = [host, port]).start() def addRouterContact(self, host, port): try: host = socket.gethostbyname(host) except socket.error: return False n = self.Node().init(NULL_ID, host, port) self.sendPing(n) return True def insertNode(self, n, contacted = True): """ Insert a node in our local table, pinging oldest contact in bucket, if necessary If all you have is a host/port, then use addContact, which calls this method after receiving the PONG from the remote node. The reason for the seperation is we can't insert a node into the table without it's peer-ID. That means of course the node passed into this method needs to be a properly formed Node object with a valid ID. """ old = self.table.insertNode(n, contacted = contacted) if old and (clock() - old.lastSeen) > MIN_PING_INTERVAL and old.id != self.node.id: # the bucket is full, check to see if old node is still around and if so, replace it ## these are the callbacks used when we ping the oldest node in a bucket def _staleNodeHandler(oldnode=old, newnode = n): """ called if the pinged node never responds """ self.table.replaceStaleNode(old, newnode) def _notStaleNodeHandler(dict, old=old): """ called when we get a pong from the old node """ dict = dict['rsp'] if dict['id'] == old.id: self.table.justSeenNode(old.id) try: df = old.ping(self.node.id) except KrpcGenericError: _staleNodeHandler() else: df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler) def findCloseNodes(self, callback=lambda a: None): """ This does a findNode on the ID one away from our own. This will allow us to populate our table with nodes on our network closest to our own. This is called as soon as we start up with an empty table """ if DEBUG: print("Debug: DHT - findCloseNodes") id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256) self.findNode(id, callback) def refreshTable(self, force = False, callback=lambda a: None): """ Refresh the table force=True will refresh table regardless of last bucket access time """ if DEBUG: print("Debug: DHT - refreshTable") for bucket in self.table.buckets: if force or (clock() - bucket.lastAccessed >= BUCKET_STALENESS): id = newIDInRange(bucket.min, bucket.max) self.findNode(id, callback) def stats(self): """ Returns the number of contacts in our routing table """ return reduce(lambda a, b: a + len(b.l), self.table.buckets, 0) #################### # RPC Handler #################### def krpc_ping(self, id, _krpc_sender, **kwargs): """Incoming RPC: got ping""" if len(id) != 20: raise KrpcProtocolError("invalid id length: %d" % len(id)) n = self.Node().init(id, *_krpc_sender) self.insertNode(n, contacted = False) return {"id" : self.node.id} def krpc_find_node(self, target, id, _krpc_sender, **kwargs): """Incoming RPC: got find_node""" if len(id) != 20: raise KrpcProtocolError("invalid id length: %d" % len(id)) if len(target) != 20: raise KrpcProtocolError("invalid target id length: %d" % len(target)) nodes = self.table.findNodes(target) nodes = map(lambda node: node.senderDict(), nodes) n = self.Node().init(id, *_krpc_sender) self.insertNode(n, contacted = False) return {"nodes" : encodeNodes(nodes), "id" : self.node.id} def krpc_get_peers(self, id, info_hash, _krpc_sender, **kwargs): """Incoming RPC: got get_peers""" if len(id) != 20: raise KrpcProtocolError("invalid id length: %d" % len(id)) if len(info_hash) != 20: raise KrpcProtocolError("invalid info_hash length: %d" % len(info_hash)) if id == NULL_ID: raise KrpcProtocolError("invalid id (NULL ID)") n = self.Node().init(id, *_krpc_sender) self.insertNode(n, contacted = False) l = self._retrieveValue(info_hash) if len(l) > 0: return {'values' : l, "id": self.node.id, "token" : self.tokensHandler.tokenToSend(info_hash, *_krpc_sender)} else: nodes = self.table.findNodes(info_hash) nodes = map(lambda node: node.senderDict(), nodes) return {'nodes' : encodeNodes(nodes), "id": self.node.id, "token" : self.tokensHandler.tokenToSend(info_hash, *_krpc_sender)} def krpc_announce_peer(self, id, info_hash, port, token, _krpc_sender, **kwargs): """Incoming RPC: got announce_peer""" if len(id) != 20: raise KrpcProtocolError("invalid id length: %d" % len(id)) if len(info_hash) != 20: raise KrpcProtocolError("invalid info_hash length: %d" % len(info_hash)) if not isinstance(port, int): try: port = int(port) except: raise KrpcProtocolError("invalid port") if not self.tokensHandler.checkToken(token, info_hash, *_krpc_sender): raise KrpcProtocolError("Got invalid token") # TODO: add a limit: maximum 3 info_hash announces per peer ip = _krpc_sender[0] if ip not in self.announce: self.announce[ip] = [] if info_hash not in self.announce[ip]: self.announce[ip].append(info_hash) if len(self.announce[ip]) > 3: raise KrpcGenericError("I only allow 3 infohash announces per peer!") self._storeValue(info_hash, encodePeer((_krpc_sender[0], port))) n = self.Node().init(id, *_krpc_sender) self.insertNode(n, contacted = False) return {"id" : self.node.id} def sendPing(self, node, callback=None): """ Ping a node """ def _pongHandler(dict, node=node, table=self.table, callback=callback): _krpc_sender = dict['_krpc_sender'] id = dict['rsp']['id'] if len(id) == 20: n = self.Node().init(dict['rsp']['id'], *_krpc_sender) table.insertNode(n) if callback: callback() def _defaultPong(err, node=node, table=self.table, callback=callback): table.nodeFailed(node) if callback: callback() try: df = node.ping(self.node.id) except KrpcGenericError: _defaultPong() else: df.addCallbacks(_pongHandler,_defaultPong) def findNode(self, id, callback, errback = None): """ Returns the the k closest nodes to that ID from the global table """ # get K nodes out of local table/cache nodes = self.table.findNodes(id) d = Deferred() if errback: d.addCallbacks(callback, errback) else: d.addCallback(callback) # create our search state state = FindNode(self, id, d.callback) self.rawserver.add_task(state.goWithNodes, 0, [nodes]) def getPeers(self, key, callback, searchlocal = True, donecallback = None): """ Get Value from global table """ if not hasattr(self, "store"): self.rawserver.add_task(self.getPeers, 3, [key, callback, searchlocal, donecallback]) return # get locals if searchlocal: l = self._retrieveValue(key) if len(l) > 0: l = decodePeers(l) self.rawserver.add_task(callback, 0, [l]) else: l = [] # create our search state nodes = self.table.findNodes(key) state = GetValue(self, key, callback, "getPeers", donecallback) self.rawserver.add_task(state.goWithNodes, 0, [nodes, l]) def announcePeer(self, key, value, callback=None): """ Store Value in global table """ def _storeValueForKey(nodes, key=key, value=value, response=callback , table=self.table): if not response: # default callback def _storedValueHandler(sender): pass response=_storedValueHandler action = StoreValue(self, key, value, response, "announcePeer") self.rawserver.add_task(action.goWithNodes, 0, [nodes]) # this call is asynch self.findNode(key, _storeValueForKey) def getPeersAndAnnounce(self, key, value, callback, searchlocal = True): """ Get value and store it """ def doneCallback(nodes, key = key, value = value): action = StoreValue(self, key, value, None, "announcePeer") self.rawserver.add_task(action.goWithNodes, 0, [nodes]) self.getPeers(key, callback, searchlocal, doneCallback)
return 1 except ValueError: return 0 def track(args): if len(args) == 0: print formatDefinitions(defaults, 80) return try: config, files = parseargs(args, defaults, 0, 0) except ValueError, e: print 'error: ' + str(e) print 'run with no arguments for parameter explanations' return r = RawServer(Event(), config['timeout_check_interval'], config['socket_timeout']) t = Tracker(config, r) r.bind(config['port'], config['bind'], True) r.listen_forever(HTTPHandler(t.get, config['min_time_between_log_flushes'])) t.save_dfile() print '# Shutting down: ' + isotime() def size_format(s): if (s < 1024): r = str(s) + 'B' elif (s < 1048576): r = str(int(s / 1024)) + 'KiB' elif (s < 1073741824l): r = str(int(s / 1048576)) + 'MiB'
class LaunchMany: def __init__(self, config, Output): try: self.config = config self.Output = Output self.torrent_dir = config['torrent_dir'] self.torrent_cache = {} self.file_cache = {} self.blocked_files = {} self.scan_period = config['parse_dir_interval'] self.stats_period = config['display_interval'] self.torrent_list = [] self.downloads = {} self.counter = 0 self.doneflag = Event() self.hashcheck_queue = [] self.hashcheck_current = None self.rawserver = RawServer(self.doneflag, config['timeout_check_interval'], config['timeout'], ipv6_enable=config['ipv6_enabled'], failfunc=self.failed, errorfunc=self.exchandler) upnp_type = UPnP_test(config['upnp_nat_access']) while True: try: self.listen_port = self.rawserver.find_and_bind( config['minport'], config['maxport'], config['bind'], ipv6_socket_style=config['ipv6_binds_v4'], upnp=upnp_type, randomizer=config['random_port']) break except socket.error as e: if upnp_type and e == UPnP_ERROR: self.Output.message( 'WARNING: COULD NOT FORWARD VIA UPnP') upnp_type = 0 continue self.failed("Couldn't listen - " + str(e)) return self.ratelimiter = RateLimiter(self.rawserver.add_task, config['upload_unit_size']) self.ratelimiter.set_upload_rate(config['max_upload_rate']) self.handler = MultiHandler(self.rawserver, self.doneflag, config) seed(createPeerID()) self.rawserver.add_task(self.scan, 0) self.rawserver.add_task(self.stats, 0) self.handler.listen_forever() self.Output.message('shutting down') self.hashcheck_queue = [] for hash in self.torrent_list: self.Output.message('dropped "{}"'.format( self.torrent_cache[hash]['path'])) self.downloads[hash].shutdown() self.rawserver.shutdown() except: data = StringIO() print_exc(file=data) Output.exception(data.getvalue()) def scan(self): self.rawserver.add_task(self.scan, self.scan_period) r = parsedir(self.torrent_dir, self.torrent_cache, self.file_cache, self.blocked_files, return_metainfo=True, errfunc=self.Output.message) (self.torrent_cache, self.file_cache, self.blocked_files, added, removed) = r for hash, data in removed.iteritems(): self.Output.message('dropped "{}"'.format(data['path'])) self.remove(hash) for hash, data in added.iteritems(): self.Output.message('added "{}"'.format(data['path'])) self.add(hash, data) def stats(self): self.rawserver.add_task(self.stats, self.stats_period) data = [] for hash in self.torrent_list: cache = self.torrent_cache[hash] if self.config['display_path']: name = cache['path'] else: name = cache['name'] size = cache['length'] d = self.downloads[hash] progress = '0.0%' peers = 0 seeds = 0 seedsmsg = "S" dist = 0.0 uprate = 0.0 dnrate = 0.0 upamt = 0 dnamt = 0 t = 0 if d.is_dead(): status = 'stopped' elif d.waiting: status = 'waiting for hash check' elif d.checking: status = d.status_msg progress = '{:.1%}'.format(d.status_done) else: stats = d.statsfunc() s = stats['stats'] if d.seed: status = 'seeding' progress = '100.0%' seeds = s.numOldSeeds seedsmsg = "s" dist = s.numCopies else: if s.numSeeds + s.numPeers: t = stats['time'] if t == 0: # unlikely t = 0.01 status = fmttime(t) else: t = -1 status = 'connecting to peers' progress = '{:.1%}'.format(stats['frac']) seeds = s.numSeeds dist = s.numCopies2 dnrate = stats['down'] peers = s.numPeers uprate = stats['up'] upamt = s.upTotal dnamt = s.downTotal if d.is_dead() or d.status_errtime + 300 > clock(): msg = d.status_err[-1] else: msg = '' data.append((name, status, progress, peers, seeds, seedsmsg, dist, uprate, dnrate, upamt, dnamt, size, t, msg)) stop = self.Output.display(data) if stop: self.doneflag.set() def remove(self, hash): self.torrent_list.remove(hash) self.downloads[hash].shutdown() del self.downloads[hash] def add(self, hash, data): c = self.counter self.counter += 1 x = '' for i in xrange(3): x = mapbase64[c & 0x3F] + x c >>= 6 peer_id = createPeerID(x) d = SingleDownload(self, hash, data['metainfo'], self.config, peer_id) self.torrent_list.append(hash) self.downloads[hash] = d d.start() def saveAs(self, hash, name, saveas, isdir): x = self.torrent_cache[hash] style = self.config['saveas_style'] if style == 1 or style == 3: if saveas: saveas = os.path.join(saveas, x['file'][:-1 - len(x['type'])]) else: saveas = x['path'][:-1 - len(x['type'])] if style == 3: if not os.path.isdir(saveas): try: os.mkdir(saveas) except: raise OSError("couldn't create directory for {} ({})" ''.format(x['path'], saveas)) if not isdir: saveas = os.path.join(saveas, name) else: if saveas: saveas = os.path.join(saveas, name) else: saveas = os.path.join(os.path.split(x['path'])[0], name) if isdir and not os.path.isdir(saveas): try: os.mkdir(saveas) except: raise OSError("couldn't create directory for {} ({})".format( x['path'], saveas)) return saveas def hashchecksched(self, hash=None): if hash: self.hashcheck_queue.append(hash) if not self.hashcheck_current: self._hashcheck_start() def _hashcheck_start(self): self.hashcheck_current = self.hashcheck_queue.pop(0) self.downloads[self.hashcheck_current].hashcheck_start( self.hashcheck_callback) def hashcheck_callback(self): self.downloads[self.hashcheck_current].hashcheck_callback() if self.hashcheck_queue: self._hashcheck_start() else: self.hashcheck_current = None def died(self, hash): if hash in self.torrent_cache: self.Output.message('DIED: "{}"'.format( self.torrent_cache[hash]['path'])) def was_stopped(self, hash): try: self.hashcheck_queue.remove(hash) except: pass if self.hashcheck_current == hash: self.hashcheck_current = None if self.hashcheck_queue: self._hashcheck_start() def failed(self, s): self.Output.message('FAILURE: ' + s) def exchandler(self, s): self.Output.exception(s)
except OSError, e: errorfunc("Couldn't allocate dir - " + str(e)) return finflag = Event() ann = [None] myid = 'M' + version.replace('.', '-') myid = myid + ('-' * (8 - len(myid))) + b2a_hex(sha(repr(time()) + ' ' + str(getpid())).digest()[-6:]) seed(myid) pieces = [info['pieces'][x:x+20] for x in xrange(0, len(info['pieces']), 20)] def failed(reason, errorfunc = errorfunc, doneflag = doneflag): doneflag.set() if reason is not None: errorfunc(reason) rawserver = RawServer(doneflag, config['timeout_check_interval'], config['timeout'], errorfunc = errorfunc, maxconnects = config['max_allow_in']) try: try: storage = Storage(files, open, path.exists, path.getsize) except IOError, e: errorfunc('trouble accessing files - ' + str(e)) return def finished(finfunc = finfunc, finflag = finflag, ann = ann, storage = storage, errorfunc = errorfunc): finflag.set() try: storage.set_readonly() except (IOError, OSError), e: errorfunc('trouble setting readonly at end - ' + str(e)) if ann[0] is not None: ann[0](1)
def download(params, filefunc, statusfunc, finfunc, errorfunc, doneflag, cols, pathFunc=None, presets={}, exchandler=None, failed=_failfunc, paramfunc=None): try: config = parse_params(params, presets) except ValueError as e: failed( 'error: {}\nrun with no args for parameter explanations'.format(e)) return if not config: errorfunc(get_usage()) return myid = createPeerID() random.seed(myid) rawserver = RawServer(doneflag, config['timeout_check_interval'], config['timeout'], ipv6_enable=config['ipv6_enabled'], failfunc=failed, errorfunc=exchandler) upnp_type = UPnP_test(config['upnp_nat_access']) try: listen_port = rawserver.find_and_bind( config['minport'], config['maxport'], config['bind'], ipv6_socket_style=config['ipv6_binds_v4'], upnp=upnp_type, randomizer=config['random_port']) except socket.error as e: failed("Couldn't listen - " + str(e)) return response = get_response(config['responsefile'], config['url'], failed) if not response: return infohash = hashlib.sha1(bencode(response['info'])).digest() d = BT1Download(statusfunc, finfunc, errorfunc, exchandler, doneflag, config, response, infohash, myid, rawserver, listen_port) if not d.saveAs(filefunc): return if pathFunc: pathFunc(d.getFilename()) hashcheck = d.initFiles(old_style=True) if not hashcheck: return if not hashcheck(): return if not d.startEngine(): return d.startRerequester() d.autoStats() statusfunc(activity='connecting to peers') if paramfunc: paramfunc({ # change_max_upload_rate(<int KiB/sec>) 'max_upload_rate': d.setUploadRate, # change_max_uploads(<int max uploads>) 'max_uploads': d.setConns, 'listen_port': listen_port, # int 'peer_id': myid, # string 'info_hash': infohash, # string # start_connection((<string ip>, <int port>), <peer id>) 'start_connection': d._startConnection, }) rawserver.listen_forever(d.getPortHandler()) d.shutdown()
def download(params, filefunc, statusfunc, finfunc, errorfunc, doneflag, cols, pathFunc=None, presets={}, exchandler=None, failed=_failfunc, paramfunc=None): try: config = parse_params(params, presets) except ValueError as e: failed('error: {}\nrun with no args for parameter explanations'.format( e)) return if not config: errorfunc(get_usage()) return myid = createPeerID() random.seed(myid) rawserver = RawServer(doneflag, config['timeout_check_interval'], config['timeout'], ipv6_enable=config['ipv6_enabled'], failfunc=failed, errorfunc=exchandler) upnp_type = UPnP_test(config['upnp_nat_access']) try: listen_port = rawserver.find_and_bind( config['minport'], config['maxport'], config['bind'], ipv6_socket_style=config['ipv6_binds_v4'], upnp=upnp_type, randomizer=config['random_port']) except socket.error as e: failed("Couldn't listen - " + str(e)) return response = get_response(config['responsefile'], config['url'], failed) if not response: return infohash = hashlib.sha1(bencode(response['info'])).digest() d = BT1Download(statusfunc, finfunc, errorfunc, exchandler, doneflag, config, response, infohash, myid, rawserver, listen_port) if not d.saveAs(filefunc): return if pathFunc: pathFunc(d.getFilename()) hashcheck = d.initFiles(old_style=True) if not hashcheck: return if not hashcheck(): return if not d.startEngine(): return d.startRerequester() d.autoStats() statusfunc(activity='connecting to peers') if paramfunc: paramfunc({ # change_max_upload_rate(<int KiB/sec>) 'max_upload_rate': d.setUploadRate, # change_max_uploads(<int max uploads>) 'max_uploads': d.setConns, 'listen_port': listen_port, # int 'peer_id': myid, # string 'info_hash': infohash, # string # start_connection((<string ip>, <int port>), <peer id>) 'start_connection': d._startConnection, }) rawserver.listen_forever(d.getPortHandler()) d.shutdown()
class ClientHandler: def __init__(self, co): self.co = co self.socket = {} self.rs = RawServer(Event(), 100, 1000) self.nh = NetworkHandler(self) # Functions called from lower level code def message_came_in(self, s, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'garbage data' if msg.has_key('error'): raise ServerError, msg['error'] socket = self.socket[s] srp = socket['srp'] if socket['state'] == 1: K, m = self.auth.client_key(msg['s'], msg['B'], msg['u'], srp['keys']) socket['key'], socket['m_out'] = K, m self._send_msg(s, {'m': socket['m_out'].digest()}) socket['state'] = 2 elif socket['state'] == 2: socket['m_in'] = SRP.host_authenticator(socket['key'], srp['keys'][0], socket['m_out'].digest()) if socket['m_in'].digest() != msg['auth']: raise ServerError, 'Bad host authentication' return self.nh.set_hmac(s, socket['m_in'], socket['m_out']) self.rs.doneflag.set() elif socket['state'] == 3: self.socket[s]['hash'] = msg['hash'] self.rs.doneflag.set() elif socket['state'] == 4: self.close(s) secret = crypt(msg['secret'], socket['key'])[0] self.auth.save_secret(secret) self.rs.doneflag.set() elif socket['state'] == 5: self.close(s) self.rs.doneflag.set() elif socket['state'] == 6: if len(msg['salt']) < 20: self._send_error(s, None, 'Bad salt length') self.close(s) raise NetworkError, 'Bad salt from server' salt = random_string(20) key = self.auth.session_key(salt, msg['salt']) socket['m_in'] = hmac.new(key, '', sha) key = self.auth.session_key(msg['salt'], salt) socket['m_out'] = hmac.new(key, '', sha) self._send_msg(s, {'auth': socket['m_in'].digest(), 'salt': salt}) socket['state'] = 7 elif socket['state'] == 7: if msg['auth'] != socket['m_out'].digest(): self._send_error(s, None, 'Bad auth') self.close(s) raise NetworkError, 'Bad server auth' self._req_mode(s, 1) self.nh.set_hmac(s, socket['m_in'], socket['m_out']) self.socket[s] = [{}, {}, {}, [], 1] self.rs.doneflag.set() else: self.close(s) def connection_flushed(self, s): queue = self.socket[s][Queue] socket = self.socket[s] socket[Flushed] = 1 while len(queue) and socket[Flushed] == 1: mid, msg = queue.pop(0) diff = read_diff(self.co, msg['handle'], msg['changenum'], None) socket[Flushed] = self._send_response(s, mid, {'diff': diff}) def connection_lost(self, s, msg): del self.socket[s] raise NetworkError, msg def request_came_in(self, s, mid, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'garbage request' self.request_handlers[msg['request']](self, s, mid, msg) def response_came_in(self, s, mid, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'bad data from server' rstate = self.socket[s][Request][mid] if msg.has_key('error'): # XXX: grody hack, diffs which don't belong in history if rstate['request'] == 'get diff': msg['diff'] = zlib.compress(bencode('')) else: raise ServerError, msg['error'] if self.response_handlers[rstate['request']](self, s, mid, msg, rstate): del self.socket[s][Request][mid] # request handlers def _request_get_change(self, s, mid, msg): resp = { 'changeset': self.co.lcrepo.get(msg['changenum'], txn=self.txn) } self._send_response(s, mid, resp) def _request_get_diff(self, s, mid, msg): if self.socket[s][Flushed] == 1: diff = read_diff(self.co, msg['handle'], msg['changenum'], self.txn) self.socket[s][Flushed] = self._send_response( s, mid, {'diff': diff}) else: self.socket[s][Queue].append((mid, msg)) request_handlers = { 'get change': _request_get_change, 'get diff': _request_get_diff } # response handlers def _response_get_head(self, s, mid, msg, rstate): return self._merge_change(s, mid, msg['head']) def _merge_change(self, s, mid, head): lstate = self.socket[s][Request][mid] lstate['head'] = head self.socket[s][UpdateInfo]['head'] = head if self.co.lcrepo.has_key(head): named, modified, added, deleted = \ handles_in_branch(self.co, lstate['heads'], [head], None) self._update_checks(s, mid, named, modified) self._update_handle_list(s, lstate, named, modified, added, deleted) self._update_finish(s, lstate) self.socket[s][UpdateInfo]['head'] = head self.rs.doneflag.set() return 1 rid = self._get_change(s, head) self.socket[s][Request][rid] = { 'request': 'get change', 'changenum': head, 'ref': mid } lstate['requests'][head] = 1 lstate['count'] = 1 return 0 def _response_get_change(self, s, mid, msg, rstate): lstate = self.socket[s][Request][rstate['ref']] if sha.new(msg['changeset']).digest() != rstate['changenum']: raise ServerError, 'bad changeset' # write it out, decode and eject from memory write_changeset(self.co, rstate['changenum'], msg['changeset'], lstate['txn']) changeset = bdecode(msg['changeset']) lstate['changes'][rstate['changenum']] = changeset del msg['changeset'] # get any precursors we don't have and haven't yet requested for change in changeset['precursors']: if self.co.lcrepo.has_key(change): continue if lstate['changes'].has_key(change): continue if lstate['requests'].has_key(change): continue rid = self._get_change(s, change) self.socket[s][Request][rid] = { 'request': 'get change', 'changenum': change, 'ref': rstate['ref'] } lstate['requests'][change] = 1 lstate['count'] += 1 # record all the diffs we'll need to request diffs = lstate['diffs'] for handle, hinfo in changeset['handles'].items(): if not hinfo.has_key('hash'): continue if not diffs.has_key(handle): diffs[handle] = {} lstate['counts'][handle] = 0 diffs[handle][rstate['changenum']] = 1 lstate['counts'][handle] += 1 # clean up state del lstate['requests'][rstate['changenum']] lstate['count'] -= 1 if lstate['count'] == 0: sync_history(self.co, lstate['head'], lstate['txn'], cache=lstate['changes']) named, modified, added, deleted = \ handles_in_branch(self.co, lstate['heads'], [lstate['head']], lstate['txn'], cache=lstate['changes']) del lstate['changes'] self._update_checks(s, rstate['ref'], named, modified) self._update_handle_list(s, lstate, named, modified, added, deleted) handle_list = lstate['handle list'] # get all the related file diffs for i in xrange(len(handle_list) - 1, -1, -1): handle = handle_list[i][1] if not diffs.has_key(handle): continue changes = diffs[handle] requested = 0 for change in changes.keys(): requested = 1 self._queue_diff(s, change, handle, rstate['ref']) lstate['count'] += requested self._get_diff(s, rstate['ref']) if lstate['count'] == 0: self._update_finish(s, lstate) del self.socket[s][Request][rstate['ref']] self.rs.doneflag.set() return 1 def _response_get_diff(self, s, mid, msg, rstate): lstate = self.socket[s][Request][rstate['ref']] # send out the next one lstate['req-outstanding'] -= 1 self._get_diff(s, rstate['ref']) diffs = lstate['diffs'] diffs[rstate['handle']][rstate['change']] = msg['diff'] lstate['counts'][rstate['handle']] -= 1 if lstate['counts'][rstate['handle']] == 0: lstate['count'] -= 1 # XXX: do better ordering WD = WriteDiff(self.co, rstate['handle'], lstate['txn']) for change, diff in diffs[rstate['handle']].items(): WD.write(diff, change) WD.close() if lstate['modified'].has_key(rstate['handle']): updates(self.co, self.socket[s][UpdateInfo], lstate, rstate['handle']) #del diffs[rstate['handle']] if lstate['count'] == 0: self._update_finish(s, lstate) self.socket[s][UpdateInfo]['head'] = lstate['head'] del self.socket[s][Request][rstate['ref']] self.rs.doneflag.set() return 1 def _response_return(self, s, mid, msg, rstate): self.rs.doneflag.set() return 1 def _response_list_repositories(self, s, mid, msg, rstate): self.socket[s][UpdateInfo] = msg['list'] self.rs.doneflag.set() return 1 response_handlers = { 'get head': _response_get_head, 'get change': _response_get_change, 'get diff': _response_get_diff, 'commit': _response_return, 'create repository': _response_return, 'destroy repository': _response_return, 'list repositories': _response_list_repositories } # Internal helper functions def _update_checks(self, s, mid, named, modified): lstate = self.socket[s][Request][mid] check_modified(self.co, modified) lstate['modified'] = {} for handle in modified: lstate['modified'][handle] = 1 self.socket[s][UpdateInfo]['modified'] = [] self.socket[s][UpdateInfo]['modified conflicts'] = [] def _update_handle_list(self, s, lstate, named, modified, added, deleted): uinfo = self.socket[s][UpdateInfo] func = lstate['update function'] handles, nconflicts = func(self.co, uinfo, named, modified, added, deleted, lstate['txn']) handle_list = lstate['handle list'] for handle, show in handles: letters = (' ', ' ') if uinfo['newfiles'].has_key(handle): letters = ('A', 'A') elif uinfo['deletes'].has_key(handle): letters = ('D', 'D') elif uinfo['names'].has_key(handle): letters = (' ', 'N') if nconflicts.has_key(handle): letters = (letters[0], 'C') handle_list.append( (handle_to_filename(self.co, handle, lstate['txn']), handle, show, letters)) handle_list.sort() handle_list.reverse() uinfo['name conflicts'] = nconflicts.keys() def _update_finish(self, s, lstate): updates(self.co, self.socket[s][UpdateInfo], lstate, None) def _req_mode(self, s, mode): self.nh.req_mode(s, mode) def _commit(self, s, repo, head): rid = self._send_request(s, { 'request': 'commit', 'repository': repo, 'changenum': head }) self.socket[s][Request][rid] = {'request': 'commit'} def _update(self, s, repo, func, txn): heads = bdecode(self.co.linforepo.get('heads')) lstate = { 'request': 'get head', 'txn': txn, 'changes': {}, 'requests': {}, 'diffs': {}, 'count': 0, 'counts': {}, 'handle list': [], 'heads': heads, 'reqq': [], 'req-outstanding': 0, 'update function': func } if repo.startswith('{') and repo.endswith('}'): head = binascii.unhexlify(repo[1:-1]) rid = self.nh.next_id(s) self.socket[s][Request][rid] = lstate if self._merge_change(s, rid, head): del self.socket[s][Request][rid] else: rid = self._send_request(s, { 'request': 'get head', 'repository': repo }) self.socket[s][Request][rid] = lstate return def _create_repo(self, s, repo): rid = self._send_request(s, { 'request': 'create repository', 'repository': repo }) self.socket[s][Request][rid] = { 'request': 'create repository', 'repository': repo } def _remove_repo(self, s, repo): rid = self._send_request(s, { 'request': 'destroy repository', 'repository': repo }) self.socket[s][Request][rid] = { 'request': 'destroy repository', 'repository': repo } def _list_repos(self, s): rid = self._send_request(s, {'request': 'list repositories'}) self.socket[s][Request][rid] = {'request': 'list repositories'} def _get_change(self, s, change): req = {'request': 'get change', 'changenum': change} return self._send_request(s, req) def _queue_diff(self, s, change, handle, mid): rstate = self.socket[s][Request][mid] rstate['reqq'].append((change, handle)) def _get_diff(self, s, mid): rstate = self.socket[s][Request][mid] while len(rstate['reqq']) and rstate['req-outstanding'] <= 40: change, handle = rstate['reqq'].pop(0) req = { 'request': 'get diff', 'changenum': change, 'handle': handle } state = { 'request': 'get diff', 'ref': mid, 'change': change, 'handle': handle } rid = self._send_request(s, req) self.socket[s][Request][rid] = state rstate['req-outstanding'] += 1 def _send_msg(self, s, data): self.nh.send_msg(s, bencode(data)) def _send_error(self, s, mid, msg): self._send_response(s, mid, {'error': msg}) def _send_request(self, s, data): return self.nh.send_request(s, bencode(data)) def _send_response(self, s, mid, data): return self.nh.send_response(s, mid, bencode(data)) # Functions to be called from higher level code def start_connection(self, dns): s = self.nh.start_connection(dns) self.socket[s] = {'state': 0, 'srp': {}} return s def close(self, s): self.nh.close(s) del self.socket[s] def get_hash(self, s): self._send_msg(s, {'op': 'get hash', 'user': self.co.user}) self.socket[s]['state'] = 3 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() secret_hash = self.socket[s]['hash'] del self.socket[s]['hash'] return secret_hash def srp_auth(self, s): socket = self.socket[s] assert socket['state'] == 0 or socket['state'] == 3 srp = socket['srp'] = {} srp['keys'] = SRP.client_begin() self._send_msg(s, { 'op': 'srp auth', 'user': self.co.user, 'A': srp['keys'][0] }) socket['state'] = 1 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def get_secret(self, s): self._send_msg(s, {'op': 'get secret'}) self.socket[s]['state'] = 4 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() return def set_password(self, s, password): socket = self.socket[s] salt, v = SRP.new_passwd(self.co.user, password) cypherv = crypt(long_to_string(v), socket['key'])[0] self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv}) socket['state'] = 5 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def secret_auth(self, s): socket = self.socket[s] self._send_msg(s, {'op': 'secret auth', 'user': self.co.user}) socket['state'] = 6 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def commit(self, s, repo, changenum, txn): self.txn = txn self._commit(s, repo, changenum) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def update(self, s, repo, func, txn): self._update(s, repo, func, txn) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() updateinfo = self.socket[s][UpdateInfo] del self.socket[s][UpdateInfo] return updateinfo def create_repo(self, s, repo): self._create_repo(s, repo) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def remove_repo(self, s, repo): self._remove_repo(s, repo) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def list_repos(self, s): self._list_repos(s) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() rlist = self.socket[s][UpdateInfo] del self.socket[s][UpdateInfo] return rlist