def __init__(self, multitorrent, rawserver, test_new_version=None, test_current_version=None): TorrentButler.__init__(self, multitorrent) self.runs = 0 self.rawserver = rawserver self.estate = set() self.old_updates = set() self.log_root = "core.AutoUpdateButler" self.logger = logging.getLogger(self.log_root) self.installable_version = None self.available_version = None self.current_version = Version.from_str(version) self.debug_mode = DEBUG self.delay = 60*60*24 if self.debug_mode: self.delay = 10 if test_new_version: test_new_version = Version.from_str(test_new_version) self.debug_mode = True self.debug('__init__() turning debug on') def _hack_get_available(url): self.debug('_hack_get_available() run#%d: returning %s' % (self.runs, str(test_new_version))) return test_new_version self._get_available = _hack_get_available if test_current_version: self.debug_mode = True self.current_version = Version.from_str(test_current_version) self.version_site = version_host # The version URL format is: # http:// VERSION_SITE / OS_NAME / (LEGACY /) BETA or STABLE # LEGACY means that the user is on a version of an OS that has # been deemed "legacy", and as such the latest client version # for their OS version may be different than the latest client # version for the OS in general. For example, if we are going # to roll a version that requires WinXP/2K or greater, or a # version that requires OSX 10.5 or greater, we may maintain # an older version for Win98 or OSX 10.4 in OS_NAME/legacy/. if os.name == 'nt': self.version_site += 'win32/' if os_version not in ('XP', '2000', '2003'): self.version_site += 'legacy/' elif osx: self.version_site += 'osx/' elif self.debug_mode: self.version_site += 'win32/' self.installer_dir = self._calc_installer_dir() # kick it off self.rawserver.add_task(0, self.check_version)
def _affect_rate(self, type, std, max_std, rate, set): rate = self._method_stddev(type, std, max_std, rate) rock_bottom = False if rate <= 4096: if debug: print "Rock bottom" rock_bottom = True rate = 4096 set(int(rate)) return rock_bottom
def started(self, torrent): """Only run the most recently added torrent""" if self.butles(torrent): removable = self.estate - set([torrent.infohash]) for i in removable: self.estate.discard(i) self.multitorrent.remove_torrent(i, del_files=True)
def parsedir(directory, parsed, files, blocked, errfunc, include_metainfo=True): """Recurses breadth-first starting from the passed 'directory' looking for .torrrent files. Stops recursing in any given branch at the first depth that .torrent files are encountered. The directory, parsed, files, and blocked arguments are passed from the previous iteration of parsedir. @param directory: root of the breadth-first search for .torrent files. @param parsed: dict mapping infohash to ConvertedMetainfo. @param files: dict mapping path -> [(modification time, size), infohash] @param blocked: dict used as set. keys are list of paths of files that were not parsed on a prior call to parsedir for some reason. Valid reasons are that the .torrent file is unparseable or that a torrent with a matching infohash is already in the parsed set. @param errfunc: error-reporting callback. @param include_metainfo: @return: The tuple (new parsed, new files, new blocked, added, removed) where 'new parsed', 'new files', and 'new blocked' are updated versions of 'parsed', 'files', and 'blocked' respectively. 'added' and 'removed' contain the changes made to the first three members of the tuple. 'added' and 'removed' are dicts mapping from infohash on to the same torrent-specific info dict that is in or was in parsed. """ if NOISY: errfunc('checking dir') dirs_to_check = [directory] new_files = {} # maps path -> [(modification time, size),infohash] new_blocked = set() while dirs_to_check: # first, recurse directories and gather torrents directory = dirs_to_check.pop() newtorrents = False try: dir_contents = os.listdir(directory) except (IOError, OSError), e: errfunc(_("Could not read directory ") + directory) continue for f in dir_contents: if f.endswith('.torrent'): newtorrents = True p = os.path.join(directory, f) try: new_files[p] = [(os.path.getmtime(p), os.path.getsize(p)), 0] except (IOError, OSError), e: errfunc(_("Could not stat ") + p + " : " + unicode(e.args[0]))
def __init__(self, rawserver): self.rawserver = rawserver old_connectionMade = rawserver.connectionMade def connectionMade(s): if rawserver.connections == 0: self._first_connection() old_connectionMade(s) assert not hasattr(rawserver, "internet_watcher"), \ "rawserver already has conncetion rate limiter installed" rawserver.connectionMade = connectionMade rawserver.internet_watcher = self self.subscribers = set() self.internet_watcher = task.LoopingCall(self._internet_watch) self.internet_watcher.start(5)
def _collect_nodes(self, local_ips): addrs = self.get_remote_endpoints() ips = set() for (ip, port) in addrs: if ip is not None and ip != "0.0.0.0" and ip not in local_ips: assert isinstance(ip, str) assert isinstance(port, int) ips.add(ip) self.rttmonitor.set_nodes_restart(ips) delay = 5 if len(ips) > 0: delay = 300 self.external_add_task(delay, self._collect_nodes, local_ips)
def _collect_nodes(self): addrs = self.get_remote_endpoints() print "_collect_nodes: addrs=", addrs local_ips = get_host_ips() ips = set() for (ip, port) in addrs: if ip is not None and ip != "0.0.0.0" and ip not in local_ips: ips.add(ip) self.rttmonitor.set_nodes_restart(ips) delay = 5 if len(ips) > 0: delay = 300 self.add_task(delay, self._collect_nodes)
def _collect_nodes(self): addrs = self.get_remote_endpoints() print "_collect_nodes: addrs=",addrs local_ips = get_host_ips() ips = set() for (ip, port) in addrs: if ip is not None and ip != "0.0.0.0" and ip not in local_ips: ips.add(ip) self.rttmonitor.set_nodes_restart(ips) delay = 5 if len(ips) > 0: delay = 300 self.add_task(delay, self._collect_nodes)
def __init__(self, config, storage, urlage, picker, numpieces, finished, total_downmeasure, downmeasure, measurefunc, kickfunc, banfunc): self.config = config self.storage = storage self.urlage = urlage self.picker = picker self.rerequester = None self.connection_manager = None self.chunksize = config['download_slice_size'] self.total_downmeasure = total_downmeasure self.downmeasure = downmeasure self.numpieces = numpieces self.finished = finished self.snub_time = config['snub_time'] self.measurefunc = measurefunc self.kickfunc = kickfunc self.banfunc = banfunc self.downloads = [] self.perip = {} self.bad_peers = {} self.discarded_bytes = 0 self.burst_avg = 0 self.piece_states = PieceSetBuckets() nothing = SparseSet(xrange(self.numpieces)) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, 0) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) self.last_update = 0 indexes = self.storage.inactive_requests.keys() indexes.sort() self.active_requests = SparseSet(indexes) self.all_requests = set()
def update_status(self): """Update torrent information based on the results of making a status request.""" df = self.multitorrent.get_torrents() yield df torrents = df.getResult() infohashes = set() au_torrents = {} for torrent in torrents: torrent = ThreadProxy(torrent, self.gui_wrap) infohashes.add(torrent.metainfo.infohash) if (not torrent.hidden and torrent.metainfo.infohash not in self.torrents): # create new torrent widget to = self.new_displayed_torrent(torrent) if torrent.is_auto_update: au_torrents[torrent.metainfo.infohash] = torrent for infohash, torrent in copy(self.torrents).iteritems(): # remove nonexistent torrents if infohash not in infohashes: self.torrents.pop(infohash) self.torrent_removed(infohash) total_completion = 0 total_bytes = 0 for infohash, torrent in copy(self.torrents).iteritems(): # update existing torrents df = self.multitorrent.torrent_status(infohash, torrent.wants_peers(), torrent.wants_files() ) yield df try: core_torrent, statistics = df.getResult() except UnknownInfohash: # looks like it's gone now if infohash in self.torrents: self.torrents.pop(infohash) self.torrent_removed(infohash) else: # the infohash might have been removed from torrents # while we were yielding above, so we need to check if infohash in self.torrents: core_torrent = ThreadProxy(core_torrent, self.gui_wrap) torrent.update(core_torrent, statistics) self.update_torrent(torrent) if statistics['fractionDone'] is not None: amount_done = statistics['fractionDone'] * torrent.metainfo.total_bytes total_completion += amount_done total_bytes += torrent.metainfo.total_bytes all_completed = False if total_bytes == 0: average_completion = 0 else: average_completion = total_completion / total_bytes if total_completion == total_bytes: all_completed = True df = self.multitorrent.auto_update_status() yield df available_version, installable_version, delay = df.getResult() if available_version is not None: if installable_version is None: self.notify_of_new_version(available_version) else: if self.installer_to_launch_at_exit is None: atexit.register(self.launch_installer_at_exit) if installable_version not in au_torrents: df = self.multitorrent.get_torrent(installable_version) yield df torrent = df.getResult() torrent = ThreadProxy(torrent, self.gui_wrap) else: torrent = au_torrents[installable_version] self.installer_to_launch_at_exit = torrent.working_path if bttime() > self.next_autoupdate_nag: self.prompt_for_quit_for_new_version(available_version) self.next_autoupdate_nag = bttime() + delay def get_global_stats(mt): stats = {} u, d = mt.get_total_rates() stats['total_uprate'] = Rate(u) stats['total_downrate'] = Rate(d) u, d = mt.get_total_totals() stats['total_uptotal'] = Size(u) stats['total_downtotal'] = Size(d) torrents = mt.get_visible_torrents() running = mt.get_visible_running() stats['num_torrents'] = len(torrents) stats['num_running_torrents'] = len(running) stats['num_connections'] = 0 for t in torrents: stats['num_connections'] += t.get_num_connections() try: stats['avg_connections'] = (stats['num_connections'] / stats['num_running_torrents']) except ZeroDivisionError: stats['avg_connections'] = 0 stats['avg_connections'] = "%.02f" % stats['avg_connections'] return stats df = self.multitorrent.call_with_obj(get_global_stats) yield df global_stats = df.getResult() yield average_completion, all_completed, global_stats
def _rechoke(self): # step 1: # get sorted in order of preference lists of peers # one for downloading torrents, and one for seeding torrents down_pref = [] seed_pref = [] for i, c in enumerate(self.connections): u = c.upload if c.download.have.numfalse == 0 or not u.interested: continue # I cry. if c.download.multidownload.storage.have.numfalse != 0: ## heuristic for downloading torrents if not c.download.is_snubbed(): ## simple download rate based down_pref.append((-c.download.get_rate(), i)) ## ratio based #dr = c.download.get_rate() #ur = max(1, u.get_rate()) #ratio = dr / ur #down_pref.append((-ratio, i)) else: ## heuristic for seeding torrents ## Uoti special ## if c._decrypt is not None: ## seed_pref.append((self.count, u.get_rate(), i)) ## elif (u.unchoke_time > self.count - self.magic_number or ## u.buffer and c.connection.is_flushed()): ## seed_pref.append((u.unchoke_time, u.get_rate(), i)) ## else: ## seed_pref.append((1, u.get_rate(), i)) ## sliding, first pass (see below) r = u.get_rate() if c._decrypt is not None: seed_pref.append((2, r, i)) else: seed_pref.append((1, r, i)) down_pref.sort() seed_pref.sort() #pprint(down_pref) #pprint(seed_pref) down_pref = [ self.connections[i] for junk, i in down_pref ] seed_pref = [ self.connections[i] for junk, junk, i in seed_pref ] max_uploads = self._max_uploads() ## sliding, second pass ## # up-side-down sum for an idea of capacity ## uprate_sum = sum(rates[-max_uploads:]) ## if max_uploads == 0: ## avg_uprate = 0 ## else: ## avg_uprate = uprate_sum / max_uploads ## #print 'avg_uprate', avg_uprate, 'of', max_uploads ## self.extra_slots = max(self.extra_slots - 1, 0) ## if avg_uprate > self.arbitrary_min: ## for r in rates: ## if r < (avg_uprate * 0.80): # magic 80% ## self.extra_slots += 2 ## break ## self.extra_slots = min(len(seed_pref), self.extra_slots) ## max_uploads += self.extra_slots ## #print 'plus', self.extra_slots # step 2: # split the peer lists by a ratio to fill the available upload slots d_uploads = max(1, int(round(max_uploads * 0.70))) s_uploads = max(1, int(round(max_uploads * 0.30))) #print 'original', 'ds', d_uploads, 'us', s_uploads extra = max(0, d_uploads - len(down_pref)) if extra > 0: s_uploads += extra d_uploads -= extra extra = max(0, s_uploads - len(seed_pref)) if extra > 0: s_uploads -= extra d_uploads = min(d_uploads + extra, len(down_pref)) #print 'ds', d_uploads, 'us', s_uploads down_pref = down_pref[:d_uploads] seed_pref = seed_pref[:s_uploads] preferred = set(down_pref) preferred.update(seed_pref) # step 3: # enforce unchoke states count = 0 to_choke = [] for i, c in enumerate(self.connections): u = c.upload if c in preferred: u.unchoke(self.count) count += 1 else: to_choke.append(c) # step 4: # enforce choke states and handle optimistics optimistics = max(self.config['min_uploads'], max_uploads - len(preferred)) #print 'optimistics', optimistics for c in to_choke: u = c.upload if c.download.have.numfalse == 0: # keep seeds choked, out of superstition u.unchoke(self.count) elif count >= optimistics: u.choke() else: # this one's optimistic u.unchoke(self.count) if u.interested: count += 1
def __init__(self, multitorrent): TorrentButler.__init__(self, multitorrent) self.started_torrents = set()
0x0441: "sw", # Swahili 0x0430: "sx", # Sutu 0x0449: "ta", # Tamil 0x041E: "th", # Thai 0x0432: "tn", # Setsuana 0x041F: "tr", # Turkish 0x0431: "ts", # Tsonga 0X0444: "tt", # Tatar 0x0422: "uk", # Ukrainian 0x0420: "ur", # Urdu 0x0443: "uz_UZ", # Uzbek - Latin 0x042A: "vi", # Vietnamese 0x0434: "xh", # Xhosa 0x043D: "yi", # Yiddish 0x0804: "zh_CN", # Chinese - China 0x0C04: "zh_HK", # Chinese - Hong Kong S.A.R. 0x1404: "zh_MO", # Chinese - Macau S.A.R 0x1004: "zh_SG", # Chinese - Singapore 0x0404: "zh_TW", # Chinese - Taiwan 0x0435: "zu", # Zulu } if __name__ == '__main__': from BitTorrent.obsoletepythonsupport import set internal = set([x.lower() for x in languages]) windows = set(locale_sucks.values()) if not windows.issuperset(internal): diff = list(internal.difference(windows)) diff.sort() print diff