def test_get_uniformed_tracker_url(self): result = get_uniformed_tracker_url("http://tracker.openbittorrent.com:80/announce") self.assertEqual(result, "http://tracker.openbittorrent.com/announce") result = get_uniformed_tracker_url("udp://tracker.openbittorrent.com:80/announce/") self.assertEqual(result, "udp://tracker.openbittorrent.com:80") result = get_uniformed_tracker_url("tcp://tracker.openbittorrent.com:80/announce/") self.assertFalse(result) result = get_uniformed_tracker_url("http://google.nl\xa0") self.assertFalse(result) result = get_uniformed_tracker_url("udp://tracker.openbittorrent.com:80") self.assertEqual(result, "udp://tracker.openbittorrent.com:80") result = get_uniformed_tracker_url("udp://tracker.openbittorrent.com") self.assertFalse(result) result = get_uniformed_tracker_url("http://tracker.openbittorrent.com/test") self.assertEqual(result, "http://tracker.openbittorrent.com/test") result = get_uniformed_tracker_url("http://tracker.openbittorrent.com:abc/test") self.assertFalse(result) result = get_uniformed_tracker_url("http://tracker.openbittorrent.com:81/test") self.assertEqual(result, "http://tracker.openbittorrent.com:81/test")
def _encode_torrent(self, message): files = message.payload.files trackers = list(message.payload.trackers) name = message.payload.name # Filter out invalid trackers for tracker in trackers: if not get_uniformed_tracker_url(tracker) or len(tracker) > 200: trackers.remove(tracker) # files is a tuple of tuples (actually a list in tuple form) max_len = self._community.dispersy_sync_bloom_filter_bits / 8 base_len = 20 + 8 + len(name) # infohash, timestamp, name tracker_len = sum([len(tracker) for tracker in trackers]) file_len = sum([len(f[0]) + 8 for f in files]) # file name, length # Check if the message fits in the bloomfilter if (base_len + tracker_len + file_len > max_len) and (len(trackers) > 10): # only use first 10 trackers, .torrents in the wild have been seen to have 1000+ trackers... trackers = trackers[:10] tracker_len = sum([len(tracker) for tracker in trackers]) if base_len + tracker_len + file_len > max_len: # reduce files by the amount we are currently to big reduce_by = max_len / (base_len + tracker_len + file_len * 1.0) nr_files_to_include = int(len(files) * reduce_by) files = sample(files, nr_files_to_include) normal_msg = (pack('!20sQ', message.payload.infohash, message.payload.timestamp), message.payload.name, tuple(files), tuple(trackers)) return zlib.compress(encode(normal_msg)),
def update_tracker_info(self, tracker_url, is_successful): """ Updates a tracker information. :param tracker_url: The given tracker_url. :param is_successful: If the check was successful. """ if tracker_url == u"DHT": return sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) tracker = self.tracker_store.get(lambda g: g.url == sanitized_tracker_url) if not tracker: self._logger.error("Trying to update the tracker info of an unknown tracker URL") return current_time = int(time.time()) failures = 0 if is_successful else tracker.failures + 1 is_alive = tracker.alive < MAX_TRACKER_FAILURES # update the dict tracker.last_check = current_time tracker.failures = failures tracker.alive = is_alive
def add_tracker(self, tracker_url): """ Adds a new tracker into the tracker info dict and the database. :param tracker_url: The new tracker URL to be added. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) if sanitized_tracker_url is None: self._logger.warn(u"skip invalid tracker: %s", repr(tracker_url)) return if sanitized_tracker_url in self._tracker_dict: self._logger.debug(u"skip existing tracker: %s", repr(tracker_url)) return # add the tracker into dict and database tracker_info = {u'last_check': 0, u'failures': 0, u'is_alive': True} # insert into database sql_stmt = u"""INSERT INTO TrackerInfo(tracker, last_check, failures, is_alive) VALUES(?,?,?,?); SELECT tracker_id FROM TrackerInfo WHERE tracker = ?; """ value_tuple = (sanitized_tracker_url, tracker_info[u'last_check'], tracker_info[u'failures'], tracker_info[u'is_alive'], sanitized_tracker_url) tracker_id, = self._session.sqlite_db.execute(sql_stmt, value_tuple).next() # update dict tracker_info[u'id'] = tracker_id self._tracker_dict[sanitized_tracker_url] = tracker_info self._tracker_id_to_url_dict[tracker_id] = sanitized_tracker_url
def update_tracker_info(self, tracker_url, is_successful): """ Updates a tracker information. :param tracker_url: The given tracker_url. :param is_successful: If the check was successful. """ if tracker_url == u"DHT": return sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) tracker = self.tracker_store.get( lambda g: g.url == sanitized_tracker_url) if not tracker: self._logger.error( "Trying to update the tracker info of an unknown tracker URL") return current_time = int(time.time()) failures = 0 if is_successful else tracker.failures + 1 is_alive = tracker.alive < MAX_TRACKER_FAILURES # update the dict tracker.last_check = current_time tracker.failures = failures tracker.alive = is_alive
def add_tracker(self, tracker_url): """ Adds a new tracker into the tracker info dict and the database. :param tracker_url: The new tracker URL to be added. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) if sanitized_tracker_url is None: self._logger.warn(u"skip invalid tracker: %s", repr(tracker_url)) return sql_stmt = u"SELECT COUNT() FROM TrackerInfo WHERE tracker = ?" num = self._session.sqlite_db.execute( sql_stmt, (sanitized_tracker_url, )).next()[0] if num > 0: self._logger.debug(u"skip existing tracker: %s", repr(tracker_url)) return # add the tracker into dict and database tracker_info = {u'last_check': 0, u'failures': 0, u'is_alive': True} # insert into database sql_stmt = u"""INSERT INTO TrackerInfo(tracker, last_check, failures, is_alive) VALUES(?,?,?,?); SELECT tracker_id FROM TrackerInfo WHERE tracker = ?; """ value_tuple = (sanitized_tracker_url, tracker_info[u'last_check'], tracker_info[u'failures'], tracker_info[u'is_alive'], sanitized_tracker_url) self._session.sqlite_db.execute(sql_stmt, value_tuple).next()
def get_tracker_info(self, tracker_url): """ Gets the tracker information with the given tracker URL. :param tracker_url: The given tracker URL. :return: The tracker info dict if exists, None otherwise. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) return self._tracker_dict.get(sanitized_tracker_url)
def __init__(self, *args, **kwargs): # Sanitize and canonicalize the tracker URL sanitized = get_uniformed_tracker_url(kwargs['url']) if sanitized: kwargs['url'] = sanitized else: raise MalformedTrackerURLException("Could not canonicalize tracker URL (%s)" % kwargs['url']) super(TrackerState, self).__init__(*args, **kwargs)
def __init__(self, *args, **kwargs): # Sanitize and canonicalize the tracker URL sanitized = get_uniformed_tracker_url(kwargs['url']) if sanitized: kwargs['url'] = sanitized else: raise MalformedTrackerURLException( "Could not canonicalize tracker URL (%s)" % kwargs['url']) super(TrackerState, self).__init__(*args, **kwargs)
def remove_tracker(self, tracker_url): """ Remove a given tracker from the database. URL is sanitized first and removed from the database. If the URL is ill formed then try removing the non- sanitized version. :param tracker_url: The URL of the tracker to be deleted. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) sql_stmt = u"DELETE FROM TrackerInfo WHERE tracker = ?;" if sanitized_tracker_url: self._session.sqlite_db.execute(sql_stmt, (sanitized_tracker_url,)) else: self._session.sqlite_db.execute(sql_stmt, (tracker_url,))
def load_blacklist(self): """ Load the tracker blacklist from tracker_blacklist.txt in the session state directory. Entries are newline separated and are supposed to be sanitized. """ blacklist_file = os.path.abspath(os.path.join(self._session.config.get_state_dir(), "tracker_blacklist.txt")) if os.path.exists(blacklist_file): with open(blacklist_file, 'r') as blacklist_file_handle: # Note that get_uniformed_tracker_url will strip the newline at the end of .readlines() self.blacklist.extend([get_uniformed_tracker_url(url) for url in blacklist_file_handle.readlines()]) else: self._logger.info("No tracker blacklist file found at %s.", blacklist_file)
def remove_tracker(self, tracker_url): """ Remove a given tracker from the database. URL is sanitized first and removed from the database. If the URL is ill formed then try removing the non- sanitized version. :param tracker_url: The URL of the tracker to be deleted. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) with db_session: options = self.tracker_store.select(lambda g: g.url in [tracker_url, sanitized_tracker_url]) for option in options[:]: option.delete()
def get_tracker_info(self, tracker_url): """ Gets the tracker information with the given tracker URL. :param tracker_url: The given tracker URL. :return: The tracker info dict if exists, None otherwise. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) if tracker_url != u"DHT" else tracker_url try: sql_stmt = u"SELECT tracker_id, tracker, last_check, failures, is_alive FROM TrackerInfo WHERE tracker = ?" result = self._session.sqlite_db.execute(sql_stmt, (sanitized_tracker_url,)).next() except StopIteration: return None return {u'id': result[0], u'last_check': result[2], u'failures': result[3], u'is_alive': bool(result[4])}
def remove_tracker(self, tracker_url): """ Remove a given tracker from the database. URL is sanitized first and removed from the database. If the URL is ill formed then try removing the non- sanitized version. :param tracker_url: The URL of the tracker to be deleted. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) sql_stmt = u"DELETE FROM TrackerInfo WHERE tracker = ?;" if sanitized_tracker_url: self._session.sqlite_db.execute(sql_stmt, (sanitized_tracker_url, )) else: self._session.sqlite_db.execute(sql_stmt, (tracker_url, ))
def remove_tracker(self, tracker_url): """ Remove a given tracker from the database. URL is sanitized first and removed from the database. If the URL is ill formed then try removing the non- sanitized version. :param tracker_url: The URL of the tracker to be deleted. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) with db_session: options = self.tracker_store.select( lambda g: g.url in [tracker_url, sanitized_tracker_url]) for option in options[:]: option.delete()
def get_tracker_info(self, tracker_url): """ Gets the tracker information with the given tracker URL. :param tracker_url: The given tracker URL. :return: The tracker info dict if exists, None otherwise. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) if tracker_url != u"DHT" else tracker_url with db_session: tracker = list(self.tracker_store.select(lambda g: g.url == sanitized_tracker_url)) if tracker: return { u'id': tracker[0].url, u'last_check': tracker[0].last_check, u'failures': tracker[0].failures, u'is_alive': tracker[0].alive } return None
def get_old_trackers(self): connection = sqlite3.connect(self.tribler_db) cursor = connection.cursor() trackers = {} for tracker_id, tracker, last_check, failures, is_alive in cursor.execute(self.select_trackers_sql): try: tracker_url_sanitized = get_uniformed_tracker_url(tracker) if not tracker_url_sanitized: continue except: # Skip malformed trackers continue trackers[tracker_url_sanitized] = ({ "last_check": last_check, "failures": failures, "alive": is_alive}) connection.close() return trackers
def load_blacklist(self): """ Load the tracker blacklist from tracker_blacklist.txt in the session state directory. Entries are newline separated and are supposed to be sanitized. """ blacklist_file = os.path.abspath( os.path.join(self._session.config.get_state_dir(), "tracker_blacklist.txt")) if os.path.exists(blacklist_file): with open(blacklist_file, 'r') as blacklist_file_handle: # Note that get_uniformed_tracker_url will strip the newline at the end of .readlines() self.blacklist.extend([ get_uniformed_tracker_url(url) for url in blacklist_file_handle.readlines() ]) else: self._logger.info("No tracker blacklist file found at %s.", blacklist_file)
def add_torrent_to_channel(self, tdef, extra_info=None): """ Add a torrent to your channel. :param tdef: The torrent definition file of the torrent to add :param extra_info: Optional extra info to add to the torrent """ if extra_info: tags = extra_info.get('description', '') else: # We only want to determine the type of the data. XXX filtering is done by the receiving side tags = default_category_filter.calculateCategory(tdef.metainfo, tdef.get_name_as_unicode()) new_entry_dict = { "infohash": tdef.get_infohash(), "title": tdef.get_name_as_unicode()[:300], # TODO: do proper size checking based on bytes "tags": tags[:200], # TODO: do proper size checking based on bytes "size": tdef.get_length(), "torrent_date": datetime.fromtimestamp(tdef.get_creation_date()), "tracker_info": get_uniformed_tracker_url(tdef.get_tracker() or '') or '', "status": NEW} # See if the torrent is already in the channel old_torrent = self.get_torrent(tdef.get_infohash()) if old_torrent: # If it is there, check if we were going to delete it if old_torrent.status == TODELETE: if old_torrent.metadata_conflicting(new_entry_dict): # Metadata from torrent we're trying to add is conflicting with the # deleted old torrent's metadata. We will replace the old metadata. new_timestamp = self._clock.tick() old_torrent.set(timestamp=new_timestamp, **new_entry_dict) old_torrent.sign() else: # No conflict. This means the user is trying to replace the deleted torrent # with the same one. Just recover the old one. old_torrent.status = COMMITTED torrent_metadata = old_torrent else: raise DuplicateTorrentFileError() else: torrent_metadata = db.TorrentMetadata.from_dict(new_entry_dict) return torrent_metadata
def get_tracker_info(self, tracker_url): """ Gets the tracker information with the given tracker URL. :param tracker_url: The given tracker URL. :return: The tracker info dict if exists, None otherwise. """ sanitized_tracker_url = get_uniformed_tracker_url( tracker_url) if tracker_url != u"DHT" else tracker_url try: sql_stmt = u"SELECT tracker_id, tracker, last_check, failures, is_alive FROM TrackerInfo WHERE tracker = ?" result = self._session.sqlite_db.execute( sql_stmt, (sanitized_tracker_url, )).next() except StopIteration: return None return { u'id': result[0], u'last_check': result[2], u'failures': result[3], u'is_alive': bool(result[4]) }
def get_tracker_info(self, tracker_url): """ Gets the tracker information with the given tracker URL. :param tracker_url: The given tracker URL. :return: The tracker info dict if exists, None otherwise. """ sanitized_tracker_url = get_uniformed_tracker_url( tracker_url) if tracker_url != u"DHT" else tracker_url with db_session: tracker = list( self.tracker_store.select( lambda g: g.url == sanitized_tracker_url)) if tracker: return { u'id': tracker[0].url, u'last_check': tracker[0].last_check, u'failures': tracker[0].failures, u'is_alive': tracker[0].alive } return None
def get_old_trackers(self): connection = sqlite3.connect(self.tribler_db) cursor = connection.cursor() trackers = {} for tracker_id, tracker, last_check, failures, is_alive in cursor.execute( self.select_trackers_sql): try: tracker_url_sanitized = get_uniformed_tracker_url(tracker) if not tracker_url_sanitized: continue except: # Skip malformed trackers continue trackers[tracker_url_sanitized] = ({ "last_check": last_check, "failures": failures, "alive": is_alive }) connection.close() return trackers
def add_tracker(self, tracker_url): """ Adds a new tracker into the tracker info dict and the database. :param tracker_url: The new tracker URL to be added. """ sanitized_tracker_url = get_uniformed_tracker_url(tracker_url) if sanitized_tracker_url is None: self._logger.warn(u"skip invalid tracker: %s", repr(tracker_url)) return with db_session: num = count(g for g in self.tracker_store if g.url == sanitized_tracker_url) if num > 0: self._logger.debug(u"skip existing tracker: %s", repr(tracker_url)) return # insert into database self.tracker_store(url=sanitized_tracker_url, last_check=0, failures=0, alive=True, torrents={})
def test_uniform_empty(self): result = get_uniformed_tracker_url(u'') self.assertIsNone(result)
def test_skip_truncated_url(self): result = get_uniformed_tracker_url(u'http://tracker.1337x.org:80/anno...') self.assertIsNone(result)
def add_tracker(self, tracker_url): sanitized_url = get_uniformed_tracker_url(tracker_url) if sanitized_url: tracker = db.TrackerState.get(url=sanitized_url) or db.TrackerState(url=sanitized_url) self.health.trackers.add(tracker)
def test_uniform_http_default_port_given(self): result = get_uniformed_tracker_url("http://torrent.ubuntu.com:80/announce") self.assertEqual(result, u'http://torrent.ubuntu.com/announce')
def test_uniform_http_no_path(self): result = get_uniformed_tracker_url("http://tracker.openbittorrent.com") self.assertIsNone(result)
def test_uniform_scheme_correct_http_training_slash(self): result = get_uniformed_tracker_url( "http://torrent.ubuntu.com:6969/announce/") self.assertEqual(result, u'http://torrent.ubuntu.com:6969/announce')
def add_tracker(self, tracker_url): sanitized_url = get_uniformed_tracker_url(tracker_url) if sanitized_url: tracker = db.TrackerState.get( url=sanitized_url) or db.TrackerState(url=sanitized_url) self.health.trackers.add(tracker)
def test_uniform_http_default_port_given(self): result = get_uniformed_tracker_url( "http://torrent.ubuntu.com:80/announce") self.assertEqual(result, u'http://torrent.ubuntu.com/announce')
def test_uniform_udp_remove_path(self): result = get_uniformed_tracker_url( "udp://tracker.openbittorrent.com:6969/announce") self.assertEqual(result, u'udp://tracker.openbittorrent.com:6969')
def test_uniform_udp_no_port(self): result = get_uniformed_tracker_url("udp://tracker.openbittorrent.com") self.assertIsNone(result)
def test_uniform_udp_remove_path(self): result = get_uniformed_tracker_url("udp://tracker.openbittorrent.com:6969/announce") self.assertEqual(result, u'udp://tracker.openbittorrent.com:6969')
def test_skip_wrong_url_scheme(self): result = get_uniformed_tracker_url(u'wss://tracker.1337x.org:80/announce') self.assertIsNone(result)
def test_skip_value_error(self): result = get_uniformed_tracker_url("ftp://tracker.1337\xffx.org:80/announce") self.assertIsNone(result)
def test_uniform_trailing_hex(self): result = get_uniformed_tracker_url("udp://tracker.1337x.org:80\xff") self.assertIsNone(result)
def test_uniform_bad_urlenc(self): result = get_uniformed_tracker_url(u'http://btjunkie.org/?do=upload') self.assertIsNone(result)
def test_uniform_scheme_correct_udp(self): result = get_uniformed_tracker_url( "udp://tracker.openbittorrent.com:80") self.assertEqual(result, u'udp://tracker.openbittorrent.com:80')
def test_skip_wrong_url_scheme(self): result = get_uniformed_tracker_url( u'wss://tracker.1337x.org:80/announce') self.assertIsNone(result)
def test_uniform_scheme_unknown(self): result = get_uniformed_tracker_url( "unknown://tracker.openbittorrent.com/announce") self.assertIsNone(result)
def test_uniform_http_no_path_nbsp(self): result = get_uniformed_tracker_url("http://google.nl\xa0") self.assertFalse(result)
def test_uniform_trailing_zero_hex(self): result = get_uniformed_tracker_url("udp://tracker.1337x.org:80\x00") self.assertEqual(result, u'udp://tracker.1337x.org:80')
def test_uniform_scheme_unknown(self): result = get_uniformed_tracker_url("unknown://tracker.openbittorrent.com/announce") self.assertIsNone(result)
def test_uniform_scheme_correct_http_training_slash(self): result = get_uniformed_tracker_url("http://torrent.ubuntu.com:6969/announce/") self.assertEqual(result, u'http://torrent.ubuntu.com:6969/announce')
def test_skip_truncated_url(self): result = get_uniformed_tracker_url( u'http://tracker.1337x.org:80/anno...') self.assertIsNone(result)
def test_uniform_scheme_correct_udp(self): result = get_uniformed_tracker_url("udp://tracker.openbittorrent.com:80") self.assertEqual(result, u'udp://tracker.openbittorrent.com:80')
def test_skip_value_error(self): result = get_uniformed_tracker_url( "ftp://tracker.1337\xffx.org:80/announce") self.assertIsNone(result)