def test_stripper_data(self): stripper = util.HTMLStripper() testdir = resources.path(os.path.join("testdata", "stripperdata")) tests = [m for m in os.listdir(testdir) if m.endswith(".in")] for mem in tests: mem = os.path.join(testdir, mem) if not os.path.isfile(mem): continue f = open(mem, "r") input_ = f.read() f.close() input_ = input_.decode("utf-8") output = stripper.strip(input_) expected = os.path.splitext(mem)[0] + ".expected" if not os.path.isfile(expected): self.assertEquals(0, 1, "%s not found." % expected) else: f = open(expected, "r") data = f.read().strip() f.close() self.assertEquals( repr(output), data, "output: %s" % repr(output))
def test_simple(self): stripper = util.HTMLStripper() for mem in [("<html", ("<html", [])), ("<html><html>", ("", [])), ("</html></html>", ("", [])), ("<p>foo</p>", ("foo", [])), ("<p>foo</p><br/>", ("foo", [])) ]: self.assertEquals(stripper.strip(mem[0]), mem[1])
def set_up(self): self.html_stripper = util.HTMLStripper() self.id_counter = itertools.count() self.setup_text() self.item_list = itemlist.ItemList() self.item_list.set_sort(itemlist.DateSort(True)) self.item_list.add_items(self.generate_items(self.initial_items)) self.make_item_view() scroller = widgetset.Scroller(False, True) scroller.add(self.item_view) self.vbox.pack_start(scroller, expand=True)
def test_garbage(self): stripper = util.HTMLStripper() for mem in [(1, ("", [])), (None, ("", [])), ({}, ("", [])) ]: self.assertEquals(stripper.strip(mem[0]), mem[1]) for mem in [("<html>", ("", [])), ("<html></html>", ("", []))]: self.assertEquals(stripper.strip(mem[0]), mem[1])
class ItemInfoBase(object): """ItemInfo represents a row in one of the item lists. This work similarly to the miro.item.Item class, except it's read-only. Subclases of this handle items from the main database, device database, and sharing database """ __metaclass__ = ItemInfoMeta #: ItemSelectInfo object that describes what to select to create an #: ItemInfoMeta select_info = None html_stripper = util.HTMLStripper() # default values for columns from the item table. For DeviceItemInfo and # SharingItemInfo, we will use these for columns that don't exist in their # item table. date_added = None watched_time = None last_watched = None parent_id = None rating = None album_tracks = None new = False keep = True was_downloaded = False expired = False eligible_for_autodownload = False is_file_item = True is_container_item = False icon_cache_path_unicode = None subtitle_encoding = None release_date = None parent_title = None feed_url = None feed_thumbnail_path_unicode = None license = None rss_id = None entry_title = None entry_description = None torrent_title = None permalink = None payment_link = None comments_link = None thumbnail_url = None url = None size = None enclosure_size = None enclosure_type = None mime_type = None enclosure_format = None auto_sync = None screenshot_path_unicode = None cover_art_path_unicode = None resume_time = 0 play_count = 0 skip_count = 0 net_lookup_enabled = False has_drm = False album = None kind = None duration_ms = None metadata_description = None show = None file_type = None artist = None episode_id = None track = None year = None genre = None episode_number = None season_number = None album_artist = None # default values for columns in the remote_downloader table downloader_size = None downloader_type = None seeders = None upload_size = None downloader_id = None _rate = None connections = None downloaded_time = None downloaded_size = None pending_reason = None retry_time = None retry_count = None short_reason_failed = None reason_failed = None leechers = None _eta = None pending_manual_download = None downloader_state = None _upload_rate = None downloader_activity = None downloader_content_type = None # default values for columns in the feed table feed_id = None feed_get_everything = None feed_auto_downloadable = False feed_expire_timedelta = None feed_expire = u'never' def __init__(self, row_data): """Create an ItemInfo object. :param row_data: data from sqlite. There should be a value for each SelectColumn that column_info() returns. """ self.row_data = row_data def __hash__(self): return hash(self.row_data) def __eq__(self, other): return self.row_data == other.row_data # NOTE: The previous ItemInfo API was all attributes, so we use properties # to try to match that. @property def filename(self): return _unicode_to_filename(self.filename_unicode) @property def downloaded(self): return self.has_filename @property def has_filename(self): return self.filename_unicode is not None @property def icon_cache_path(self): return _unicode_to_filename(self.icon_cache_path_unicode) @property def cover_art_path(self): return _unicode_to_filename(self.cover_art_path_unicode) @property def screenshot_path(self): return _unicode_to_filename(self.screenshot_path_unicode) @property def feed_thumbnail_path(self): return _unicode_to_filename(self.feed_thumbnail_path_unicode) @property def is_playable(self): return self.has_filename and self.file_type != u'other' @property def is_torrent(self): return self.downloader_type == u'BitTorrent' @property def is_torrent_folder(self): return self.is_torrent and self.is_container_item def looks_like_torrent(self): return self.is_torrent or filetypes.is_torrent_filename(self.url) @property def description(self): if self.metadata_description: return self.metadata_description elif self.entry_description: return self.entry_description else: return None @property def description_stripped(self): if not hasattr(self, '_description_stripped'): self._description_stripped = ItemInfo.html_stripper.strip( self.description) return self._description_stripped @property def thumbnail(self): if (self.cover_art_path_unicode is not None and fileutil.exists(self.cover_art_path)): return self.cover_art_path if (self.icon_cache_path_unicode is not None and fileutil.exists(self.icon_cache_path)): return self.icon_cache_path if (self.screenshot_path_unicode is not None and fileutil.exists(self.screenshot_path)): return self.screenshot_path if self.is_container_item: return resources.path("images/thumb-default-folder.png") if self.feed_thumbnail_path is not None: return self.feed_thumbnail_path # default if self.file_type == u'audio': return resources.path("images/thumb-default-audio.png") else: return resources.path("images/thumb-default-video.png") @property def is_external(self): """Is this an externally downloaded item.""" return False @property def remote(self): return self.source_type == u'sharing' @property def device(self): return self.source_type == u'device' @property def has_shareable_url(self): """Does this item have a URL that the user can share with others? This returns True when the item has a non-file URL. """ return self.url is not None and not self.url.startswith(u"file:") @property def file_format(self): """Returns string with the format of the video. """ if self.looks_like_torrent(): return u'.torrent' if self.enclosure_format is not None: return self.enclosure_format return filetypes.calc_file_format(self.filename, self.downloader_content_type) @property def video_watched(self): return self.watched_time is not None @property def expiration_date(self): """When will this item expire? :returns: a datetime.datetime object or None if it doesn't expire. """ if (self.watched_time is None or self.keep or not self.has_filename or self.is_file_item): return None if self.feed_expire == u'never': return None elif self.feed_expire == u"feed": if self.feed_expire_timedelta is None: logging.warn("feed_expire is 'feed', but " "feed_expire_timedelta is None") return None expire_time = self.feed_expire_time_parsed if expire_time is None: logging.warn("feed_expire is 'feed', but " "feed_expire_time_parsed failed") return None elif self.feed_expire == u"system": days = app.config.get(prefs.EXPIRE_AFTER_X_DAYS) if days <= 0: return None expire_time = datetime.timedelta(days=days) else: raise AssertionError("Unknown expire value: %s" % self.feed_expire) return self.watched_time + expire_time @property def feed_expire_time_parsed(self): if self.feed_expire_timedelta is None: return None try: expire_time_split = self.feed_expire_timedelta.split(":") return datetime.timedelta(*(int(c) for c in expire_time_split)) except StandardError: logging.warn("Error parsing feed_expire_timedelta", exc_info=True) return None @property def expiration_date_text(self): return displaytext.expiration_date(self.expiration_date) @property def can_be_saved(self): return self.has_filename and not self.keep @property def is_download(self): return (self.downloader_state in ('downloading', 'paused', 'offline') or self.pending_manual_download) @property def is_paused(self): return self.downloader_state == 'paused' @property def is_seeding(self): return self.downloader_state == 'uploading' @property def startup_activity(self): if self.pending_manual_download: return self.pending_reason elif self.downloader_activity: return self.downloader_activity elif self.is_retrying: return self._startup_activity_retry else: return _("starting up...") @property def is_retrying(self): return self.retry_count is not None and self.retry_time is not None @property def _startup_activity_retry(self): if self.retry_time > datetime.datetime.now(): retry_delta = self.retry_time - datetime.datetime.now() time_str = displaytext.time_string(retry_delta.seconds) return _('no connection - retrying in %(time)s', {"time": time_str}) else: return _('no connection - retrying soon') @property def download_progress(self): """Calculate how for a download has progressed. :returns: [0.0, 1.0] depending on how much has been downloaded, or None if we don't have the info to make this calculation """ if self.downloaded_size in (0, None): # Download hasn't started yet. Give the downloader a little more # time before deciding that the eta is unknown. return 0.0 if self.downloaded_size is None or self.downloader_size is None: # unknown total size, return None return None return float(self.downloaded_size) / self.downloader_size @property def eta(self): if self.is_paused: return None else: return self._eta @property def rate(self): if self.is_paused: return None else: return self._rate @property def upload_rate(self): if self.is_paused: return None else: return self._upload_rate @property def download_rate_text(self): return displaytext.download_rate(self.rate) @property def upload_rate_text(self): return displaytext.download_rate(self.upload_rate) @property def upload_ratio(self): if self.downloaded_size: return float(self.upload_size) / self.downloaded_size else: return 0.0 @property def upload_ratio_text(self): return "%0.2f" % self.upload_ratio @property def eta_text(self): return displaytext.time_string_0_blank(self.eta) @property def downloaded_size_text(self): return displaytext.size_string(self.downloaded_size) @property def upload_size_text(self): return displaytext.size_string(self.upload_size) @property def is_failed_download(self): return self.downloader_state == 'failed' @property def pending_auto_dl(self): return (self.feed_auto_downloadable and not self.was_downloaded and self.feed_auto_downloadable and (self.feed_get_everything or self.eligible_for_autodownload)) @property def title_sort_key(self): return util.name_sort_key(self.title) @property def artist_sort_key(self): return util.name_sort_key(self.artist) @property def album_sort_key(self): return util.name_sort_key(self.album) @property def has_parent(self): return self.parent_id is not None @property def parent_title_for_sort(self): """value to use for sorting by parent title. This will sort items by their parent title (torrent folder name or feed name, but if 2 torrents have the same name, or a torrent and a feed have the same name, then they will be separated) """ return (self.parent_title, self.feed_id, self.parent_id) @property def album_artist_sort_key(self): if self.album_artist: return util.name_sort_key(self.album_artist) else: return self.artist_sort_key @property def description_oneline(self): return self.description_stripped[0].replace('\n', '$') @property def auto_rating(self): """Guess at a rating based on the number of times the files has been played vs. skipped and the item's age. """ # TODO: we may want to take into consideration average ratings for this # artist and this album, total play count and skip counts, and average # manual rating SKIP_FACTOR = 1.5 # rating goes to 1 when user skips 40% of the time UNSKIPPED_FACTOR = 2 # rating goes to 5 when user plays 3 times without # skipping # TODO: should divide by log of item's age if self.play_count > 0: if self.skip_count > 0: return min(5, max(1, int(self.play_count - SKIP_FACTOR * self.skip_count))) else: return min(5, int(UNSKIPPED_FACTOR * self.play_count)) elif self.skip_count > 0: return 1 else: return None @property def duration(self): if self.duration_ms is None: return None else: return self.duration_ms // 1000 def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.title) def __str__(self): return '<%s: %s>' % (self.__class__.__name__, self.title)