def get_rssfeed_parsed(self, rssfeed_data, site_cookies_dict=None, user_agent=None): """ rssfeed_data: A dictionary containing rss feed data as stored in the YaRSS2 config. site_cookies_dict: A dictionary of cookie values to be used for this rssfeed. """ return_dict = {} rssfeeds_dict = {} cookie_header = {} return_dict["user_agent"] = user_agent if site_cookies_dict: cookie_header = http.get_cookie_header(site_cookies_dict) return_dict["cookie_header"] = cookie_header self.log.info("Fetching RSS Feed: '%s' with Cookie: '%s' and User-agent: '%s'." % (rssfeed_data["name"], http.get_cookie_header(cookie_header), user_agent)) # Will abort after 10 seconds if server doesn't answer try: parsed_feed = feedparser.parse(rssfeed_data["url"], request_headers=cookie_header, agent=user_agent, timeout=10) except Exception, e: self.log.warn("Exception occured in feedparser: " + str(e)) self.log.warn("Feedparser was called with url: '%s' using cookies: '%s' and User-agent: '%s'" % (rssfeed_data["url"], http.get_cookie_header(cookie_header), user_agent)) self.log.warn("Stacktrace:\n" + common.get_exception_string()) return None
def test_populate_data_fields(self): dummy = DummyClass() rssfeed = yarss2.yarss_config.get_fresh_rssfeed_config() domain = "testdomain.com" url = "http://%s/blabla/blalba.php" % domain rssfeed["name"] = "Test" rssfeed["url"] = url rssfeed["update_interval"] = int(10) rssfeed["site"] = urlparse.urlparse(url).netloc rssfeed["obey_ttl"] = False cookies = {'uid': '92323', 'passkey': 'aksdf9798d767sadf8678as6df9df'} dummy.rssfeed = rssfeed dummy.cookies = { '0': { 'active': True, 'key': '0', 'site': domain, 'value': cookies } } dialog = DialogRSSFeed(dummy, rssfeed) dialog.dialog = dummy data = dialog.get_data_fields(cookies=True) self.assertEquals(data["name"], rssfeed["name"]) self.assertEquals(data["url"], rssfeed["url"]) self.assertEquals(data["obey_ttl"], rssfeed["obey_ttl"]) self.assertEquals(data["update_interval"], rssfeed["update_interval"]) self.assertEquals(data["site"], rssfeed["site"]) cookies_hdr = http.get_cookie_header(cookies) self.assertEquals(data["cookies"], cookies_hdr.get("Cookie", ""))
def test_get_cookie_header(self): url = "http://basename.com/øashdnf/klasflas/dfnmalskdfn/malskdfnmasloal" cookies = {} cookies["0"] = { "active": True, "site": "basename.com", "value": { "key1": "value1", "key2": "value2" } } cookies["1"] = { "active": True, "site": "non-matching-base-name.com", "value": { "key3": "value3", "key4": "value4" } } cookies["2"] = { "active": False, "site": "basename.com", "value": { "key5": "value5", "key6": "value6" } } expected_cookie = "key2=value2; key1=value1" header = http.get_cookie_header(cookies, url) self.assertTrue("Cookie" in header) self.assertEquals(sorted(header["Cookie"].split("; ")), sorted(expected_cookie.split("; ")))
def test_get_cookie_header(self): url = "http://basename.com/øashdnf/klasflas/dfnmalskdfn/malskdfnmasloal" cookies = {} cookies["0"] = {"active": True, "site": "basename.com", "value": {"key1": "value1", "key2": "value2"}} cookies["1"] = {"active": True, "site": "non-matching-base-name.com", "value": {"key3": "value3", "key4": "value4"}} cookies["2"] = {"active": False, "site": "basename.com", "value": {"key5": "value5", "key6": "value6"}} expected_cookie = "key2=value2; key1=value1" header = http.get_cookie_header(cookies, url) self.assertTrue(header.has_key("Cookie")) self.assertEquals(header["Cookie"], expected_cookie)
def populate_data_fields(self): if self.rssfeed: self.glade.get_widget("txt_name").set_text(self.rssfeed["name"]) self.glade.get_widget("txt_url").set_text(self.rssfeed["url"]) self.glade.get_widget("spinbutton_updatetime").set_value( self.rssfeed["update_interval"]) self.glade.get_widget("checkbutton_on_startup").set_active( self.rssfeed["update_on_startup"]) self.glade.get_widget("checkbox_obey_ttl").set_active( self.rssfeed["obey_ttl"]) self.glade.get_widget("checkbox_prefer_magnet").set_active( self.rssfeed["prefer_magnet"]) cookies = http.get_matching_cookies_dict(self.gtkUI.cookies, self.rssfeed["site"]) cookies_hdr = http.get_cookie_header(cookies) self.glade.get_widget("txt_cookies").set_text( cookies_hdr.get("Cookie", "")) # Disable the fields field if "key" in self.rssfeed and self.rssfeed[ "key"] == yarss2.yarss_config.DUMMY_RSSFEED_KEY: self.glade.get_widget("txt_name").set_property( "editable", False) self.glade.get_widget("txt_name").unset_flags(gtk.CAN_FOCUS) self.glade.get_widget("txt_url").set_property( "editable", False) self.glade.get_widget("txt_url").unset_flags(gtk.CAN_FOCUS) self.glade.get_widget("spinbutton_updatetime").set_sensitive( False) self.glade.get_widget("checkbutton_on_startup").set_active( False) self.glade.get_widget("checkbutton_on_startup").set_sensitive( False) self.glade.get_widget("checkbox_obey_ttl").set_active(False) self.glade.get_widget("checkbox_obey_ttl").set_sensitive(False) self.glade.get_widget("checkbox_prefer_magnet").set_active( False) self.glade.get_widget("checkbox_prefer_magnet").set_sensitive( False) self.glade.get_widget("button_save").set_sensitive(False)
def get_rssfeed_parsed(self, rssfeed_data, site_cookies_dict=None): """ rssfeed_data: A dictionary containing rss feed data as stored in the YaRSS2 config. site_cookies_dict: A dictionary of cookie values to be used for this rssfeed. """ return_dict = {} rssfeeds_dict = {} cookie_header = {} if site_cookies_dict: cookie_header = http.get_cookie_header(site_cookies_dict) return_dict["cookie_header"] = cookie_header self.log.info("Fetching RSS Feed: '%s' with Cookie: '%s'." % (rssfeed_data["name"], cookie_header)) # Will abort after 10 seconds if server doesn't answer try: parsed_feed = feedparser.parse(rssfeed_data["url"], request_headers=cookie_header, agent=self.agent, timeout=10) except Exception, e: self.log.warn("Exception occured in feedparser:" + str(e)) self.log.warn("Feedparser was called with url: '%s' and header: '%s'" % (rssfeed_data["url"], cookie_header)) self.log.warn("Stacktrace:" + common.get_exception_string()) return None
def get_rssfeed_parsed(self, rssfeed_data, site_cookies_dict=None, user_agent=None): """ rssfeed_data: A dictionary containing rss feed data as stored in the YaRSS2 config. site_cookies_dict: A dictionary of cookie values to be used for this rssfeed. """ return_dict = {} rssfeeds_dict = {} cookie_header = {} return_dict["user_agent"] = user_agent if site_cookies_dict: cookie_header = http.get_cookie_header(site_cookies_dict) return_dict["cookie_header"] = cookie_header self.log.info("Fetching RSS Feed: '%s' with Cookie: '%s' and User-agent: '%s'." % (rssfeed_data["name"], http.get_cookie_header(cookie_header), user_agent)) # Will abort after 10 seconds if server doesn't answer try: parsed_feed = fetch_and_parse_rssfeed(rssfeed_data["url"], user_agent=user_agent, request_headers=cookie_header, timeout=10) except Exception as e: self.log.warning("Exception occured in feedparser: " + str(e)) self.log.warning("Feedparser was called with url: '%s' using cookies: '%s' and User-agent: '%s'" % (rssfeed_data["url"], http.get_cookie_header(cookie_header), user_agent)) self.log.warning("Stacktrace:\n" + common.get_exception_string()) raise DelugeError("Exception occured in feedparser: " + str(e)) return_dict["raw_result"] = parsed_feed # Error parsing if parsed_feed["bozo"] == 1: return_dict["bozo_exception"] = parsed_feed["bozo_exception"] # Store ttl value if present if "ttl" in parsed_feed["feed"]: return_dict["ttl"] = parsed_feed["feed"]["ttl"] key = 0 no_publish_time = False for item in parsed_feed['items']: # Some RSS feeds do not have a proper timestamp magnet = None torrent = None # Empty item if feed is empty if not item: continue published_date = item.get('published_date', None) if published_date is None: no_publish_time = True return_dict["warning"] = "Published time not available!" # Find the link link = self.get_link(item) magnet = self.get_magnet_link(item) # link or enclosures url is magnet if link is not None and link.startswith("magnet:"): magnet = link else: torrent = link if rssfeed_data.get("prefer_magnet", None) and magnet: link = magnet rssfeeds_dict[key] = self._new_rssfeeds_dict_item(item['title'], link=link, torrent=torrent, magnet=magnet, published_date=published_date) key += 1 if no_publish_time: self.log.warning("Published time is not available!") if key > 0: return_dict["items"] = rssfeeds_dict return return_dict