def get_rssfeed_parsed(self, rssfeed_data, site_cookies_dict=None, user_agent=None):
"""
rssfeed_data: A dictionary containing rss feed data as stored in the YaRSS2 config.
site_cookies_dict: A dictionary of cookie values to be used for this rssfeed.
"""
return_dict = {}
rssfeeds_dict = {}
cookie_header = {}
return_dict["user_agent"] = user_agent
if site_cookies_dict:
cookie_header = http.get_cookie_header(site_cookies_dict)
return_dict["cookie_header"] = cookie_header
self.log.info("Fetching RSS Feed: '%s' with Cookie: '%s' and User-agent: '%s'." %
(rssfeed_data["name"], http.get_cookie_header(cookie_header), user_agent))
# Will abort after 10 seconds if server doesn't answer
try:
parsed_feed = feedparser.parse(rssfeed_data["url"], request_headers=cookie_header,
agent=user_agent,
timeout=10)
except Exception, e:
self.log.warn("Exception occured in feedparser: " + str(e))
self.log.warn("Feedparser was called with url: '%s' using cookies: '%s' and User-agent: '%s'" %
(rssfeed_data["url"], http.get_cookie_header(cookie_header), user_agent))
self.log.warn("Stacktrace:\n" + common.get_exception_string())
return None
def test_populate_data_fields(self):
dummy = DummyClass()
rssfeed = yarss2.yarss_config.get_fresh_rssfeed_config()
domain = "testdomain.com"
url = "http://%s/blabla/blalba.php" % domain
rssfeed["name"] = "Test"
rssfeed["url"] = url
rssfeed["update_interval"] = int(10)
rssfeed["site"] = urlparse.urlparse(url).netloc
rssfeed["obey_ttl"] = False
cookies = {'uid': '92323', 'passkey': 'aksdf9798d767sadf8678as6df9df'}
dummy.rssfeed = rssfeed
dummy.cookies = {
'0': {
'active': True,
'key': '0',
'site': domain,
'value': cookies
}
}
dialog = DialogRSSFeed(dummy, rssfeed)
dialog.dialog = dummy
data = dialog.get_data_fields(cookies=True)
self.assertEquals(data["name"], rssfeed["name"])
self.assertEquals(data["url"], rssfeed["url"])
self.assertEquals(data["obey_ttl"], rssfeed["obey_ttl"])
self.assertEquals(data["update_interval"], rssfeed["update_interval"])
self.assertEquals(data["site"], rssfeed["site"])
cookies_hdr = http.get_cookie_header(cookies)
self.assertEquals(data["cookies"], cookies_hdr.get("Cookie", ""))
def test_get_cookie_header(self):
url = "http://basename.com/øashdnf/klasflas/dfnmalskdfn/malskdfnmasloal"
cookies = {}
cookies["0"] = {
"active": True,
"site": "basename.com",
"value": {
"key1": "value1",
"key2": "value2"
}
}
cookies["1"] = {
"active": True,
"site": "non-matching-base-name.com",
"value": {
"key3": "value3",
"key4": "value4"
}
}
cookies["2"] = {
"active": False,
"site": "basename.com",
"value": {
"key5": "value5",
"key6": "value6"
}
}
expected_cookie = "key2=value2; key1=value1"
header = http.get_cookie_header(cookies, url)
self.assertTrue("Cookie" in header)
self.assertEquals(sorted(header["Cookie"].split("; ")),
sorted(expected_cookie.split("; ")))
def test_get_cookie_header(self):
url = "http://basename.com/øashdnf/klasflas/dfnmalskdfn/malskdfnmasloal"
cookies = {}
cookies["0"] = {"active": True, "site": "basename.com",
"value": {"key1": "value1", "key2": "value2"}}
cookies["1"] = {"active": True, "site": "non-matching-base-name.com",
"value": {"key3": "value3", "key4": "value4"}}
cookies["2"] = {"active": False, "site": "basename.com",
"value": {"key5": "value5", "key6": "value6"}}
expected_cookie = "key2=value2; key1=value1"
header = http.get_cookie_header(cookies, url)
self.assertTrue(header.has_key("Cookie"))
self.assertEquals(header["Cookie"], expected_cookie)
def populate_data_fields(self):
if self.rssfeed:
self.glade.get_widget("txt_name").set_text(self.rssfeed["name"])
self.glade.get_widget("txt_url").set_text(self.rssfeed["url"])
self.glade.get_widget("spinbutton_updatetime").set_value(
self.rssfeed["update_interval"])
self.glade.get_widget("checkbutton_on_startup").set_active(
self.rssfeed["update_on_startup"])
self.glade.get_widget("checkbox_obey_ttl").set_active(
self.rssfeed["obey_ttl"])
self.glade.get_widget("checkbox_prefer_magnet").set_active(
self.rssfeed["prefer_magnet"])
cookies = http.get_matching_cookies_dict(self.gtkUI.cookies,
self.rssfeed["site"])
cookies_hdr = http.get_cookie_header(cookies)
self.glade.get_widget("txt_cookies").set_text(
cookies_hdr.get("Cookie", ""))
# Disable the fields field
if "key" in self.rssfeed and self.rssfeed[
"key"] == yarss2.yarss_config.DUMMY_RSSFEED_KEY:
self.glade.get_widget("txt_name").set_property(
"editable", False)
self.glade.get_widget("txt_name").unset_flags(gtk.CAN_FOCUS)
self.glade.get_widget("txt_url").set_property(
"editable", False)
self.glade.get_widget("txt_url").unset_flags(gtk.CAN_FOCUS)
self.glade.get_widget("spinbutton_updatetime").set_sensitive(
False)
self.glade.get_widget("checkbutton_on_startup").set_active(
False)
self.glade.get_widget("checkbutton_on_startup").set_sensitive(
False)
self.glade.get_widget("checkbox_obey_ttl").set_active(False)
self.glade.get_widget("checkbox_obey_ttl").set_sensitive(False)
self.glade.get_widget("checkbox_prefer_magnet").set_active(
False)
self.glade.get_widget("checkbox_prefer_magnet").set_sensitive(
False)
self.glade.get_widget("button_save").set_sensitive(False)
def get_rssfeed_parsed(self, rssfeed_data, site_cookies_dict=None):
"""
rssfeed_data: A dictionary containing rss feed data as stored in the YaRSS2 config.
site_cookies_dict: A dictionary of cookie values to be used for this rssfeed.
"""
return_dict = {}
rssfeeds_dict = {}
cookie_header = {}
if site_cookies_dict:
cookie_header = http.get_cookie_header(site_cookies_dict)
return_dict["cookie_header"] = cookie_header
self.log.info("Fetching RSS Feed: '%s' with Cookie: '%s'." % (rssfeed_data["name"], cookie_header))
# Will abort after 10 seconds if server doesn't answer
try:
parsed_feed = feedparser.parse(rssfeed_data["url"], request_headers=cookie_header, agent=self.agent, timeout=10)
except Exception, e:
self.log.warn("Exception occured in feedparser:" + str(e))
self.log.warn("Feedparser was called with url: '%s' and header: '%s'" % (rssfeed_data["url"], cookie_header))
self.log.warn("Stacktrace:" + common.get_exception_string())
return None
def get_rssfeed_parsed(self, rssfeed_data, site_cookies_dict=None, user_agent=None):
"""
rssfeed_data: A dictionary containing rss feed data as stored in the YaRSS2 config.
site_cookies_dict: A dictionary of cookie values to be used for this rssfeed.
"""
return_dict = {}
rssfeeds_dict = {}
cookie_header = {}
return_dict["user_agent"] = user_agent
if site_cookies_dict:
cookie_header = http.get_cookie_header(site_cookies_dict)
return_dict["cookie_header"] = cookie_header
self.log.info("Fetching RSS Feed: '%s' with Cookie: '%s' and User-agent: '%s'." %
(rssfeed_data["name"], http.get_cookie_header(cookie_header), user_agent))
# Will abort after 10 seconds if server doesn't answer
try:
parsed_feed = fetch_and_parse_rssfeed(rssfeed_data["url"], user_agent=user_agent,
request_headers=cookie_header, timeout=10)
except Exception as e:
self.log.warning("Exception occured in feedparser: " + str(e))
self.log.warning("Feedparser was called with url: '%s' using cookies: '%s' and User-agent: '%s'" %
(rssfeed_data["url"], http.get_cookie_header(cookie_header), user_agent))
self.log.warning("Stacktrace:\n" + common.get_exception_string())
raise DelugeError("Exception occured in feedparser: " + str(e))
return_dict["raw_result"] = parsed_feed
# Error parsing
if parsed_feed["bozo"] == 1:
return_dict["bozo_exception"] = parsed_feed["bozo_exception"]
# Store ttl value if present
if "ttl" in parsed_feed["feed"]:
return_dict["ttl"] = parsed_feed["feed"]["ttl"]
key = 0
no_publish_time = False
for item in parsed_feed['items']:
# Some RSS feeds do not have a proper timestamp
magnet = None
torrent = None
# Empty item if feed is empty
if not item:
continue
published_date = item.get('published_date', None)
if published_date is None:
no_publish_time = True
return_dict["warning"] = "Published time not available!"
# Find the link
link = self.get_link(item)
magnet = self.get_magnet_link(item)
# link or enclosures url is magnet
if link is not None and link.startswith("magnet:"):
magnet = link
else:
torrent = link
if rssfeed_data.get("prefer_magnet", None) and magnet:
link = magnet
rssfeeds_dict[key] = self._new_rssfeeds_dict_item(item['title'], link=link,
torrent=torrent, magnet=magnet,
published_date=published_date)
key += 1
if no_publish_time:
self.log.warning("Published time is not available!")
if key > 0:
return_dict["items"] = rssfeeds_dict
return return_dict
