class RSSFeedHandlingTestCase(unittest.TestCase):
def setUp(self):
self.log = log
self.rssfeedhandler = RSSFeedHandler(self.log)
def test_get_rssfeed_parsed(self):
file_url = yarss2.util.common.get_resource(common.testdata_rssfeed_filename, path="tests/")
rssfeed_data = {"name": "Test", "url": file_url, "site:": "only used whith cookie arguments"}
site_cookies = {"uid": "18463", "passkey": "b830f87d023037f9393749s932"}
parsed_feed = self.rssfeedhandler.get_rssfeed_parsed(rssfeed_data, site_cookies_dict=site_cookies)
# When needing to dump the result in json format
#common.json_dump(parsed_feed["items"], "freebsd_rss_items_dump2.json")
self.assertTrue(parsed_feed.has_key("items"))
items = parsed_feed["items"]
stored_items = common.load_json_testdata()
self.assertTrue(yarss2.util.common.dicts_equals(items, stored_items, debug=False))
self.assertEquals(parsed_feed["cookie_header"], {'Cookie': 'uid=18463; passkey=b830f87d023037f9393749s932'})
def test_get_link(self):
file_url = yarss2.util.common.get_resource(common.testdata_rssfeed_filename, path="tests/")
from yarss2.lib.feedparser import feedparser
parsed_feed = feedparser.parse(file_url)
item = None
for e in parsed_feed["items"]:
item = e
break
# Item has enclosure, so it should use that link
self.assertEquals(self.rssfeedhandler.get_link(item), item.enclosures[0]["href"])
del item["links"][:]
# Item no longer has enclosures, so it should return the regular link
self.assertEquals(self.rssfeedhandler.get_link(item), item["link"])
def test_get_size(self):
file_url = yarss2.util.common.get_resource("t1.rss", path="tests/data/feeds/")
from yarss2.lib.feedparser import feedparser
parsed_feed = feedparser.parse(file_url)
size = self.rssfeedhandler.get_size(parsed_feed["items"][0])
self.assertEquals(len(size), 1)
self.assertEquals(size[0], (4541927915.52, u'4.23 GB'))
size = self.rssfeedhandler.get_size(parsed_feed["items"][1])
self.assertEquals(len(size), 1)
self.assertEquals(size[0], (402349096.96, u'383.71 MB'))
size = self.rssfeedhandler.get_size(parsed_feed["items"][2])
self.assertEquals(len(size), 1)
self.assertEquals(size[0], (857007476))
size = self.rssfeedhandler.get_size(parsed_feed["items"][3])
self.assertEquals(len(size), 2)
self.assertEquals(size[0], (14353107637))
self.assertEquals(size[1], (13529146982.4, u'12.6 GB'))
def get_default_rssfeeds_dict(self):
match_option_dict = {}
match_option_dict["regex_include"] = ""
match_option_dict["regex_exclude"] = ""
match_option_dict["regex_include_ignorecase"] = True
match_option_dict["regex_exclude_ignorecase"] = True
match_option_dict["custom_text_lines"] = None
rssfeed_matching = {}
rssfeed_matching["0"] = {"matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-amd64-all"}
rssfeed_matching["1"] = {"matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-i386-all"}
rssfeed_matching["2"] = {"matches": False, "link": "", "title": "fREEbsd-9.0-RELEASE-i386-all"}
return match_option_dict, rssfeed_matching
def test_update_rssfeeds_dict_matching(self):
options, rssfeed_parsed = self.get_default_rssfeeds_dict()
options["regex_include"] = "FreeBSD"
matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options)
self.assertEquals(len(matching.keys()), len(rssfeed_parsed.keys()))
# Also make sure the items in 'matching' correspond to the matching items in rssfeed_parsed
count = 0
for key in rssfeed_parsed.keys():
if rssfeed_parsed[key]["matches"]:
self.assertTrue(matching.has_key(key), "The matches dict does not contain the matching key '%s'" % key)
count += 1
self.assertEquals(count, len(matching.keys()),
"The number of items in matches dict (%d) does not match the number of matching items (%d)" % (count, len(matching.keys())))
options["regex_include_ignorecase"] = False
matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options)
self.assertEquals(len(matching.keys()), len(rssfeed_parsed.keys()) - 1)
#options["regex_include_ignorecase"] = True
options["regex_exclude"] = "i386"
matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options)
self.assertEquals(len(matching.keys()), len(rssfeed_parsed.keys()) - 2)
# Fresh options
options, rssfeed_parsed = self.get_default_rssfeeds_dict()
# Custom line with unicode characters, norwegian ø and å, as well as Latin Small Letter Lambda with stroke
options["custom_text_lines"] = [u"Test line with æ and å, as well as ƛ"]
options["regex_include"] = "æ"
matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options)
self.assertEquals(len(matching.keys()), 1)
for key in matching.keys():
self.assertEquals(matching[key]["title"], options["custom_text_lines"][0])
self.assertEquals(matching[key]["regex_include_match"], (15, 17))
options["regex_include"] = "with.*ƛ"
matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options)
self.assertEquals(len(matching.keys()), 1)
for key in matching.keys():
self.assertEquals(matching[key]["title"], options["custom_text_lines"][0])
self.assertEquals(matching[key]["regex_include_match"], (10, 39))
# Test exclude span
options["regex_include"] = ".*"
options["regex_exclude"] = "line.*å"
matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options)
for key in rssfeed_parsed.keys():
if not rssfeed_parsed[key]["matches"]:
self.assertEquals(rssfeed_parsed[key]["title"], options["custom_text_lines"][0])
self.assertEquals(rssfeed_parsed[key]["regex_exclude_match"], (5, 24))
break
def test_fetch_feed_torrents(self):
config = common.get_test_config_dict() # 0 is the rssfeed key
matche_result = self.rssfeedhandler.fetch_feed_torrents(config, "0")
matches = matche_result["matching_torrents"]
self.assertTrue(len(matches) == 3)
class TorrentHandlingTestCase(unittest.TestCase):
def setUp(self): # NOQA
self.log = log
self.config = test_common.get_test_config()
# get_test_config will load a new core.conf with the default values.
# Must save to save to file so that torrent.py.TorrentOptions loads the default values
self.config.core_config.save()
global test_component
test_component = TestComponent(add_retval=True)
def test_add_torrent(self):
handler = TorrentHandler(self.log)
filename = yarss2.util.common.get_resource(
"FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/")
torrent_info = {"link": filename, "site_cookies_dict": {}}
torrent_download = handler.add_torrent(torrent_info)
torrent_added = test_component.added.pop()
self.assertTrue(torrent_added.success)
self.assertFalse(torrent_added.filedump is None,
"Filedump is not None")
self.assertEquals(torrent_added.filename, os.path.split(filename)[1])
self.assertFalse(torrent_added.filedump is None)
self.assertEquals(torrent_download.url, filename)
def test_add_torrent_raise_AddTorrentError(self): # noqa: N802
handler = TorrentHandler(self.log)
filename = yarss2.util.common.get_resource(
"FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/")
torrent_info = {"link": filename, "site_cookies_dict": {}}
with mock.patch.object(TestComponent, 'add') as test_component_add:
test_component_add.side_effect = AddTorrentError(
'Torrent already in session (%s).' % 1)
torrent_added = handler.add_torrent(torrent_info)
self.assertFalse(torrent_added.success)
def test_add_torrent_magnet_link(self):
handler = TorrentHandler(self.log)
torrent_url = "magnet:blbalba/url.magnet.link"
torrent_info = {"link": torrent_url, "site_cookies_dict": {}}
download = handler.add_torrent(torrent_info)
self.assertTrue(download.success)
self.assertTrue(download.is_magnet)
self.assertEquals(test_component.added.pop().magnet, torrent_url)
def test_add_torrent_ret_false(self):
handler = TorrentHandler(self.log)
torrent_url = "http://url.com/file.torrent"
cookies = {}
global test_component
test_component.download_success = False
handler.download_torrent_file = test_component.download_torrent_file
torrent_info = {"link": torrent_url, "site_cookies_dict": cookies}
torrent_download = handler.add_torrent(torrent_info)
self.assertFalse(torrent_download.success)
# Set by download_torrent_file
self.assertEquals(torrent_download.torrent_url, torrent_url)
self.assertEquals(torrent_download.cookies, cookies)
test_component.download_success = True
def test_add_torrent_with_subscription_data(self):
handler = TorrentHandler(self.log)
subscription_data = yarss2.yarss_config.get_fresh_subscription_config()
subscription_data["move_completed"] = "move/path"
subscription_data["download_location"] = "download/path"
subscription_data[
"add_torrents_in_paused_state"] = GeneralSubsConf.DEFAULT
download = TorrentDownload()
torrent_info = {
"link": "http://url.com/file.torrent",
"site_cookies_dict": {},
"subscription_data": subscription_data,
"torrent_download": download
}
d = handler.add_torrent(torrent_info)
self.assertTrue(d.success)
added = test_component.added.pop()
self.assertTrue(added.options["move_completed"])
self.assertEquals(added.options["move_completed_path"],
subscription_data["move_completed"])
self.assertEquals(added.options["download_location"],
subscription_data["download_location"])
# When using DEFAULT, the default value for add_paused on TorrentSettings is False
self.assertEquals(added.options["add_paused"], False)
def test_get_torrent(self):
handler = TorrentHandler(self.log)
handler.download_torrent_file = test_component.download_torrent_file
filename = yarss2.util.common.get_resource(
"FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/")
test_component.use_filedump = read_file(filename)
torrent_info = {
"link": "http://url.com/file.torrent",
"site_cookies_dict": {
"cookiekey": "cookievalue"
},
"user_agent": "test"
}
download = handler.get_torrent(torrent_info)
self.assertEquals(download.headers, {'User-Agent': 'test'})
self.assertEquals(download.cookies, {'cookiekey': 'cookievalue'})
self.assertFalse(download.is_magnet)
def test_get_torrent_magnet(self):
handler = TorrentHandler(self.log)
torrent_info = {"link": "magnet:hash"}
download = handler.get_torrent(torrent_info)
self.assertTrue(download.is_magnet)
def get_test_rssfeeds_match_dict(self):
match_option_dict = {}
match_option_dict["regex_include"] = ""
match_option_dict["regex_exclude"] = ""
match_option_dict["regex_include_ignorecase"] = True
match_option_dict["regex_exclude_ignorecase"] = True
match_option_dict["custom_text_lines"] = None
rssfeed_matching = {}
rssfeed_matching["0"] = {
"matches": False,
"link": "",
"title": "FreeBSD-9.0-RELEASE-amd64-all"
}
rssfeed_matching["1"] = {
"matches": False,
"link": "",
"title": "FreeBSD-9.0-RELEASE-i386-all"
}
rssfeed_matching["2"] = {
"matches": False,
"link": "",
"title": "fREEbsd-9.0-RELEASE-i386-all"
}
return match_option_dict, rssfeed_matching
def test_add_torrents(self):
handler = TorrentHandler(self.log)
from yarss2.rssfeed_handling import RSSFeedHandler
self.rssfeedhandler = RSSFeedHandler(self.log)
# Override method download_torrent_file
handler.download_torrent_file = test_component.download_torrent_file
filename = yarss2.util.common.get_resource(
"FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/")
test_component.use_filedump = read_file(filename)
config = get_test_config_dict() # 0 is the rssfeed key
match_result = self.rssfeedhandler.fetch_feed_torrents(config, "0")
matching_torrents = match_result["matching_torrents"]
saved_subscriptions = []
def save_subscription_func(subscription_data):
saved_subscriptions.append(subscription_data)
handler.add_torrents(save_subscription_func, matching_torrents,
self.config.get_config())
self.assertEquals(len(saved_subscriptions), 3)
handler.use_filedump = None
class TorrentHandlingTestCase(unittest.TestCase):
def setUp(self):
self.log = log
self.config = common.get_test_config()
# get_test_config will load a new core.conf with the default values.
# Must save to save to file so that torrent.py.TorrentOptions loads the default values
self.config.core_config.save()
global test_component
test_component = TestComponent(True)
def test_add_torrent(self):
handler = TorrentHandler(self.log)
filename = yarss2.util.common.get_resource("FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/")
torrent_info = {"link": filename, "site_cookies_dict": {}}
torrent_download = handler.add_torrent(torrent_info)
torrent_added = test_component.added.pop()
self.assertTrue(torrent_added.success)
self.assertFalse(torrent_added.filedump is None, "Filedump is not None")
self.assertEquals(torrent_added.filename, os.path.split(filename)[1])
self.assertFalse(torrent_added.filedump is None)
self.assertEquals(torrent_download.url, filename)
def test_add_torrent_magnet_link(self):
handler = TorrentHandler(self.log)
torrent_url = "magnet:blbalba/url.magnet.link"
torrent_info = {"link": torrent_url, "site_cookies_dict": {}}
download = handler.add_torrent(torrent_info)
self.assertTrue(download.success)
self.assertTrue(download.is_magnet)
self.assertEquals(test_component.added.pop().magnet, torrent_url)
def test_add_torrent_ret_false(self):
handler = TorrentHandler(self.log)
torrent_url = "http://url.com/file.torrent"
cookies_dict = {}
global test_component
test_component.download_success = False
handler.download_torrent_file = test_component.download_torrent_file
torrent_info = {"link": torrent_url, "site_cookies_dict": cookies_dict}
torrent_download = handler.add_torrent(torrent_info)
self.assertFalse(torrent_download.success)
# Set by download_torrent_file
self.assertEquals(torrent_download.torrent_url, torrent_url)
self.assertEquals(torrent_download.cookies_dict, cookies_dict)
test_component.download_success = True
def test_add_torrent_with_subscription_data(self):
handler = TorrentHandler(self.log)
subscription_data = yarss2.yarss_config.get_fresh_subscription_config()
subscription_data["move_completed"] = "move/path"
subscription_data["download_location"] = "download/path"
subscription_data["add_torrents_in_paused_state"] = GeneralSubsConf.DEFAULT
download = TorrentDownload()
torrent_info = {
"link": "http://url.com/file.torrent",
"site_cookies_dict": {},
"subscription_data": subscription_data,
"torrent_download": download,
}
d = handler.add_torrent(torrent_info)
self.assertTrue(d.success)
added = test_component.added.pop()
self.assertTrue(added.options["move_completed"])
self.assertEquals(added.options["move_completed_path"], subscription_data["move_completed"])
self.assertEquals(added.options["download_location"], subscription_data["download_location"])
# When using DEFAULT, the default value for add_paused on TorrentSettings is False
self.assertEquals(added.options["add_paused"], False)
def get_test_rssfeeds_match_dict(self):
match_option_dict = {}
match_option_dict["regex_include"] = ""
match_option_dict["regex_exclude"] = ""
match_option_dict["regex_include_ignorecase"] = True
match_option_dict["regex_exclude_ignorecase"] = True
match_option_dict["custom_text_lines"] = None
rssfeed_matching = {}
rssfeed_matching["0"] = {"matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-amd64-all"}
rssfeed_matching["1"] = {"matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-i386-all"}
rssfeed_matching["2"] = {"matches": False, "link": "", "title": "fREEbsd-9.0-RELEASE-i386-all"}
return match_option_dict, rssfeed_matching
def test_add_torrents(self):
handler = TorrentHandler(self.log)
from yarss2.rssfeed_handling import RSSFeedHandler
self.rssfeedhandler = RSSFeedHandler(self.log)
# Override method download_torrent_file
handler.download_torrent_file = test_component.download_torrent_file
filename = yarss2.util.common.get_resource("FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/")
test_component.use_filedump = read_file(filename)
config = get_test_config_dict() # 0 is the rssfeed key
match_result = self.rssfeedhandler.fetch_feed_torrents(config, "0")
matching_torrents = match_result["matching_torrents"]
saved_subscriptions = []
def save_subscription_func(subscription_data):
saved_subscriptions.append(subscription_data)
handler.add_torrents(save_subscription_func, matching_torrents, self.config.get_config())
self.assertEquals(len(saved_subscriptions), 1)
handler.use_filedump = None
class RSSFeedScheduler(object):
"""Handles scheduling the RSS Feed fetches."""
def __init__(self, config, logger):
self.yarss_config = config
self.rssfeed_timers = {}
self.run_queue = RSSFeedRunQueue()
self.log = logger
self.rssfeedhandler = RSSFeedHandler(logger)
self.torrent_handler = TorrentHandler(logger)
self.add_torrent_func = self.torrent_handler.add_torrents # To make it possible to disable adding torrents in testing
def enable_timers(self):
"""Creates the LoopingCall timers, one for each RSS Feed"""
config = self.yarss_config.get_config()
for key in config["rssfeeds"]:
self.set_timer(config["rssfeeds"][key]["key"], config["rssfeeds"][key]['update_interval'])
self.log.info("Scheduled RSS Feed '%s' with interval %s" %
(config["rssfeeds"][key]["name"], config["rssfeeds"][key]["update_interval"]))
def disable_timers(self):
for key in self.rssfeed_timers.keys():
self.rssfeed_timers[key]["timer"].stop()
del self.rssfeed_timers[key]
def set_timer(self, key, interval):
"""Schedule a timer for the specified interval."""
try:
interval = int(interval)
except:
self.log.error("Failed to convert interval '%s' to int!" % str(interval))
# Already exists, so reschedule if interval has changed
if self.rssfeed_timers.has_key(key):
# Interval is the same, so return
if self.rssfeed_timers[key]["update_interval"] == interval:
return False
self.rssfeed_timers[key]["timer"].stop()
self.rssfeed_timers[key]["update_interval"] = interval
else:
# New timer
# Second argument, the rssfeedkey is passed as argument in the callback method
#timer = LoopingCall(self.rssfeed_update_handler, (key))
timer = LoopingCall(self.queue_rssfeed_update, key)
self.rssfeed_timers[key] = {"timer": timer, "update_interval": interval}
self.rssfeed_timers[key]["timer"].start(interval * 60, now=False) # Multiply to get seconds
return True
def delete_timer(self, key):
"""Delete timer with the specified key."""
if not self.rssfeed_timers.has_key(key):
self.log.warn("Cannot delete timer. No timer with key %s" % key)
return False
self.rssfeed_timers[key]["timer"].stop()
del self.rssfeed_timers[key]
return True
def rssfeed_update_handler(self, rssfeed_key=None, subscription_key=None):
"""Goes through all the feeds and runs the active ones.
Multiple subscriptions on one RSS Feed will download the RSS feed page only once
"""
if subscription_key:
self.log.info("Manually running Subscription '%s'" %
(self.yarss_config.get_config()["subscriptions"][subscription_key]["name"]))
elif rssfeed_key:
if self.yarss_config.get_config()["rssfeeds"][rssfeed_key]["active"] is False:
return
#self.log.info("Running RSS Feed '%s'" % (self.yarss_config.get_config()["rssfeeds"][rssfeed_key]["name"]))
fetch_result = self.rssfeedhandler.fetch_feed_torrents(self.yarss_config.get_config(), rssfeed_key,
subscription_key=subscription_key)
matching_torrents = fetch_result["matching_torrents"]
# Fetching the torrent files. Do this slow task in non-main thread.
for torrent in matching_torrents:
torrent["torrent_download"] = self.torrent_handler.get_torrent(torrent)
# Update TTL value?
if fetch_result.has_key("ttl"):
# Subscription is run directly. Get RSS Feed key
if not rssfeed_key:
rssfeed_key = self.yarss_config.get_config()["subscriptions"][subscription_key]["rssfeed_key"]
self.log.info("Rescheduling RSS Feed '%s' with interval '%s' according to TTL." %
(self.yarss_config.get_config()["rssfeeds"][rssfeed_key]["name"], fetch_result["ttl"]))
self.set_timer(rssfeed_key, fetch_result["ttl"])
# Set new interval in config
self.yarss_config.get_config()["rssfeeds"][rssfeed_key]["update_interval"] = fetch_result["ttl"]
# Send YARSSConfigChangedEvent to GUI with updated config.
try:
# Tests throws KeyError for EventManager when running this method, so wrap this in try/except
component.get("EventManager").emit(YARSSConfigChangedEvent(self.yarss_config.get_config()))
except KeyError:
pass
def save_subscription_func(subscription_data):
self.yarss_config.generic_save_config("subscriptions", data_dict=subscription_data)
return (self.add_torrent_func, save_subscription_func,
fetch_result["matching_torrents"], self.yarss_config.get_config())
def add_torrents_callback(self, args):
"""This i called with the results from rssfeed_update_handler
add_torrent_func must be called on the main thread
"""
if args is None:
return
add_torrent_func, save_subscription_func, matching_torrents, config = args
add_torrent_func(save_subscription_func, matching_torrents, config)
def queue_rssfeed_update(self, *args, **kwargs):
d = self.run_queue.push(self.rssfeed_update_handler, *args, **kwargs)
d.addCallback(self.add_torrents_callback)
return d
