class RSSFeedHandlingTestCase(unittest.TestCase): def setUp(self): self.log = log self.rssfeedhandler = RSSFeedHandler(self.log) def test_get_rssfeed_parsed(self): file_url = yarss2.util.common.get_resource(common.testdata_rssfeed_filename, path="tests/") rssfeed_data = {"name": "Test", "url": file_url, "site:": "only used whith cookie arguments"} site_cookies = {"uid": "18463", "passkey": "b830f87d023037f9393749s932"} parsed_feed = self.rssfeedhandler.get_rssfeed_parsed(rssfeed_data, site_cookies_dict=site_cookies) # When needing to dump the result in json format #common.json_dump(parsed_feed["items"], "freebsd_rss_items_dump2.json") self.assertTrue(parsed_feed.has_key("items")) items = parsed_feed["items"] stored_items = common.load_json_testdata() self.assertTrue(yarss2.util.common.dicts_equals(items, stored_items, debug=False)) self.assertEquals(parsed_feed["cookie_header"], {'Cookie': 'uid=18463; passkey=b830f87d023037f9393749s932'}) def test_get_link(self): file_url = yarss2.util.common.get_resource(common.testdata_rssfeed_filename, path="tests/") from yarss2.lib.feedparser import feedparser parsed_feed = feedparser.parse(file_url) item = None for e in parsed_feed["items"]: item = e break # Item has enclosure, so it should use that link self.assertEquals(self.rssfeedhandler.get_link(item), item.enclosures[0]["href"]) del item["links"][:] # Item no longer has enclosures, so it should return the regular link self.assertEquals(self.rssfeedhandler.get_link(item), item["link"]) def test_get_size(self): file_url = yarss2.util.common.get_resource("t1.rss", path="tests/data/feeds/") from yarss2.lib.feedparser import feedparser parsed_feed = feedparser.parse(file_url) size = self.rssfeedhandler.get_size(parsed_feed["items"][0]) self.assertEquals(len(size), 1) self.assertEquals(size[0], (4541927915.52, u'4.23 GB')) size = self.rssfeedhandler.get_size(parsed_feed["items"][1]) self.assertEquals(len(size), 1) self.assertEquals(size[0], (402349096.96, u'383.71 MB')) size = self.rssfeedhandler.get_size(parsed_feed["items"][2]) self.assertEquals(len(size), 1) self.assertEquals(size[0], (857007476)) size = self.rssfeedhandler.get_size(parsed_feed["items"][3]) self.assertEquals(len(size), 2) self.assertEquals(size[0], (14353107637)) self.assertEquals(size[1], (13529146982.4, u'12.6 GB')) def get_default_rssfeeds_dict(self): match_option_dict = {} match_option_dict["regex_include"] = "" match_option_dict["regex_exclude"] = "" match_option_dict["regex_include_ignorecase"] = True match_option_dict["regex_exclude_ignorecase"] = True match_option_dict["custom_text_lines"] = None rssfeed_matching = {} rssfeed_matching["0"] = {"matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-amd64-all"} rssfeed_matching["1"] = {"matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-i386-all"} rssfeed_matching["2"] = {"matches": False, "link": "", "title": "fREEbsd-9.0-RELEASE-i386-all"} return match_option_dict, rssfeed_matching def test_update_rssfeeds_dict_matching(self): options, rssfeed_parsed = self.get_default_rssfeeds_dict() options["regex_include"] = "FreeBSD" matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options) self.assertEquals(len(matching.keys()), len(rssfeed_parsed.keys())) # Also make sure the items in 'matching' correspond to the matching items in rssfeed_parsed count = 0 for key in rssfeed_parsed.keys(): if rssfeed_parsed[key]["matches"]: self.assertTrue(matching.has_key(key), "The matches dict does not contain the matching key '%s'" % key) count += 1 self.assertEquals(count, len(matching.keys()), "The number of items in matches dict (%d) does not match the number of matching items (%d)" % (count, len(matching.keys()))) options["regex_include_ignorecase"] = False matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options) self.assertEquals(len(matching.keys()), len(rssfeed_parsed.keys()) - 1) #options["regex_include_ignorecase"] = True options["regex_exclude"] = "i386" matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options) self.assertEquals(len(matching.keys()), len(rssfeed_parsed.keys()) - 2) # Fresh options options, rssfeed_parsed = self.get_default_rssfeeds_dict() # Custom line with unicode characters, norwegian ø and å, as well as Latin Small Letter Lambda with stroke options["custom_text_lines"] = [u"Test line with æ and å, as well as ƛ"] options["regex_include"] = "æ" matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options) self.assertEquals(len(matching.keys()), 1) for key in matching.keys(): self.assertEquals(matching[key]["title"], options["custom_text_lines"][0]) self.assertEquals(matching[key]["regex_include_match"], (15, 17)) options["regex_include"] = "with.*ƛ" matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options) self.assertEquals(len(matching.keys()), 1) for key in matching.keys(): self.assertEquals(matching[key]["title"], options["custom_text_lines"][0]) self.assertEquals(matching[key]["regex_include_match"], (10, 39)) # Test exclude span options["regex_include"] = ".*" options["regex_exclude"] = "line.*å" matching, msg = self.rssfeedhandler.update_rssfeeds_dict_matching(rssfeed_parsed, options) for key in rssfeed_parsed.keys(): if not rssfeed_parsed[key]["matches"]: self.assertEquals(rssfeed_parsed[key]["title"], options["custom_text_lines"][0]) self.assertEquals(rssfeed_parsed[key]["regex_exclude_match"], (5, 24)) break def test_fetch_feed_torrents(self): config = common.get_test_config_dict() # 0 is the rssfeed key matche_result = self.rssfeedhandler.fetch_feed_torrents(config, "0") matches = matche_result["matching_torrents"] self.assertTrue(len(matches) == 3)
class TorrentHandlingTestCase(unittest.TestCase): def setUp(self): # NOQA self.log = log self.config = test_common.get_test_config() # get_test_config will load a new core.conf with the default values. # Must save to save to file so that torrent.py.TorrentOptions loads the default values self.config.core_config.save() global test_component test_component = TestComponent(add_retval=True) def test_add_torrent(self): handler = TorrentHandler(self.log) filename = yarss2.util.common.get_resource( "FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/") torrent_info = {"link": filename, "site_cookies_dict": {}} torrent_download = handler.add_torrent(torrent_info) torrent_added = test_component.added.pop() self.assertTrue(torrent_added.success) self.assertFalse(torrent_added.filedump is None, "Filedump is not None") self.assertEquals(torrent_added.filename, os.path.split(filename)[1]) self.assertFalse(torrent_added.filedump is None) self.assertEquals(torrent_download.url, filename) def test_add_torrent_raise_AddTorrentError(self): # noqa: N802 handler = TorrentHandler(self.log) filename = yarss2.util.common.get_resource( "FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/") torrent_info = {"link": filename, "site_cookies_dict": {}} with mock.patch.object(TestComponent, 'add') as test_component_add: test_component_add.side_effect = AddTorrentError( 'Torrent already in session (%s).' % 1) torrent_added = handler.add_torrent(torrent_info) self.assertFalse(torrent_added.success) def test_add_torrent_magnet_link(self): handler = TorrentHandler(self.log) torrent_url = "magnet:blbalba/url.magnet.link" torrent_info = {"link": torrent_url, "site_cookies_dict": {}} download = handler.add_torrent(torrent_info) self.assertTrue(download.success) self.assertTrue(download.is_magnet) self.assertEquals(test_component.added.pop().magnet, torrent_url) def test_add_torrent_ret_false(self): handler = TorrentHandler(self.log) torrent_url = "http://url.com/file.torrent" cookies = {} global test_component test_component.download_success = False handler.download_torrent_file = test_component.download_torrent_file torrent_info = {"link": torrent_url, "site_cookies_dict": cookies} torrent_download = handler.add_torrent(torrent_info) self.assertFalse(torrent_download.success) # Set by download_torrent_file self.assertEquals(torrent_download.torrent_url, torrent_url) self.assertEquals(torrent_download.cookies, cookies) test_component.download_success = True def test_add_torrent_with_subscription_data(self): handler = TorrentHandler(self.log) subscription_data = yarss2.yarss_config.get_fresh_subscription_config() subscription_data["move_completed"] = "move/path" subscription_data["download_location"] = "download/path" subscription_data[ "add_torrents_in_paused_state"] = GeneralSubsConf.DEFAULT download = TorrentDownload() torrent_info = { "link": "http://url.com/file.torrent", "site_cookies_dict": {}, "subscription_data": subscription_data, "torrent_download": download } d = handler.add_torrent(torrent_info) self.assertTrue(d.success) added = test_component.added.pop() self.assertTrue(added.options["move_completed"]) self.assertEquals(added.options["move_completed_path"], subscription_data["move_completed"]) self.assertEquals(added.options["download_location"], subscription_data["download_location"]) # When using DEFAULT, the default value for add_paused on TorrentSettings is False self.assertEquals(added.options["add_paused"], False) def test_get_torrent(self): handler = TorrentHandler(self.log) handler.download_torrent_file = test_component.download_torrent_file filename = yarss2.util.common.get_resource( "FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/") test_component.use_filedump = read_file(filename) torrent_info = { "link": "http://url.com/file.torrent", "site_cookies_dict": { "cookiekey": "cookievalue" }, "user_agent": "test" } download = handler.get_torrent(torrent_info) self.assertEquals(download.headers, {'User-Agent': 'test'}) self.assertEquals(download.cookies, {'cookiekey': 'cookievalue'}) self.assertFalse(download.is_magnet) def test_get_torrent_magnet(self): handler = TorrentHandler(self.log) torrent_info = {"link": "magnet:hash"} download = handler.get_torrent(torrent_info) self.assertTrue(download.is_magnet) def get_test_rssfeeds_match_dict(self): match_option_dict = {} match_option_dict["regex_include"] = "" match_option_dict["regex_exclude"] = "" match_option_dict["regex_include_ignorecase"] = True match_option_dict["regex_exclude_ignorecase"] = True match_option_dict["custom_text_lines"] = None rssfeed_matching = {} rssfeed_matching["0"] = { "matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-amd64-all" } rssfeed_matching["1"] = { "matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-i386-all" } rssfeed_matching["2"] = { "matches": False, "link": "", "title": "fREEbsd-9.0-RELEASE-i386-all" } return match_option_dict, rssfeed_matching def test_add_torrents(self): handler = TorrentHandler(self.log) from yarss2.rssfeed_handling import RSSFeedHandler self.rssfeedhandler = RSSFeedHandler(self.log) # Override method download_torrent_file handler.download_torrent_file = test_component.download_torrent_file filename = yarss2.util.common.get_resource( "FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/") test_component.use_filedump = read_file(filename) config = get_test_config_dict() # 0 is the rssfeed key match_result = self.rssfeedhandler.fetch_feed_torrents(config, "0") matching_torrents = match_result["matching_torrents"] saved_subscriptions = [] def save_subscription_func(subscription_data): saved_subscriptions.append(subscription_data) handler.add_torrents(save_subscription_func, matching_torrents, self.config.get_config()) self.assertEquals(len(saved_subscriptions), 3) handler.use_filedump = None
class TorrentHandlingTestCase(unittest.TestCase): def setUp(self): self.log = log self.config = common.get_test_config() # get_test_config will load a new core.conf with the default values. # Must save to save to file so that torrent.py.TorrentOptions loads the default values self.config.core_config.save() global test_component test_component = TestComponent(True) def test_add_torrent(self): handler = TorrentHandler(self.log) filename = yarss2.util.common.get_resource("FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/") torrent_info = {"link": filename, "site_cookies_dict": {}} torrent_download = handler.add_torrent(torrent_info) torrent_added = test_component.added.pop() self.assertTrue(torrent_added.success) self.assertFalse(torrent_added.filedump is None, "Filedump is not None") self.assertEquals(torrent_added.filename, os.path.split(filename)[1]) self.assertFalse(torrent_added.filedump is None) self.assertEquals(torrent_download.url, filename) def test_add_torrent_magnet_link(self): handler = TorrentHandler(self.log) torrent_url = "magnet:blbalba/url.magnet.link" torrent_info = {"link": torrent_url, "site_cookies_dict": {}} download = handler.add_torrent(torrent_info) self.assertTrue(download.success) self.assertTrue(download.is_magnet) self.assertEquals(test_component.added.pop().magnet, torrent_url) def test_add_torrent_ret_false(self): handler = TorrentHandler(self.log) torrent_url = "http://url.com/file.torrent" cookies_dict = {} global test_component test_component.download_success = False handler.download_torrent_file = test_component.download_torrent_file torrent_info = {"link": torrent_url, "site_cookies_dict": cookies_dict} torrent_download = handler.add_torrent(torrent_info) self.assertFalse(torrent_download.success) # Set by download_torrent_file self.assertEquals(torrent_download.torrent_url, torrent_url) self.assertEquals(torrent_download.cookies_dict, cookies_dict) test_component.download_success = True def test_add_torrent_with_subscription_data(self): handler = TorrentHandler(self.log) subscription_data = yarss2.yarss_config.get_fresh_subscription_config() subscription_data["move_completed"] = "move/path" subscription_data["download_location"] = "download/path" subscription_data["add_torrents_in_paused_state"] = GeneralSubsConf.DEFAULT download = TorrentDownload() torrent_info = { "link": "http://url.com/file.torrent", "site_cookies_dict": {}, "subscription_data": subscription_data, "torrent_download": download, } d = handler.add_torrent(torrent_info) self.assertTrue(d.success) added = test_component.added.pop() self.assertTrue(added.options["move_completed"]) self.assertEquals(added.options["move_completed_path"], subscription_data["move_completed"]) self.assertEquals(added.options["download_location"], subscription_data["download_location"]) # When using DEFAULT, the default value for add_paused on TorrentSettings is False self.assertEquals(added.options["add_paused"], False) def get_test_rssfeeds_match_dict(self): match_option_dict = {} match_option_dict["regex_include"] = "" match_option_dict["regex_exclude"] = "" match_option_dict["regex_include_ignorecase"] = True match_option_dict["regex_exclude_ignorecase"] = True match_option_dict["custom_text_lines"] = None rssfeed_matching = {} rssfeed_matching["0"] = {"matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-amd64-all"} rssfeed_matching["1"] = {"matches": False, "link": "", "title": "FreeBSD-9.0-RELEASE-i386-all"} rssfeed_matching["2"] = {"matches": False, "link": "", "title": "fREEbsd-9.0-RELEASE-i386-all"} return match_option_dict, rssfeed_matching def test_add_torrents(self): handler = TorrentHandler(self.log) from yarss2.rssfeed_handling import RSSFeedHandler self.rssfeedhandler = RSSFeedHandler(self.log) # Override method download_torrent_file handler.download_torrent_file = test_component.download_torrent_file filename = yarss2.util.common.get_resource("FreeBSD-9.0-RELEASE-amd64-dvd1.torrent", path="tests/data/") test_component.use_filedump = read_file(filename) config = get_test_config_dict() # 0 is the rssfeed key match_result = self.rssfeedhandler.fetch_feed_torrents(config, "0") matching_torrents = match_result["matching_torrents"] saved_subscriptions = [] def save_subscription_func(subscription_data): saved_subscriptions.append(subscription_data) handler.add_torrents(save_subscription_func, matching_torrents, self.config.get_config()) self.assertEquals(len(saved_subscriptions), 1) handler.use_filedump = None
class RSSFeedScheduler(object): """Handles scheduling the RSS Feed fetches.""" def __init__(self, config, logger): self.yarss_config = config self.rssfeed_timers = {} self.run_queue = RSSFeedRunQueue() self.log = logger self.rssfeedhandler = RSSFeedHandler(logger) self.torrent_handler = TorrentHandler(logger) self.add_torrent_func = self.torrent_handler.add_torrents # To make it possible to disable adding torrents in testing def enable_timers(self): """Creates the LoopingCall timers, one for each RSS Feed""" config = self.yarss_config.get_config() for key in config["rssfeeds"]: self.set_timer(config["rssfeeds"][key]["key"], config["rssfeeds"][key]['update_interval']) self.log.info("Scheduled RSS Feed '%s' with interval %s" % (config["rssfeeds"][key]["name"], config["rssfeeds"][key]["update_interval"])) def disable_timers(self): for key in self.rssfeed_timers.keys(): self.rssfeed_timers[key]["timer"].stop() del self.rssfeed_timers[key] def set_timer(self, key, interval): """Schedule a timer for the specified interval.""" try: interval = int(interval) except: self.log.error("Failed to convert interval '%s' to int!" % str(interval)) # Already exists, so reschedule if interval has changed if self.rssfeed_timers.has_key(key): # Interval is the same, so return if self.rssfeed_timers[key]["update_interval"] == interval: return False self.rssfeed_timers[key]["timer"].stop() self.rssfeed_timers[key]["update_interval"] = interval else: # New timer # Second argument, the rssfeedkey is passed as argument in the callback method #timer = LoopingCall(self.rssfeed_update_handler, (key)) timer = LoopingCall(self.queue_rssfeed_update, key) self.rssfeed_timers[key] = {"timer": timer, "update_interval": interval} self.rssfeed_timers[key]["timer"].start(interval * 60, now=False) # Multiply to get seconds return True def delete_timer(self, key): """Delete timer with the specified key.""" if not self.rssfeed_timers.has_key(key): self.log.warn("Cannot delete timer. No timer with key %s" % key) return False self.rssfeed_timers[key]["timer"].stop() del self.rssfeed_timers[key] return True def rssfeed_update_handler(self, rssfeed_key=None, subscription_key=None): """Goes through all the feeds and runs the active ones. Multiple subscriptions on one RSS Feed will download the RSS feed page only once """ if subscription_key: self.log.info("Manually running Subscription '%s'" % (self.yarss_config.get_config()["subscriptions"][subscription_key]["name"])) elif rssfeed_key: if self.yarss_config.get_config()["rssfeeds"][rssfeed_key]["active"] is False: return #self.log.info("Running RSS Feed '%s'" % (self.yarss_config.get_config()["rssfeeds"][rssfeed_key]["name"])) fetch_result = self.rssfeedhandler.fetch_feed_torrents(self.yarss_config.get_config(), rssfeed_key, subscription_key=subscription_key) matching_torrents = fetch_result["matching_torrents"] # Fetching the torrent files. Do this slow task in non-main thread. for torrent in matching_torrents: torrent["torrent_download"] = self.torrent_handler.get_torrent(torrent) # Update TTL value? if fetch_result.has_key("ttl"): # Subscription is run directly. Get RSS Feed key if not rssfeed_key: rssfeed_key = self.yarss_config.get_config()["subscriptions"][subscription_key]["rssfeed_key"] self.log.info("Rescheduling RSS Feed '%s' with interval '%s' according to TTL." % (self.yarss_config.get_config()["rssfeeds"][rssfeed_key]["name"], fetch_result["ttl"])) self.set_timer(rssfeed_key, fetch_result["ttl"]) # Set new interval in config self.yarss_config.get_config()["rssfeeds"][rssfeed_key]["update_interval"] = fetch_result["ttl"] # Send YARSSConfigChangedEvent to GUI with updated config. try: # Tests throws KeyError for EventManager when running this method, so wrap this in try/except component.get("EventManager").emit(YARSSConfigChangedEvent(self.yarss_config.get_config())) except KeyError: pass def save_subscription_func(subscription_data): self.yarss_config.generic_save_config("subscriptions", data_dict=subscription_data) return (self.add_torrent_func, save_subscription_func, fetch_result["matching_torrents"], self.yarss_config.get_config()) def add_torrents_callback(self, args): """This i called with the results from rssfeed_update_handler add_torrent_func must be called on the main thread """ if args is None: return add_torrent_func, save_subscription_func, matching_torrents, config = args add_torrent_func(save_subscription_func, matching_torrents, config) def queue_rssfeed_update(self, *args, **kwargs): d = self.run_queue.push(self.rssfeed_update_handler, *args, **kwargs) d.addCallback(self.add_torrents_callback) return d