def setUp(self) -> None: super().setUp() self.metadata_conn = sqlite3.Connection(":memory:", isolation_level=None) self.metadata_pool = lambda: nullcontext(self.metadata_conn) self.scraper = scrape.MetadataTipScraper( api=self.api, metadata_pool=self.metadata_pool, user_access=site.UserAccess(auth=TEST_AUTH), ) # Set up mock response for feed feed_content = importlib_resources.read_binary( "btn_cache.tests", "test_feed.xml" ) # Don't need full query string for match self.requests_mocker.get( "https://broadcasthe.net/feeds.php", content=feed_content, headers={"Content-Type": "application/xml"}, ) self.torrents: Dict[str, api_types.TorrentEntry] = {} for i in range(1, 6): entry = TEST_ENTRY.copy() entry["TorrentID"] = str(i) entry["Time"] = str(i) self.torrents[str(i)] = entry result = api_types.GetTorrentsResult(results="5", torrents=self.torrents) self.api_mock = self.mock_api_request( "getTorrents", [self.key, {}, 2 ** 31, 0], result )
def test_get_torrent(self, mock: requests_mock.Mocker) -> None: mock.get( "https://broadcasthe.net/torrents.php?action=download&id=456" "&torrent_pass=dummy_passkey", complete_qs=True, text="torrent", ) response = site.UserAccess(auth=AUTH).get_torrent(456) self.assertEqual(response.status_code, 200) self.assertEqual(response.text, "torrent")
def setUp(self) -> None: super().setUp() self.metadata_conn = sqlite3.Connection(":memory:", isolation_level=None) self.metadata_pool = lambda: nullcontext(self.metadata_conn) self.scraper = scrape.MetadataTipScraper( api=self.api, metadata_pool=self.metadata_pool, user_access=site.UserAccess(auth=TEST_AUTH), )
def test_get_feed(self, mock: requests_mock.Mocker) -> None: mock.get( "https://broadcasthe.net/feeds.php?feed=torrents_all&user=123" "&auth=dummy_auth&authkey=dummy_authkey" "&passkey=dummy_passkey", complete_qs=True, text="response", ) response = site.UserAccess(auth=AUTH).get_feed("torrents_all") self.assertEqual(response.status_code, 200) self.assertEqual(response.text, "response")
def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--verbose", "-v", action="count") parser.add_argument("--path", type=pathlib.Path, required=True) parser.add_argument( "--disable", action="append", choices=("metadata", "metadata_tip", "snatchlist"), ) parser.add_argument("--api_max_calls", type=int, default=150) parser.add_argument("--api_period", type=int, default=3600) parser.add_argument("--web_request_rate", type=float, default=0.2) parser.add_argument("--web_request_burst", type=float, default=10) parser.add_argument("--snatchlist_period", type=float, default=3600) parser.add_argument("--parent", type=int) args = parser.parse_args() if args.verbose: level = logging.DEBUG else: level = logging.INFO logging.basicConfig( stream=sys.stdout, level=level, format="%(asctime)s %(levelname)s %(threadName)s " "%(filename)s:%(lineno)d %(message)s", ) storage = storage_lib.Storage(args.path) session = requests.Session() rate_limiter = ratelimit.RateLimiter(rate=args.web_request_rate, burst=args.web_request_burst) api_rate_limiter = ratelimit.APIRateLimiter(max_calls=args.api_max_calls, period=args.api_period) auth = storage.get_user_auth() user_access = site.UserAccess(auth=auth, session=session, rate_limiter=rate_limiter) if auth.api_key is None: raise ValueError("api_key is required") api = api_lib.RateLimitedAPI(auth.api_key, rate_limiter=api_rate_limiter, session=session) def metadata_factory() -> sqlite3.Connection: conn = sqlite3.Connection(storage.metadata_db_path, isolation_level=None) cur = conn.cursor() cur.execute("pragma busy_timeout = 5000") # Metadata updates use temp tables with small data sizes cur.execute("pragma temp_store = MEMORY") cur.execute("pragma trusted_schema = OFF") cur.execute("pragma journal_mode = WAL") cur.execute(f"pragma mmap_size = {2**32}") cur.execute("pragma synchronous = NORMAL") return conn metadata_pool = dbver.null_pool(metadata_factory) def user_factory() -> sqlite3.Connection: conn = sqlite3.Connection(storage.user_db_path, isolation_level=None) cur = conn.cursor() cur.execute("pragma busy_timeout = 5000") cur.execute("pragma trusted_schema = OFF") cur.execute("pragma journal_mode = WAL") cur.execute(f"pragma mmap_size = {2**28}") cur.execute("pragma synchronous = NORMAL") return conn user_pool = dbver.null_pool(user_factory) disable: Set[list] = set(args.disable) if args.disable else set() daemons: Dict[str, daemon_lib.Daemon] = {} if "metadata" not in disable: daemons["metadata_scraper"] = scrape.MetadataScraper( api=api, metadata_pool=metadata_pool) if "metadata_tip" not in disable: daemons["metadata_tip_scraper"] = scrape.MetadataTipScraper( api=api, user_access=user_access, metadata_pool=metadata_pool) if "snatchlist" not in disable: daemons["snatchlist_scraper"] = scrape.SnatchlistScraper( api=api, user_pool=user_pool, period=args.snatchlist_period) if args.parent: daemons["parent_checker"] = ParentChecker(args.parent) executor = concurrent.futures.ThreadPoolExecutor(max_workers=8) def signal_handler(signum: int, _: Any) -> None: _LOG.info("terminating due to signal %d", signum) for daemon in daemons.values(): daemon.terminate() try: # Set signal handlers within the try-finally, so we'll be sure to unset # them if we get a signal while setting them signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) futures: List[concurrent.futures.Future] = [] for name, daemon in daemons.items(): executor = concurrent.futures.ThreadPoolExecutor( thread_name_prefix=name) futures.append(executor.submit(daemon.run)) # Wait for any daemon to die or be terminated concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED) # Ensure all daemons are terminated; all are killed if one dies for daemon in daemons.values(): daemon.terminate() # Re-raise any exceptions for future in futures: future.result() finally: signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL)
async def get_access() -> btn_site.UserAccess: return btn_site.UserAccess( auth=await get_auth(), session=await get_requests_session() )
def test_empty_auth(self, _: requests_mock.Mocker) -> None: access = site.UserAccess(auth=site.UserAuth()) with self.assertRaises(ValueError): access.get_feed("dummy_feed") with self.assertRaises(ValueError): access.get_torrent(456)