def downloader_for_url(url: str) -> str:
"""
Returns the name of the downloader that gallery-dl would use for the given URL.
Returns an empty string if gallery-dl does not recognize the URL.
"""
u = uri_normalizer.normalizes(url)
if match := extractor.find(u):
return match.category
def test_add_module(self):
uri = "fake:foobar"
self.assertIsNone(extractor.find(uri))
classes = extractor.add_module(sys.modules[__name__])
self.assertEqual(len(classes), 1)
self.assertEqual(classes[0].pattern, FakeExtractor.pattern)
self.assertEqual(classes[0], FakeExtractor)
self.assertIsInstance(extractor.find(uri), FakeExtractor)
def test_add_module(self):
uri = "fake:foobar"
self.assertIsNone(extractor.find(uri))
classes = extractor.add_module(sys.modules[__name__])
self.assertEqual(len(classes), 1)
self.assertEqual(classes[0].pattern, FakeExtractor.pattern)
self.assertEqual(classes[0], FakeExtractor)
self.assertIsInstance(extractor.find(uri), FakeExtractor)
def test_find(self):
for uri in self.VALID_URIS:
result = extractor.find(uri)
self.assertIsInstance(result, Extractor, uri)
for not_found in ("", "/tmp/file.ext"):
self.assertIsNone(extractor.find(not_found))
for invalid in (None, [], {}, 123, b"test:"):
with self.assertRaises(TypeError):
extractor.find(invalid)
def test_find(self):
for uri in self.VALID_URIS:
result = extractor.find(uri)
self.assertIsInstance(result, Extractor, uri)
for not_found in ("", "/tmp/file.ext"):
self.assertIsNone(extractor.find(not_found))
for invalid in (None, [], {}, 123, b"test:"):
with self.assertRaises(TypeError):
extractor.find(invalid)
def test_check_cookies_expires(self):
extr = extractor.find("test:")
self.assertFalse(extr._cookiejar, "empty")
self.assertFalse(extr.cookiedomain, "empty")
now = int(time.time())
log = logging.getLogger("test")
extr._cookiejar.set("a", "1", expires=now-100)
with mock.patch.object(log, "warning") as mw:
self.assertFalse(extr._check_cookies(("a",)))
self.assertEqual(mw.call_count, 1)
self.assertEqual(mw.call_args[0], ("Cookie '%s' has expired", "a"))
extr._cookiejar.set("a", "1", expires=now+100)
with mock.patch.object(log, "warning") as mw:
self.assertFalse(extr._check_cookies(("a",)))
self.assertEqual(mw.call_count, 1)
self.assertEqual(mw.call_args[0], (
"Cookie '%s' will expire in less than %s hour%s", "a", 1, ""))
extr._cookiejar.set("a", "1", expires=now+100+7200)
with mock.patch.object(log, "warning") as mw:
self.assertFalse(extr._check_cookies(("a",)))
self.assertEqual(mw.call_count, 1)
self.assertEqual(mw.call_args[0], (
"Cookie '%s' will expire in less than %s hour%s", "a", 3, "s"))
extr._cookiejar.set("a", "1", expires=now+100+24*3600)
with mock.patch.object(log, "warning") as mw:
self.assertTrue(extr._check_cookies(("a",)))
self.assertEqual(mw.call_count, 0)
def test_oauth1(self):
for category in ("flickr", "smugmug", "tumblr"):
extr = extractor.find("oauth:" + category)
with patch.object(extr, "_oauth1_authorization_flow") as m:
for msg in extr:
pass
self.assertEqual(len(m.mock_calls), 1)
def test_oauth2(self):
for category in ("deviantart", "reddit"):
extr = extractor.find("oauth:" + category)
with patch.object(extr, "_oauth2_authorization_code_grant") as m:
for msg in extr:
pass
self.assertEqual(len(m.mock_calls), 1)
def _test_warning(self, filename, exc):
config.set((), "cookies", filename)
log = logging.getLogger("test")
with mock.patch.object(log, "warning") as mock_warning:
cookies = extractor.find("test:").session.cookies
self.assertEqual(len(cookies), 0)
self.assertEqual(mock_warning.call_count, 1)
self.assertEqual(mock_warning.call_args[0][0], "cookies: %s")
self.assertIsInstance(mock_warning.call_args[0][1], exc)
def test_oauth2_mastodon(self):
extr = extractor.find("oauth:mastodon:pawoo.net")
with patch.object(extr, "_oauth2_authorization_code_grant") as m, \
patch.object(extr, "_register") as r:
for msg in extr:
pass
self.assertEqual(len(r.mock_calls), 0)
self.assertEqual(len(m.mock_calls), 1)
def _test_warning(self, filename, exc):
config.set(CKEY, filename)
log = logging.getLogger("test")
with mock.patch.object(log, "warning") as mock_warning:
cookies = extractor.find("test:").session.cookies
self.assertEqual(len(cookies), 0)
self.assertEqual(mock_warning.call_count, 1)
self.assertEqual(mock_warning.call_args[0][0], "cookies: %s")
self.assertIsInstance(mock_warning.call_args[0][1], exc)
def test_find(self):
valid_uris = (
"https://example.org/file.jpg",
"tumblr:foobar",
"oauth:flickr",
"test:pixiv:",
"recursive:https://example.org/document.html",
)
for uri in valid_uris:
result = extractor.find(uri)
self.assertIsInstance(result, Extractor, uri)
for not_found in ("", "/tmp/file.ext"):
self.assertIsNone(extractor.find(not_found))
for invalid in (None, [], {}, 123, b"test:"):
with self.assertRaises(TypeError):
extractor.find(invalid)
def test_cookiefile(self):
config.set((), "cookies", self.cookiefile)
cookies = extractor.find("test:").session.cookies
self.assertEqual(len(cookies), 1)
cookie = next(iter(cookies))
self.assertEqual(cookie.domain, ".example.org")
self.assertEqual(cookie.path, "/")
self.assertEqual(cookie.name, "NAME")
self.assertEqual(cookie.value, "VALUE")
def test_cookiefile(self):
config.set(CKEY, self.cookiefile)
cookies = extractor.find("test:").session.cookies
self.assertEqual(len(cookies), 1)
cookie = next(iter(cookies))
self.assertEqual(cookie.domain, ".example.org")
self.assertEqual(cookie.path, "/")
self.assertEqual(cookie.name, "NAME")
self.assertEqual(cookie.value, "VALUE")
def test_wait_seconds(self):
extr = extractor.find("test:")
seconds = 5
until = time.time() + seconds
with patch("time.sleep") as sleep, patch.object(extr, "log") as log:
extr.wait(seconds=seconds)
sleep.assert_called_once_with(6.0)
calls = log.info.mock_calls
self.assertEqual(len(calls), 1)
self._assert_isotime(calls[0][1][1], until)
def test_from_url(self):
for uri in self.VALID_URIS:
cls = extractor.find(uri).__class__
extr = cls.from_url(uri)
self.assertIs(type(extr), cls)
self.assertIsInstance(extr, Extractor)
for not_found in ("", "/tmp/file.ext"):
self.assertIsNone(FakeExtractor.from_url(not_found))
for invalid in (None, [], {}, 123, b"test:"):
with self.assertRaises(TypeError):
FakeExtractor.from_url(invalid)
def test_from_url(self):
for uri in self.VALID_URIS:
cls = extractor.find(uri).__class__
extr = cls.from_url(uri)
self.assertIs(type(extr), cls)
self.assertIsInstance(extr, Extractor)
for not_found in ("", "/tmp/file.ext"):
self.assertIsNone(FakeExtractor.from_url(not_found))
for invalid in (None, [], {}, 123, b"test:"):
with self.assertRaises(TypeError):
FakeExtractor.from_url(invalid)
def test_wait_until(self):
extr = extractor.find("test:")
until = time.time() + 5
with patch("time.sleep") as sleep, patch.object(extr, "log") as log:
extr.wait(until=until)
calls = sleep.mock_calls
self.assertEqual(len(calls), 1)
self.assertAlmostEqual(calls[0][1][0], 6.0, places=1)
calls = log.info.mock_calls
self.assertEqual(len(calls), 1)
self._assert_isotime(calls[0][1][1], until)
def test_wait_until_datetime(self):
extr = extractor.find("test:")
until = datetime.utcnow() + timedelta(seconds=5)
until_local = datetime.now() + timedelta(seconds=5)
with patch("time.sleep") as sleep, patch.object(extr, "log") as log:
extr.wait(until=until)
calls = sleep.mock_calls
self.assertEqual(len(calls), 1)
self.assertAlmostEqual(calls[0][1][0], 6.0, places=1)
calls = log.info.mock_calls
self.assertEqual(len(calls), 1)
self._assert_isotime(calls[0][1][1], until_local)
def test_oauth2_mastodon_unknown(self):
extr = extractor.find("oauth:mastodon:example.com")
with patch.object(extr, "_oauth2_authorization_code_grant") as m, \
patch.object(extr, "_register") as r:
r.return_value = {
"client-id": "foo",
"client-secret": "bar",
}
for msg in extr:
pass
self.assertEqual(len(r.mock_calls), 1)
self.assertEqual(len(m.mock_calls), 1)
def _prepare_destination(cls, content=None, part=True, extension=None):
name = "file-{}".format(cls.fnum)
cls.fnum += 1
kwdict = {
"category": "test",
"subcategory": "test",
"name": name,
"extension": extension,
}
pathfmt = PathFormat(extractor.find("test:"))
pathfmt.set_directory(kwdict)
pathfmt.set_keywords(kwdict)
if content:
mode = "w" + ("b" if isinstance(content, bytes) else "")
with pathfmt.open(mode) as file:
file.write(content)
return pathfmt
def test_check_cookies_domain(self):
extr = extractor.find("test:")
self.assertFalse(extr._cookiejar, "empty")
extr.cookiedomain = ".example.org"
self.assertFalse(extr._check_cookies(("a",)))
self.assertFalse(extr._check_cookies(("a", "b")))
extr._cookiejar.set("a", "1")
self.assertFalse(extr._check_cookies(("a",)))
extr._cookiejar.set("a", "1", domain=extr.cookiedomain)
self.assertTrue(extr._check_cookies(("a",)))
extr._cookiejar.set("a", "1", domain="www" + extr.cookiedomain)
self.assertEqual(len(extr._cookiejar), 3)
self.assertTrue(extr._check_cookies(("a",)))
extr._cookiejar.set("b", "2", domain=extr.cookiedomain)
extr._cookiejar.set("c", "3", domain=extr.cookiedomain)
self.assertTrue(extr._check_cookies(("a", "b", "c")))
def test_check_cookies(self):
extr = extractor.find("test:")
self.assertFalse(extr._cookiejar, "empty")
self.assertFalse(extr.cookiedomain, "empty")
# always returns False when checking for empty cookie list
self.assertFalse(extr._check_cookies(()))
self.assertFalse(extr._check_cookies(("a",)))
self.assertFalse(extr._check_cookies(("a", "b")))
self.assertFalse(extr._check_cookies(("a", "b", "c")))
extr._cookiejar.set("a", "1")
self.assertTrue(extr._check_cookies(("a",)))
self.assertFalse(extr._check_cookies(("a", "b")))
self.assertFalse(extr._check_cookies(("a", "b", "c")))
extr._cookiejar.set("b", "2")
self.assertTrue(extr._check_cookies(("a",)))
self.assertTrue(extr._check_cookies(("a", "b")))
self.assertFalse(extr._check_cookies(("a", "b", "c")))
def _get_extractor(category):
for msg in extractor.find("test:" + category):
if msg[0] == Message.Queue:
return extractor.find(msg[1])
def test_add(self):
uri = "fake:foobar"
self.assertIsNone(extractor.find(uri))
extractor.add(FakeExtractor)
self.assertIsInstance(extractor.find(uri), FakeExtractor)
def test_dict(self):
cookies = extractor.find("test:").session.cookies
self.assertEqual(len(cookies), len(self.cdict))
self.assertEqual(sorted(cookies.keys()), sorted(self.cdict.keys()))
self.assertEqual(sorted(cookies.values()), sorted(self.cdict.values()))
def test_blacklist(self):
link_uri = "https://example.org/file.jpg"
test_uri = "test:"
fake_uri = "fake:"
self.assertIsInstance(extractor.find(link_uri), DLExtractor)
self.assertIsInstance(extractor.find(test_uri), Extractor)
self.assertIsNone(extractor.find(fake_uri))
with extractor.blacklist(["directlink"]):
self.assertIsNone(extractor.find(link_uri))
self.assertIsInstance(extractor.find(test_uri), Extractor)
self.assertIsNone(extractor.find(fake_uri))
with extractor.blacklist([], [DLExtractor, FakeExtractor]):
self.assertIsNone(extractor.find(link_uri))
self.assertIsInstance(extractor.find(test_uri), Extractor)
self.assertIsNone(extractor.find(fake_uri))
with extractor.blacklist(["test"], [DLExtractor]):
self.assertIsNone(extractor.find(link_uri))
self.assertIsNone(extractor.find(test_uri))
self.assertIsNone(extractor.find(fake_uri))
def __init__(self):
self.extractor = extractor.find("test:")
self.pathfmt = util.PathFormat(self.extractor)
self.out = output.NullOutput()
self.get_logger = logging.getLogger
self.hooks = collections.defaultdict(list)
def test_blacklist(self):
link_uri = "https://example.org/file.jpg"
test_uri = "test:"
fake_uri = "fake:"
self.assertIsInstance(extractor.find(link_uri), DLExtractor)
self.assertIsInstance(extractor.find(test_uri), Extractor)
self.assertIsNone(extractor.find(fake_uri))
with extractor.blacklist(["directlink"]):
self.assertIsNone(extractor.find(link_uri))
self.assertIsInstance(extractor.find(test_uri), Extractor)
self.assertIsNone(extractor.find(fake_uri))
with extractor.blacklist([], [DLExtractor, FakeExtractor]):
self.assertIsNone(extractor.find(link_uri))
self.assertIsInstance(extractor.find(test_uri), Extractor)
self.assertIsNone(extractor.find(fake_uri))
with extractor.blacklist(["test"], [DLExtractor]):
self.assertIsNone(extractor.find(link_uri))
self.assertIsNone(extractor.find(test_uri))
self.assertIsNone(extractor.find(fake_uri))
def __init__(self):
self.extractor = extractor.find("test:")
self.pathfmt = path.PathFormat(self.extractor)
self.out = output.NullOutput()
self.get_logger = logging.getLogger
def setUpClass(cls):
cls.extractor = extractor.find("test:")
cls.extractor.log.job = None
cls.dir = tempfile.TemporaryDirectory()
cls.fnum = 0
config.set((), "base-directory", cls.dir.name)
def test_add(self):
uri = "fake:foobar"
self.assertIsNone(extractor.find(uri))
extractor.add(FakeExtractor)
self.assertIsInstance(extractor.find(uri), FakeExtractor)
def test_dict(self):
cookies = extractor.find("test:").session.cookies
self.assertEqual(len(cookies), len(self.cdict))
self.assertEqual(sorted(cookies.keys()), sorted(self.cdict.keys()))
self.assertEqual(sorted(cookies.values()), sorted(self.cdict.values()))
def _get_extractor(category):
for msg in extractor.find("test:" + category):
if msg[0] == Message.Queue:
return extractor.find(msg[1])
def setUpClass(cls):
cls.extractor = extractor.find("test:")
cls.dir = tempfile.TemporaryDirectory()
config.set(("base-directory", ), cls.dir.name)
def setUpClass(cls):
cls.extractor = extractor.find("test:")
cls.dir = tempfile.TemporaryDirectory()
cls.fnum = 0
config.set(("base-directory",), cls.dir.name)