def test_xml_element(self): el = Element("tag", attrib={"key": "value"}) el.text = "test" assert validate(xml_element("tag"), el).tag == "tag" assert validate(xml_element(text="test"), el).text == "test" assert validate(xml_element(attrib={"key": text}), el).attrib == {"key": "value"}
def test_xml_findall(self): el = Element("parent") children = [Element("child") for i in range(10)] for child in children: el.append(child) assert validate(xml_findall("child"), el) == children
def test_length(self): assert validate(length(1), [1, 2, 3]) == [1, 2, 3] def invalid_length(): validate(length(2), [1]) self.assertRaises(ValueError, invalid_length)
def test_basic(self): assert validate(1, 1) == 1 assert validate(int, 1) == 1 assert validate(transform(int), "1") == 1 assert validate(text, "abc") == "abc" assert validate(text, u"日本語") == u"日本語" assert validate(transform(text), 1) == "1" assert validate(list, ["a", 1]) == ["a", 1] assert validate(dict, {"a": 1}) == {"a": 1} assert validate(lambda n: 0 < n < 5, 3) == 3
def test_basic(self): assert validate(1, 1) == 1 assert validate(int, 1) == 1 assert validate(transform(int), "1") == 1 assert validate(text, "abc") == "abc" assert validate(text, u"日本語") == u"日本語" assert validate(transform(text), 1) == "1" assert validate(list, ["a", 1]) == ["a", 1] assert validate(dict, {"a": 1}) == {"a": 1} assert validate(lambda n: 0 < n < 5, 3) == 3
def _schema_videodetails(cls, data): schema = validate.Schema( {"videoDetails": { "videoId": str, "author": str, "title": str, validate.optional("isLive"): validate.transform(bool), validate.optional("isLiveContent"): validate.transform(bool), validate.optional("isLiveDvrEnabled"): validate.transform(bool), validate.optional("isLowLatencyLiveStream"): validate.transform(bool), validate.optional("isPrivate"): validate.transform(bool), }}, validate.get("videoDetails"), ) videoDetails = validate.validate(schema, data) log.trace(f"videoDetails = {videoDetails!r}") return validate.validate( validate.union_get("videoId", "author", "title", "isLive"), videoDetails)
def _schema_playabilitystatus(cls, data): schema = validate.Schema( { "playabilityStatus": { "status": str, validate.optional("reason"): str } }, validate.get("playabilityStatus"), validate.union_get("status", "reason")) return validate.validate(schema, data)
def test_xml_xpath(self): root = Element("root") foo = Element("foo") bar = Element("bar") baz = Element("baz") root.append(foo) root.append(bar) foo.append(baz) assert validate(xml_xpath("./descendant-or-self::node()"), root) == [root, foo, baz, bar], "Returns correct node set" assert validate(xml_xpath("./child::qux"), root) is None, "Returns None when node set is empty" assert validate( xml_xpath("name(.)"), root) == "root", "Returns function values instead of node sets" self.assertRaises(ValueError, validate, xml_xpath("."), "not an Element")
def test_dict_success(self): schema = validate.union({ "foo": str, "bar": self.upper, validate.optional("baz"): int, }) assert validate.validate(schema, "value") == { "foo": "value", "bar": "VALUE" }
def test_nested_dict_keys(self): assert validate({text: { text: int }}, {"a": { "b": 1, "c": 2 }}) == { "a": { "b": 1, "c": 2 } }
def test_strict(self): dictionary = { ("foo", "bar", "baz"): "foo-bar-baz", "foo": { "bar": { "baz": "qux" } } } assert validate.validate( validate.get(("foo", "bar", "baz"), strict=True), dictionary) == "foo-bar-baz"
def _schema_videodetails(cls, data): schema = validate.Schema( {"videoDetails": { "videoId": str, "author": str, "title": str, validate.optional("isLiveContent"): validate.transform(bool) }}, validate.get("videoDetails"), validate.union_get("videoId", "author", "title", "isLiveContent") ) return validate.validate(schema, data)
def _get_token_req_url(self): token_req_host_re = re.compile(r"""jQuery\.get\s*\(['"]([^'"]+)['"]""") schema = validate.Schema( validate.xml_xpath_string(".//script[contains(text(), 'LIVE_URL')]/text()"), validate.any(None, validate.all( validate.transform(token_req_host_re.search), validate.any(None, validate.all( validate.get(1), validate.url(), )), )), ) token_req_host = validate.validate(schema, self.page) log.debug(f"token_req_host={token_req_host}") token_req_str_re = re.compile(r"""Math\.floor\(Date\.now\(\)\s*/\s*3600000\),\s*['"]([^'"]+)['"]""") schema = validate.Schema( validate.xml_xpath_string(".//script[contains(text(), 'LIVE_URL')]/text()"), validate.any(None, validate.all( validate.transform(token_req_str_re.search), validate.any(None, validate.all( validate.get(1), str, )), )), ) token_req_str = validate.validate(schema, self.page) log.debug(f"token_req_str={token_req_str}") if not token_req_str: return date = int(time.time() // 3600) token_req_token = self.transform_token(token_req_str, date) or self.transform_token(token_req_str, date - 1) if token_req_host and token_req_token: return update_qsd(token_req_host, {"rsk": token_req_token})
def _get_live_url(self): live_url_re = re.compile(r"""LIVE_URL\s*=\s*['"]([^'"]+)['"]""") schema = validate.Schema( validate.xml_xpath_string(".//script[contains(text(), 'LIVE_URL')]/text()"), validate.any(None, validate.all( validate.transform(live_url_re.search), validate.any(None, validate.all( validate.get(1), validate.url(), )), )), ) live_url = validate.validate(schema, self.page) log.debug(f"live_url={live_url}") return live_url
def test_nested(self): assert validate.validate( validate.union_get( ("foo", "bar"), ("baz", "qux"), ), { "foo": { "bar": 1 }, "baz": { "qux": 2 } }, ) == (1, 2)
def _schema_streamingdata(cls, data): schema = validate.Schema( { "streamingData": { validate.optional("hlsManifestUrl"): str, validate.optional("formats"): [ validate.all( { "itag": int, "qualityLabel": str, validate.optional("url"): validate.url(scheme="http") }, validate.union_get("url", "qualityLabel")) ], validate.optional("adaptiveFormats"): [ validate.all( { "itag": int, "mimeType": validate.all( str, validate.transform( cls._re_mime_type.search), validate.union_get("type", "codecs"), ), validate.optional("url"): validate.url(scheme="http"), validate.optional("qualityLabel"): str }, validate.union_get("url", "qualityLabel", "itag", "mimeType")) ] } }, validate.get("streamingData"), validate.union_get("hlsManifestUrl", "formats", "adaptiveFormats")) hls_manifest, formats, adaptive_formats = validate.validate( schema, data) return hls_manifest, formats or [], adaptive_formats or []
def _get_vod_uvid(self, parsed_html): json_re = re.compile(r"window\.nowPlaying\.setData\(({.*?})\);") return validate.validate(validate.Schema( validate.xml_xpath_string(".//script[contains(text(), 'window.nowPlaying.setData')]"), validate.text, validate.transform(json_re.search), validate.any( None, validate.all( validate.get(1), validate.parse_json(), { "id": validate.text, "series_title": validate.text, "title": validate.text, "season": validate.text, "episode": validate.text, }, ), ), ), parsed_html)
def _schema_videodetails(cls, data): schema = validate.Schema( { "videoDetails": { "videoId": str, "author": str, "title": str, validate.optional("isLive"): validate.transform(bool), validate.optional("isLiveContent"): validate.transform(bool), validate.optional("isLiveDvrEnabled"): validate.transform(bool), validate.optional("isLowLatencyLiveStream"): validate.transform(bool), validate.optional("isPrivate"): validate.transform(bool), }, "microformat": validate.all( validate.any( validate.all( {"playerMicroformatRenderer": dict}, validate.get("playerMicroformatRenderer") ), validate.all( {"microformatDataRenderer": dict}, validate.get("microformatDataRenderer") ) ), { "category": str } ) }, validate.union_get( ("videoDetails", "videoId"), ("videoDetails", "author"), ("microformat", "category"), ("videoDetails", "title"), ("videoDetails", "isLive") ) ) videoDetails = validate.validate(schema, data) log.trace(f"videoDetails = {videoDetails!r}") return videoDetails
def test_xml_xpath_string(self): root = Element("root") foo = Element("foo") bar = Element("bar") root.set("attr", "") foo.set("attr", "FOO") bar.set("attr", "BAR") root.append(foo) root.append(bar) assert validate(xml_xpath_string("./baz"), root) is None, "Returns None if nothing was found" assert validate(xml_xpath_string("./@attr"), root) is None, "Returns None if string is empty" assert validate(xml_xpath_string("./foo/@attr"), root) == "FOO", "Returns the attr value of foo" assert validate(xml_xpath_string("./bar/@attr"), root) == "BAR", "Returns the attr value of bar" assert validate(xml_xpath_string("count(./*)"), root) == "2", "Wraps arbitrary functions" assert validate( xml_xpath_string("./*/@attr"), root) == "FOO", "Returns the first item of a set of nodes"
def test_all(self): assert validate(all(int, lambda n: 0 < n < 5), 3) == 3 assert validate(all(transform(int), lambda n: 0 < n < 5), 3.33) == 3
def test_startswith(self): assert validate(startswith("abc"), "abcedf")
def test_attr(self): el = Element("foo") el.text = "bar" assert validate(attr({"text": text}), el).text == "bar"
def test_xml_find(self): el = Element("parent") el.append(Element("foo")) el.append(Element("bar")) assert validate(xml_find("bar"), el).tag == "bar"
def test_all(self): assert validate(all(int, lambda n: 0 < n < 5), 3) == 3 assert validate(all(transform(int), lambda n: 0 < n < 5), 3.33) == 3
def invalid_length(): validate(length(2), [1])
def test_dict_keys(self): assert validate({text: int}, {"a": 1, "b": 2}) == {"a": 1, "b": 2} assert validate({transform(text): transform(int)}, {1: 3.14, 3.14: 1}) == {"1": 3, "3.14": 1}
def test_list_tuple_set_frozenset(self): assert validate([int], [1, 2]) assert validate(set([int]), set([1, 2])) == set([1, 2]) assert validate(tuple([int]), tuple([1, 2])) == tuple([1, 2])
def test_dict(self): assert validate({"key": 5}, {"key": 5}) == {"key": 5} assert validate({"key": int}, {"key": 5}) == {"key": 5} assert validate({"n": int, "f": float}, {"n": 5, "f": 3.14}) == {"n": 5, "f": 3.14}
def test_list(self): assert validate([1, 0], [1, 0, 1, 1]) == [1, 0, 1, 1] assert validate([1, 0], []) == [] assert validate(all([0, 1], lambda l: len(l) > 2), [0, 1, 0]) == [0, 1, 0]
def test_union(self): assert validate(union((get("foo"), get("bar"))), {"foo": "alpha", "bar": "beta"}) == ("alpha", "beta")
def test_any(self): assert validate(any(int, dict), 5) == 5 assert validate(any(int, dict), {}) == {} assert validate(any(int), 4) == 4
def test_union(self): assert validate(union((get("foo"), get("bar"))), { "foo": "alpha", "bar": "beta" }) == ("alpha", "beta")
def test_nested_dict_keys(self): assert validate({text: {text: int}}, {"a": {"b": 1, "c": 2}}) == {"a": {"b": 1, "c": 2}}
def test_list_tuple_set_frozenset(self): assert validate([int], [1, 2]) assert validate(set([int]), set([1, 2])) == set([1, 2]) assert validate(tuple([int]), tuple([1, 2])) == tuple([1, 2])
def test_get_re(self): m = re.match(r"(\d+)p", "720p") assert validate(get(1), m) == "720"
def test_startswith(self): assert validate(startswith("abc"), "abcedf")
def test_hasattr(self): el = Element("foo") assert validate(hasattr("tag"), el) == el
def test_dict_optional_keys(self): assert validate({"a": 1, optional("b"): 2}, {"a": 1}) == {"a": 1} assert validate({"a": 1, optional("b"): 2}, {"a": 1, "b": 2}) == {"a": 1, "b": 2}
def test_endswith(self): assert validate(endswith(u"åäö"), u"xyzåäö")
def test_getattr(self): el = Element("foo") assert validate(getattr("tag"), el) == "foo" assert validate(getattr("invalid", "default"), el) == "default"
def test_map(self): assert validate(map(lambda v: v[0]), [(1, 2), (3, 4)]) == [1, 3]
def invalid_length(): validate(length(2), [1])
def test_map_dict(self): assert validate(map(lambda k, v: (v, k)), {"foo": "bar"}) == {"bar": "foo"}
def test_xml_findtext(self): el = Element("foo") el.text = "bar" assert validate(xml_findtext("."), el) == "bar"
def test_get(self): assert validate(get("key"), {"key": "value"}) == "value" assert validate(get("invalidkey", "default"), {"key": "value"}) == "default"
def test_url(self): url_ = "https://google.se/path" assert validate(url(), url_) assert validate(url(scheme="http"), url_) assert validate(url(path="/path"), url_)
def test_get_re(self): m = re.match("(\d+)p", "720p") assert validate(get(1), m) == "720"
def test_endswith(self): assert validate(endswith("åäö"), "xyzåäö")
def test_getattr(self): el = Element("foo") assert validate(getattr("tag"), el) == "foo" assert validate(getattr("invalid", "default"), el) == "default"
def test_any(self): assert validate(any(int, dict), 5) == 5 assert validate(any(int, dict), {}) == {} assert validate(any(int), 4) == 4
def test_attr(self): el = Element("foo") el.text = "bar" assert validate(attr({"text": text}), el).text == "bar"
def test_list(self): assert validate([1, 0], [1, 0, 1, 1]) == [1, 0, 1, 1] assert validate([1, 0], []) == [] assert validate(all([0, 1], lambda l: len(l) > 2), [0, 1, 0]) == [0, 1, 0]
def test_xml_findtext(self): el = Element("foo") el.text = "bar" assert validate(xml_findtext("."), el) == "bar"
def test_hasattr(self): el = Element("foo") assert validate(hasattr("tag"), el) == el
def test_xml_find(self): el = Element("parent") el.append(Element("foo")) el.append(Element("bar")) assert validate(xml_find("bar"), el).tag == "bar"
def test_filter(self): assert validate(filter(lambda i: i > 5), [10, 5, 4, 6, 7]) == [10, 6, 7]
def test_url(self): url_ = "https://google.se/path" assert validate(url(), url_) assert validate(url(scheme="http"), url_) assert validate(url(path="/path"), url_)