def test_xml_element(self):
el = Element("tag", attrib={"key": "value"})
el.text = "test"
assert validate(xml_element("tag"), el).tag == "tag"
assert validate(xml_element(text="test"), el).text == "test"
assert validate(xml_element(attrib={"key": text}), el).attrib == {"key": "value"}
def test_xml_findall(self):
el = Element("parent")
children = [Element("child") for i in range(10)]
for child in children:
el.append(child)
assert validate(xml_findall("child"), el) == children
def test_length(self):
assert validate(length(1), [1, 2, 3]) == [1, 2, 3]
def invalid_length():
validate(length(2), [1])
self.assertRaises(ValueError, invalid_length)
def test_basic(self):
assert validate(1, 1) == 1
assert validate(int, 1) == 1
assert validate(transform(int), "1") == 1
assert validate(text, "abc") == "abc"
assert validate(text, u"日本語") == u"日本語"
assert validate(transform(text), 1) == "1"
assert validate(list, ["a", 1]) == ["a", 1]
assert validate(dict, {"a": 1}) == {"a": 1}
assert validate(lambda n: 0 < n < 5, 3) == 3
def test_basic(self):
assert validate(1, 1) == 1
assert validate(int, 1) == 1
assert validate(transform(int), "1") == 1
assert validate(text, "abc") == "abc"
assert validate(text, u"日本語") == u"日本語"
assert validate(transform(text), 1) == "1"
assert validate(list, ["a", 1]) == ["a", 1]
assert validate(dict, {"a": 1}) == {"a": 1}
assert validate(lambda n: 0 < n < 5, 3) == 3
def _schema_videodetails(cls, data):
schema = validate.Schema(
{"videoDetails": {
"videoId": str,
"author": str,
"title": str,
validate.optional("isLive"): validate.transform(bool),
validate.optional("isLiveContent"): validate.transform(bool),
validate.optional("isLiveDvrEnabled"): validate.transform(bool),
validate.optional("isLowLatencyLiveStream"): validate.transform(bool),
validate.optional("isPrivate"): validate.transform(bool),
}},
validate.get("videoDetails"),
)
videoDetails = validate.validate(schema, data)
log.trace(f"videoDetails = {videoDetails!r}")
return validate.validate(
validate.union_get("videoId", "author", "title", "isLive"),
videoDetails)
def _schema_playabilitystatus(cls, data):
schema = validate.Schema(
{
"playabilityStatus": {
"status": str,
validate.optional("reason"): str
}
}, validate.get("playabilityStatus"),
validate.union_get("status", "reason"))
return validate.validate(schema, data)
def test_xml_xpath(self):
root = Element("root")
foo = Element("foo")
bar = Element("bar")
baz = Element("baz")
root.append(foo)
root.append(bar)
foo.append(baz)
assert validate(xml_xpath("./descendant-or-self::node()"),
root) == [root, foo, baz,
bar], "Returns correct node set"
assert validate(xml_xpath("./child::qux"),
root) is None, "Returns None when node set is empty"
assert validate(
xml_xpath("name(.)"),
root) == "root", "Returns function values instead of node sets"
self.assertRaises(ValueError, validate, xml_xpath("."),
"not an Element")
def test_dict_success(self):
schema = validate.union({
"foo": str,
"bar": self.upper,
validate.optional("baz"): int,
})
assert validate.validate(schema, "value") == {
"foo": "value",
"bar": "VALUE"
}
def test_nested_dict_keys(self):
assert validate({text: {
text: int
}}, {"a": {
"b": 1,
"c": 2
}}) == {
"a": {
"b": 1,
"c": 2
}
}
def test_strict(self):
dictionary = {
("foo", "bar", "baz"): "foo-bar-baz",
"foo": {
"bar": {
"baz": "qux"
}
}
}
assert validate.validate(
validate.get(("foo", "bar", "baz"), strict=True),
dictionary) == "foo-bar-baz"
def _schema_videodetails(cls, data):
schema = validate.Schema(
{"videoDetails": {
"videoId": str,
"author": str,
"title": str,
validate.optional("isLiveContent"): validate.transform(bool)
}},
validate.get("videoDetails"),
validate.union_get("videoId", "author", "title", "isLiveContent")
)
return validate.validate(schema, data)
def _get_token_req_url(self):
token_req_host_re = re.compile(r"""jQuery\.get\s*\(['"]([^'"]+)['"]""")
schema = validate.Schema(
validate.xml_xpath_string(".//script[contains(text(), 'LIVE_URL')]/text()"),
validate.any(None, validate.all(
validate.transform(token_req_host_re.search),
validate.any(None, validate.all(
validate.get(1),
validate.url(),
)),
)),
)
token_req_host = validate.validate(schema, self.page)
log.debug(f"token_req_host={token_req_host}")
token_req_str_re = re.compile(r"""Math\.floor\(Date\.now\(\)\s*/\s*3600000\),\s*['"]([^'"]+)['"]""")
schema = validate.Schema(
validate.xml_xpath_string(".//script[contains(text(), 'LIVE_URL')]/text()"),
validate.any(None, validate.all(
validate.transform(token_req_str_re.search),
validate.any(None, validate.all(
validate.get(1),
str,
)),
)),
)
token_req_str = validate.validate(schema, self.page)
log.debug(f"token_req_str={token_req_str}")
if not token_req_str:
return
date = int(time.time() // 3600)
token_req_token = self.transform_token(token_req_str, date) or self.transform_token(token_req_str, date - 1)
if token_req_host and token_req_token:
return update_qsd(token_req_host, {"rsk": token_req_token})
def _get_live_url(self):
live_url_re = re.compile(r"""LIVE_URL\s*=\s*['"]([^'"]+)['"]""")
schema = validate.Schema(
validate.xml_xpath_string(".//script[contains(text(), 'LIVE_URL')]/text()"),
validate.any(None, validate.all(
validate.transform(live_url_re.search),
validate.any(None, validate.all(
validate.get(1),
validate.url(),
)),
)),
)
live_url = validate.validate(schema, self.page)
log.debug(f"live_url={live_url}")
return live_url
def test_nested(self):
assert validate.validate(
validate.union_get(
("foo", "bar"),
("baz", "qux"),
),
{
"foo": {
"bar": 1
},
"baz": {
"qux": 2
}
},
) == (1, 2)
def _schema_streamingdata(cls, data):
schema = validate.Schema(
{
"streamingData": {
validate.optional("hlsManifestUrl"):
str,
validate.optional("formats"): [
validate.all(
{
"itag":
int,
"qualityLabel":
str,
validate.optional("url"):
validate.url(scheme="http")
}, validate.union_get("url", "qualityLabel"))
],
validate.optional("adaptiveFormats"): [
validate.all(
{
"itag":
int,
"mimeType":
validate.all(
str,
validate.transform(
cls._re_mime_type.search),
validate.union_get("type", "codecs"),
),
validate.optional("url"):
validate.url(scheme="http"),
validate.optional("qualityLabel"):
str
},
validate.union_get("url", "qualityLabel", "itag",
"mimeType"))
]
}
}, validate.get("streamingData"),
validate.union_get("hlsManifestUrl", "formats", "adaptiveFormats"))
hls_manifest, formats, adaptive_formats = validate.validate(
schema, data)
return hls_manifest, formats or [], adaptive_formats or []
def _get_vod_uvid(self, parsed_html):
json_re = re.compile(r"window\.nowPlaying\.setData\(({.*?})\);")
return validate.validate(validate.Schema(
validate.xml_xpath_string(".//script[contains(text(), 'window.nowPlaying.setData')]"),
validate.text,
validate.transform(json_re.search),
validate.any(
None,
validate.all(
validate.get(1),
validate.parse_json(),
{
"id": validate.text,
"series_title": validate.text,
"title": validate.text,
"season": validate.text,
"episode": validate.text,
},
),
),
), parsed_html)
def _schema_videodetails(cls, data):
schema = validate.Schema(
{
"videoDetails": {
"videoId": str,
"author": str,
"title": str,
validate.optional("isLive"): validate.transform(bool),
validate.optional("isLiveContent"): validate.transform(bool),
validate.optional("isLiveDvrEnabled"): validate.transform(bool),
validate.optional("isLowLatencyLiveStream"): validate.transform(bool),
validate.optional("isPrivate"): validate.transform(bool),
},
"microformat": validate.all(
validate.any(
validate.all(
{"playerMicroformatRenderer": dict},
validate.get("playerMicroformatRenderer")
),
validate.all(
{"microformatDataRenderer": dict},
validate.get("microformatDataRenderer")
)
),
{
"category": str
}
)
},
validate.union_get(
("videoDetails", "videoId"),
("videoDetails", "author"),
("microformat", "category"),
("videoDetails", "title"),
("videoDetails", "isLive")
)
)
videoDetails = validate.validate(schema, data)
log.trace(f"videoDetails = {videoDetails!r}")
return videoDetails
def test_xml_xpath_string(self):
root = Element("root")
foo = Element("foo")
bar = Element("bar")
root.set("attr", "")
foo.set("attr", "FOO")
bar.set("attr", "BAR")
root.append(foo)
root.append(bar)
assert validate(xml_xpath_string("./baz"),
root) is None, "Returns None if nothing was found"
assert validate(xml_xpath_string("./@attr"),
root) is None, "Returns None if string is empty"
assert validate(xml_xpath_string("./foo/@attr"),
root) == "FOO", "Returns the attr value of foo"
assert validate(xml_xpath_string("./bar/@attr"),
root) == "BAR", "Returns the attr value of bar"
assert validate(xml_xpath_string("count(./*)"),
root) == "2", "Wraps arbitrary functions"
assert validate(
xml_xpath_string("./*/@attr"),
root) == "FOO", "Returns the first item of a set of nodes"
def test_all(self):
assert validate(all(int, lambda n: 0 < n < 5), 3) == 3
assert validate(all(transform(int), lambda n: 0 < n < 5), 3.33) == 3
def test_startswith(self):
assert validate(startswith("abc"), "abcedf")
def test_attr(self):
el = Element("foo")
el.text = "bar"
assert validate(attr({"text": text}), el).text == "bar"
def test_xml_find(self):
el = Element("parent")
el.append(Element("foo"))
el.append(Element("bar"))
assert validate(xml_find("bar"), el).tag == "bar"
def test_all(self):
assert validate(all(int, lambda n: 0 < n < 5), 3) == 3
assert validate(all(transform(int), lambda n: 0 < n < 5), 3.33) == 3
def invalid_length():
validate(length(2), [1])
def test_dict_keys(self):
assert validate({text: int},
{"a": 1, "b": 2}) == {"a": 1, "b": 2}
assert validate({transform(text): transform(int)},
{1: 3.14, 3.14: 1}) == {"1": 3, "3.14": 1}
def test_list_tuple_set_frozenset(self):
assert validate([int], [1, 2])
assert validate(set([int]), set([1, 2])) == set([1, 2])
assert validate(tuple([int]), tuple([1, 2])) == tuple([1, 2])
def test_dict(self):
assert validate({"key": 5}, {"key": 5}) == {"key": 5}
assert validate({"key": int}, {"key": 5}) == {"key": 5}
assert validate({"n": int, "f": float},
{"n": 5, "f": 3.14}) == {"n": 5, "f": 3.14}
def test_list(self):
assert validate([1, 0], [1, 0, 1, 1]) == [1, 0, 1, 1]
assert validate([1, 0], []) == []
assert validate(all([0, 1], lambda l: len(l) > 2), [0, 1, 0]) == [0, 1, 0]
def test_union(self):
assert validate(union((get("foo"), get("bar"))),
{"foo": "alpha", "bar": "beta"}) == ("alpha", "beta")
def test_any(self):
assert validate(any(int, dict), 5) == 5
assert validate(any(int, dict), {}) == {}
assert validate(any(int), 4) == 4
def test_union(self):
assert validate(union((get("foo"), get("bar"))), {
"foo": "alpha",
"bar": "beta"
}) == ("alpha", "beta")
def test_nested_dict_keys(self):
assert validate({text: {text: int}},
{"a": {"b": 1, "c": 2}}) == {"a": {"b": 1, "c": 2}}
def test_list_tuple_set_frozenset(self):
assert validate([int], [1, 2])
assert validate(set([int]), set([1, 2])) == set([1, 2])
assert validate(tuple([int]), tuple([1, 2])) == tuple([1, 2])
def test_get_re(self):
m = re.match(r"(\d+)p", "720p")
assert validate(get(1), m) == "720"
def test_startswith(self):
assert validate(startswith("abc"), "abcedf")
def test_hasattr(self):
el = Element("foo")
assert validate(hasattr("tag"), el) == el
def test_dict_optional_keys(self):
assert validate({"a": 1, optional("b"): 2}, {"a": 1}) == {"a": 1}
assert validate({"a": 1, optional("b"): 2},
{"a": 1, "b": 2}) == {"a": 1, "b": 2}
def test_endswith(self):
assert validate(endswith(u"åäö"), u"xyzåäö")
def test_getattr(self):
el = Element("foo")
assert validate(getattr("tag"), el) == "foo"
assert validate(getattr("invalid", "default"), el) == "default"
def test_map(self):
assert validate(map(lambda v: v[0]), [(1, 2), (3, 4)]) == [1, 3]
def invalid_length():
validate(length(2), [1])
def test_map_dict(self):
assert validate(map(lambda k, v: (v, k)), {"foo": "bar"}) == {"bar": "foo"}
def test_xml_findtext(self):
el = Element("foo")
el.text = "bar"
assert validate(xml_findtext("."), el) == "bar"
def test_get(self):
assert validate(get("key"), {"key": "value"}) == "value"
assert validate(get("invalidkey", "default"), {"key": "value"}) == "default"
def test_url(self):
url_ = "https://google.se/path"
assert validate(url(), url_)
assert validate(url(scheme="http"), url_)
assert validate(url(path="/path"), url_)
def test_get_re(self):
m = re.match("(\d+)p", "720p")
assert validate(get(1), m) == "720"
def test_endswith(self):
assert validate(endswith("åäö"), "xyzåäö")
def test_getattr(self):
el = Element("foo")
assert validate(getattr("tag"), el) == "foo"
assert validate(getattr("invalid", "default"), el) == "default"
def test_any(self):
assert validate(any(int, dict), 5) == 5
assert validate(any(int, dict), {}) == {}
assert validate(any(int), 4) == 4
def test_attr(self):
el = Element("foo")
el.text = "bar"
assert validate(attr({"text": text}), el).text == "bar"
def test_list(self):
assert validate([1, 0], [1, 0, 1, 1]) == [1, 0, 1, 1]
assert validate([1, 0], []) == []
assert validate(all([0, 1], lambda l: len(l) > 2),
[0, 1, 0]) == [0, 1, 0]
def test_xml_findtext(self):
el = Element("foo")
el.text = "bar"
assert validate(xml_findtext("."), el) == "bar"
def test_hasattr(self):
el = Element("foo")
assert validate(hasattr("tag"), el) == el
def test_xml_find(self):
el = Element("parent")
el.append(Element("foo"))
el.append(Element("bar"))
assert validate(xml_find("bar"), el).tag == "bar"
def test_filter(self):
assert validate(filter(lambda i: i > 5),
[10, 5, 4, 6, 7]) == [10, 6, 7]
def test_url(self):
url_ = "https://google.se/path"
assert validate(url(), url_)
assert validate(url(scheme="http"), url_)
assert validate(url(path="/path"), url_)
