def apply_first_condition(res: Response, req: Request, conditions: TList[Condition]) -> Response:
condition: TOption[Condition] = conditions.find(
lambda c: when_optional_filter(c.when, {'req': req.to_dict(), 'res': res.to_dict()})
)
if condition.is_none():
return res
return Response.from_dict(
{
"body": res.body,
"type": condition.get().type,
"encoding": res.encoding.get(),
"headers": res.headers,
"url": res.url,
"status_code": res.status_code,
"elapsed": res.elapsed,
"elapsed_sec": res.elapsed_sec,
},
)
def test_from_requests_decide_encoding(self, title, headers, text, content,
encoding, apparent_encoding, default_encoding, expected):
Requests = namedtuple('Requests', ('headers', 'text', 'content', 'encoding', 'apparent_encoding'))
actual = Response._decide_encoding(
Requests(headers, text, content, encoding, apparent_encoding),
TOption(default_encoding)
)
assert actual == expected
def test_from_requests_decide_encoding(self, title, headers, text, content,
encoding, apparent_encoding,
default_encoding, expected):
Requests = namedtuple(
'Requests',
('headers', 'text', 'content', 'encoding', 'apparent_encoding'))
actual = Response._decide_encoding(
Requests(headers, text, content, encoding, apparent_encoding),
TOption(default_encoding))
assert actual == expected
def make_response(type: str, status_code: int) -> Response:
return Response.from_dict({
"body": b'{"body": true}',
"type": type,
"encoding": "utf-8",
"headers": {
"content-type": "application/json"
},
"url": "http://test",
"status_code": status_code,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
})
def make_response(text: str, encoding: str, body_encoding: str) -> Response:
return Response.from_dict({
"body": text.encode(body_encoding),
"type": "json",
"encoding": encoding,
"headers": {
"content-type": "application/json"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
})
def apply_first_condition(res: Response, req: Request,
conditions: TList[Condition]) -> Response:
# TODO: remove TOption (owlmixin... find)
condition: TOption[Condition] = TOption(
conditions.find(lambda c: when_optional_filter(c.when, {
'req': req.to_dict(),
'res': res.to_dict()
})))
if condition.is_none():
return res
return Response.from_dict(
{
"body": res.body,
"type": condition.get().type,
"encoding": res.encoding.get(),
"headers": res.headers,
"url": res.url,
"status_code": res.status_code,
"elapsed": res.elapsed,
"elapsed_sec": res.elapsed_sec,
}, )
def make_response(type: str, status_code: int) -> Response:
return Response.from_dict({
"body": b'{"body": true}',
"type": type,
"encoding": "utf-8",
"headers": {
"content-type": "application/json"
},
"url": "http://test",
"status_code": status_code,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
})
def make_response(text: str, encoding: str, body_encoding: str) -> Response:
return Response.from_dict({
"body": text.encode(body_encoding),
"type": "json",
"encoding": encoding,
"headers": {
"content-type": "application/json"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
})
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import datetime
import pytest
from owlmixin.util import load_yaml
from jumeaux.addons.judgement.same import Executor
from jumeaux.models import JudgementAddOnPayload, Response, CaseInsensitiveDict, JudgementAddOnReference
EMPTY_KEYS = {'changed': [], 'added': [], 'removed': []}
RES_ONE = Response.from_dict({
'body': b'a',
"type": "unknown",
'headers': CaseInsensitiveDict({}),
'url': 'url',
'status_code': 200,
'elapsed': datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
})
RES_OTHER = Response.from_dict({
'body': b'b',
"type": "unknown",
'headers': CaseInsensitiveDict({}),
'url': 'url',
'status_code': 200,
'elapsed': datetime.timedelta(seconds=2),
"elapsed_sec": 2.0,
})
{
'title': 'Check point 2',
'conditions': [
{
'added': ['<add><99>']
}
]
}
]
}
RES_ONE = Response.from_dict({
'body': b'a',
"type": "unknown",
'headers': CaseInsensitiveDict({}),
'url': 'url',
'status_code': 200,
'elapsed': datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
})
RES_OTHER = Response.from_dict({
'body': b'b',
"type": "unknown",
'headers': CaseInsensitiveDict({}),
'url': 'url',
'status_code': 200,
'elapsed': datetime.timedelta(seconds=2),
"elapsed_sec": 2.0,
})
<span>Main contents</span>
</div>
</body>
</html>
"""
NORMAL_CASE = ("Normal",
"""
force: False
""",
Response.from_dict({
"body": NORMAL_BODY.encode('utf8'),
"type": "html",
"encoding": 'utf8',
"headers": {
"content-type": "text/html"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
}),
{
"html": {
"head": {
"title": {
"##value": "タイトル"
},
"meta": [
{
"#name": "format-detection",
"#content": "telephone=no",
<author>次郎</author>
<title>次郎22</title>
</book>
</catalog>
"""
NORMAL_CASE = ("Normal",
"""
force: False
""",
Response.from_dict({
"body": NORMAL_BODY.encode('euc-jp'),
"type": "xml",
"encoding": 'euc-jp',
"headers": {
"content-type": "application/xml"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
}),
{
"catalog": {
"book": [
{"@id": "bk001", "author": "Ichiro", "title": "Ichiro55"},
{"@id": "bk002", "author": "次郎", "title": "次郎22"}
]
}
}
)
<div id="main">
<span>Main contents</span>
</div>
</body>
</html>
"""
NORMAL_CASE = ("Normal", """
force: False
""",
Response.from_dict({
"body": NORMAL_BODY.encode('utf8'),
"type": "html",
"encoding": 'utf8',
"headers": {
"content-type": "text/html"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
}), {
"html": {
"head": {
"title": {
"##value": "タイトル"
},
"meta": [{
"#name": "format-detection",
"#content": "telephone=no",
"##value": ""
}, {
}]
},
ensure_ascii=False)
CORRUPTION_BODY_BYTES: bytes = '{"normal": "次郎", '.encode(
'euc-jp') + '"corruption": "三郎"}'.encode('utf8')
NORMAL_CASE = ("Normal", """
force: False
""",
Response.from_dict({
"body": NORMAL_BODY.encode('euc-jp'),
"type": "json",
"encoding": 'euc-jp',
"headers": {
"content-type": "application/json; charset=euc-jp"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
}), NORMAL_BODY.encode('euc-jp'), 'euc-jp', """
{
"items": [
{
"favorites": [
"apple",
"orange"
],
"id": 1,
"name": "Ichiro"
},
def challenge(arg: ChallengeArg) -> dict:
""" Response is dict like `Trial` because Status(OwlEnum) can't be pickled.
"""
name: str = arg.req.name.get_or(str(arg.seq))
log_prefix = f"[{arg.seq} / {arg.number_of_request}]"
logger.info_lv3(f"{log_prefix} {'-'*80}")
logger.info_lv3(f"{log_prefix} {arg.seq}. {arg.req.name.get_or(arg.req.path)}")
logger.info_lv3(f"{log_prefix} {'-'*80}")
path_str_one = arg.path_one.map(lambda x: re.sub(x.before, x.after, arg.req.path)).get_or(arg.req.path)
path_str_other = arg.path_other.map(lambda x: re.sub(x.before, x.after, arg.req.path)).get_or(arg.req.path)
qs_str_one = create_query_string(arg.req.qs, arg.query_one, arg.req.url_encoding)
qs_str_other = create_query_string(arg.req.qs, arg.query_other, arg.req.url_encoding)
url_one = f'{arg.host_one}{path_str_one}?{qs_str_one}'
url_other = f'{arg.host_other}{path_str_other}?{qs_str_other}'
# Get two responses
req_time = now()
try:
logger.info_lv3(f"{log_prefix} One URL: {url_one}")
logger.debug(f"{log_prefix} One PROXY: {arg.proxy_one.map(lambda x: x.to_dict()).get()}")
logger.info_lv3(f"{log_prefix} Other URL: {url_other}")
logger.debug(f"{log_prefix} Other PROXY: {arg.proxy_other.map(lambda x: x.to_dict()).get()}")
r_one, r_other = concurrent_request(arg.session, arg.req.headers,
url_one, url_other,
arg.proxy_one, arg.proxy_other)
logger.info_lv3(
f"{log_prefix} One: {r_one.status_code} / {to_sec(r_one.elapsed)}s / {len(r_one.content)}b / {r_one.headers.get('content-type')}" # noqa
)
logger.info_lv3(
f"{log_prefix} Other: {r_other.status_code} / {to_sec(r_other.elapsed)}s / {len(r_other.content)}b / {r_other.headers.get('content-type')}" # noqa
)
except ConnectionError:
logger.info_lv1(f"{log_prefix} 💀 {arg.req.name.get()}")
# TODO: Integrate logic into create_trial
return {
"seq": arg.seq,
"name": name,
"tags": [],
"request_time": req_time.isoformat(),
"status": 'failure',
"path": arg.req.path,
"queries": arg.req.qs,
"headers": arg.req.headers,
"one": {
"url": url_one,
"type": "unknown",
},
"other": {
"url": url_other,
"type": "unknown",
}
}
res_one: Response = res2res(Response.from_requests(r_one, arg.default_response_encoding_one), arg.req)
res_other: Response = res2res(Response.from_requests(r_other, arg.default_response_encoding_other), arg.req)
dict_one: TOption[dict] = res2dict(res_one)
dict_other: TOption[dict] = res2dict(res_other)
# Create diff
# Either dict_one or dic_other is None, it means that it can't be analyzed, therefore return None
ddiff = None if dict_one.is_none() or dict_other.is_none() \
else {} if res_one.body == res_other.body \
else DeepDiff(dict_one.get(), dict_other.get())
initial_diffs_by_cognition: Optional[TDict[DiffKeys]] = TDict({
"unknown": DiffKeys.from_dict({
"changed": TList(ddiff.get('type_changes', {}).keys() | ddiff.get('values_changed', {}).keys())
.map(to_jumeaux_xpath)
.order_by(_),
"added": TList(ddiff.get('dictionary_item_added', {}) | ddiff.get('iterable_item_added', {}).keys())
.map(to_jumeaux_xpath)
.order_by(_),
"removed": TList(ddiff.get('dictionary_item_removed', {}) | ddiff.get('iterable_item_removed', {}).keys())
.map(to_jumeaux_xpath)
.order_by(_)
})
}) if ddiff is not None else None
# Judgement
status, diffs_by_cognition = judgement(res_one, res_other, dict_one, dict_other,
name, arg.req.path, arg.req.qs, arg.req.headers, initial_diffs_by_cognition)
status_symbol = "O" if status == Status.SAME else "X"
log_msg = f"{log_prefix} {status_symbol} ({res_one.status_code} - {res_other.status_code}) <{res_one.elapsed_sec}s - {res_other.elapsed_sec}s> {arg.req.name.get_or(arg.req.path)}" # noqa
(logger.info_lv2 if status == Status.SAME else logger.info_lv1)(log_msg)
file_one: Optional[str] = None
file_other: Optional[str] = None
prop_file_one: Optional[str] = None
prop_file_other: Optional[str] = None
if store_criterion(status, name, arg.req, res_one, res_other):
dir = f'{arg.res_dir}/{arg.key}'
file_one = f'one/({arg.seq}){name}'
file_other = f'other/({arg.seq}){name}'
write_to_file(file_one, dir, dump(res_one))
write_to_file(file_other, dir, dump(res_other))
if not dict_one.is_none():
prop_file_one = f'one-props/({arg.seq}){name}.json'
write_to_file(prop_file_one, dir, TDict(dict_one.get()).to_json().encode('utf-8', errors='replace'))
if not dict_other.is_none():
prop_file_other = f'other-props/({arg.seq}){name}.json'
write_to_file(prop_file_other, dir, TDict(dict_other.get()).to_json().encode('utf-8', errors='replace'))
return global_addon_executor.apply_did_challenge(
DidChallengeAddOnPayload.from_dict({
"trial": Trial.from_dict({
"seq": arg.seq,
"name": name,
"tags": [], # TODO: tags created by reqs2reqs
"request_time": req_time.isoformat(),
"status": status,
"path": arg.req.path,
"queries": arg.req.qs,
"headers": arg.req.headers,
"diffs_by_cognition": diffs_by_cognition,
"one": {
"url": res_one.url,
"type": res_one.type,
"status_code": res_one.status_code,
"byte": res_one.byte,
"response_sec": res_one.elapsed_sec,
"content_type": res_one.content_type,
"mime_type": res_one.mime_type,
"encoding": res_one.encoding,
"file": file_one,
"prop_file": prop_file_one,
},
"other": {
"url": res_other.url,
"type": res_other.type,
"status_code": res_other.status_code,
"byte": res_other.byte,
"response_sec": res_other.elapsed_sec,
"content_type": res_other.content_type,
"mime_type": res_other.mime_type,
"encoding": res_other.encoding,
"file": file_other,
"prop_file": prop_file_other,
}
})
}),
DidChallengeAddOnReference.from_dict({
"res_one": res_one,
"res_other": res_other,
"res_one_props": dict_one,
"res_other_props": dict_other,
})
).trial.to_dict()
[Module2 alpha]
Name: Jumeaux Viewer
Version: 1.0.0 (r1585)
""".strip()
PATTERN1 = ("Normal", """
force: False
header_regexp: '\\[(.+)\\]'
record_regexp: '([^:]+): (.+)'
""",
Response.from_dict({
"body": PATTERN1_BODY.encode('utf-8'),
"type": "plain",
"encoding": 'utf-8',
"headers": {
"content-type": "text/plain; charset=utf-8"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
}), {
"Module1": {
"Name": "Jumeaux",
"License": "MIT",
"Version": "0.33.0"
},
"Module2 alpha": {
"Name": "Jumeaux Viewer",
"Version": "1.0.0 (r: 1585:1586)"
}
})
""".replace(os.linesep, '').replace(' ', '')
CORRUPTION_BODY_BYTES: bytes = \
'<!DOCTYPE html><html><head><title>タイトル</title></head>'.encode('utf8') + \
'<body><div>コンテンツ</div></body></html>'.encode('euc-jp')
NORMAL_CASE = ("Normal",
"""
force: False
""",
Response.from_dict({
"body": NORMAL_BODY.encode('euc-jp'),
"type": "html",
"encoding": 'euc-jp',
"headers": {
"content-type": "text/html; charset=euc-jp"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
}),
NORMAL_BODY.encode('euc-jp'),
'euc-jp',
"""<html>
<head>
<title>
タイトル
</title>
<meta content="telephone=no" name="format-detection"/>
<meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>
<script charset="utf-8" type="text/javascript">
Name: Jumeaux Viewer
Version: 1.0.0 (r1585)
""".strip()
PATTERN1 = ("Normal",
"""
force: False
header_regexp: '\\[(.+)\\]'
record_regexp: '([^:]+): (.+)'
""",
Response.from_dict({
"body": PATTERN1_BODY.encode('utf-8'),
"type": "plain",
"encoding": 'utf-8',
"headers": {
"content-type": "text/plain; charset=utf-8"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
}),
{
"Module1": {
"Name": "Jumeaux",
"License": "MIT",
"Version": "0.33.0"
},
"Module2 alpha": {
"Name": "Jumeaux Viewer",
"Version": "1.0.0 (r: 1585:1586)"
}
def challenge(arg: ChallengeArg) -> dict:
""" Response is dict like `Trial` because Status(OwlEnum) can't be pickled.
"""
name: str = arg.req.name.get_or(str(arg.seq))
log_prefix = f"[{arg.seq} / {arg.number_of_request}]"
logger.info_lv3(f"{log_prefix} {'-'*80}")
logger.info_lv3(
f"{log_prefix} {arg.seq}. {arg.req.name.get_or(arg.req.path)}")
logger.info_lv3(f"{log_prefix} {'-'*80}")
path_str_one = arg.path_one.map(
lambda x: re.sub(x.before, x.after, arg.req.path)).get_or(arg.req.path)
path_str_other = arg.path_other.map(
lambda x: re.sub(x.before, x.after, arg.req.path)).get_or(arg.req.path)
qs_str_one = create_query_string(arg.req.qs, arg.query_one,
arg.req.url_encoding)
qs_str_other = create_query_string(arg.req.qs, arg.query_other,
arg.req.url_encoding)
url_one = f'{arg.host_one}{path_str_one}?{qs_str_one}'
url_other = f'{arg.host_other}{path_str_other}?{qs_str_other}'
# Get two responses
req_time = now()
try:
logger.info_lv3(f"{log_prefix} One URL: {url_one}")
logger.debug(
f"{log_prefix} One PROXY: {arg.proxy_one.map(lambda x: x.to_dict()).get()}"
)
logger.info_lv3(f"{log_prefix} Other URL: {url_other}")
logger.debug(
f"{log_prefix} Other PROXY: {arg.proxy_other.map(lambda x: x.to_dict()).get()}"
)
r_one, r_other = concurrent_request(arg.session, arg.req.headers,
url_one, url_other, arg.proxy_one,
arg.proxy_other)
logger.info_lv3(
f"{log_prefix} One: {r_one.status_code} / {to_sec(r_one.elapsed)}s / {len(r_one.content)}b / {r_one.headers.get('content-type')}" # noqa
)
logger.info_lv3(
f"{log_prefix} Other: {r_other.status_code} / {to_sec(r_other.elapsed)}s / {len(r_other.content)}b / {r_other.headers.get('content-type')}" # noqa
)
except ConnectionError:
logger.info_lv1(f"{log_prefix} 💀 {arg.req.name.get()}")
# TODO: Integrate logic into create_trial
return {
"seq": arg.seq,
"name": name,
"tags": [],
"request_time": req_time.isoformat(),
"status": 'failure',
"path": arg.req.path,
"queries": arg.req.qs,
"headers": arg.req.headers,
"one": {
"url": url_one,
"type": "unknown",
},
"other": {
"url": url_other,
"type": "unknown",
}
}
res_one: Response = res2res(
Response.from_requests(r_one, arg.default_response_encoding_one),
arg.req)
res_other: Response = res2res(
Response.from_requests(r_other, arg.default_response_encoding_other),
arg.req)
dict_one: TOption[dict] = res2dict(res_one)
dict_other: TOption[dict] = res2dict(res_other)
# Create diff
# Either dict_one or dic_other is None, it means that it can't be analyzed, therefore return None
ddiff = None if dict_one.is_none() or dict_other.is_none() \
else {} if res_one.body == res_other.body \
else DeepDiff(dict_one.get(), dict_other.get())
diff_keys: Optional[DiffKeys] = DiffKeys.from_dict({
"changed":
TList(
ddiff.get('type_changes', {}).keys()
| ddiff.get('values_changed', {}).keys()).map(
lambda x: x.replace('[', '<').replace(']', '>')).order_by(_),
"added":
TList(
ddiff.get('dictionary_item_added', {})
| ddiff.get('iterable_item_added', {}).keys()).map(
lambda x: x.replace('[', '<').replace(']', '>')).order_by(_),
"removed":
TList(
ddiff.get('dictionary_item_removed', {})
| ddiff.get('iterable_item_removed', {}).keys()).map(
lambda x: x.replace('[', '<').replace(']', '>')).order_by(_)
}) if ddiff is not None else None
# Judgement
status: Status = judgement(res_one, res_other, dict_one, dict_other, name,
arg.req.path, arg.req.qs, arg.req.headers,
diff_keys)
status_symbol = "O" if status == Status.SAME else "X"
log_msg = f"{log_prefix} {status_symbol} ({res_one.status_code} - {res_other.status_code}) <{res_one.elapsed_sec}s - {res_other.elapsed_sec}s> {arg.req.name.get_or(arg.req.path)}" # noqa
(logger.info_lv2 if status == Status.SAME else logger.info_lv1)(log_msg)
file_one: str = None
file_other: str = None
prop_file_one: str = None
prop_file_other: str = None
if store_criterion(status, name, arg.req, res_one, res_other):
dir = f'{arg.res_dir}/{arg.key}'
file_one = f'one/({arg.seq}){name}'
file_other = f'other/({arg.seq}){name}'
write_to_file(file_one, dir, dump(res_one))
write_to_file(file_other, dir, dump(res_other))
if not dict_one.is_none():
prop_file_one = f'one-props/({arg.seq}){name}.json'
write_to_file(
prop_file_one, dir,
TDict(dict_one.get()).to_json().encode('utf-8',
errors='replace'))
if not dict_other.is_none():
prop_file_other = f'other-props/({arg.seq}){name}.json'
write_to_file(
prop_file_other, dir,
TDict(dict_other.get()).to_json().encode('utf-8',
errors='replace'))
return global_addon_executor.apply_did_challenge(
DidChallengeAddOnPayload.from_dict({
"trial":
Trial.from_dict({
"seq": arg.seq,
"name": name,
"tags": [], # TODO: tags created by reqs2reqs
"request_time": req_time.isoformat(),
"status": status,
"path": arg.req.path,
"queries": arg.req.qs,
"headers": arg.req.headers,
"diff_keys": diff_keys,
"one": {
"url": res_one.url,
"type": res_one.type,
"status_code": res_one.status_code,
"byte": res_one.byte,
"response_sec": res_one.elapsed_sec,
"content_type": res_one.content_type,
"mime_type": res_one.mime_type,
"encoding": res_one.encoding,
"file": file_one,
"prop_file": prop_file_one,
},
"other": {
"url": res_other.url,
"type": res_other.type,
"status_code": res_other.status_code,
"byte": res_other.byte,
"response_sec": res_other.elapsed_sec,
"content_type": res_other.content_type,
"mime_type": res_other.mime_type,
"encoding": res_other.encoding,
"file": file_other,
"prop_file": prop_file_other,
}
})
}),
DidChallengeAddOnReference.from_dict({
"res_one": res_one,
"res_other": res_other,
})).trial.to_dict()
<book id="bk002">
<author>次郎</author>
<title>次郎22</title>
</book>
</catalog>
"""
NORMAL_CASE = ("Normal", """
force: False
""",
Response.from_dict({
"body": NORMAL_BODY.encode('euc-jp'),
"type": "xml",
"encoding": 'euc-jp',
"headers": {
"content-type": "application/xml"
},
"url": "http://test",
"status_code": 200,
"elapsed": datetime.timedelta(seconds=1),
"elapsed_sec": 1.0,
}), {
"catalog": {
"book": [{
"@id": "bk001",
"author": "Ichiro",
"title": "Ichiro55"
}, {
"@id": "bk002",
"author": "次郎",
"title": "次郎22"
}]