def guess_url_encoding(query_str: str, encodings: TList[str]) -> TOption[str]:
for e in encodings:
try:
urlparser.parse_qs(query_str, encoding=e, errors='strict')
return TOption(e)
except UnicodeDecodeError:
pass
return TOption(None)
def find_permission_members_by_space(
args: Args, config: Config) -> TDict[TOption[TList[str]]]:
global groups_by_member
api = ApiClient(config.base_url, os.environ['USER'],
os.environ['PASSWORD'])
groups_by_member = api.fetch_group() \
.flat_map(lambda g: api.fetch_members(g).map(lambda m: {'member': m, 'group': g})) \
.group_by(_['member'].username) \
.map_values(lambda ds: ds.map(_['group'].name))
return TDict(
api.fetch_spaces() \
.map(lambda s: {'key': s.key, 'name': s.name, 'permissions': api.fetch_space_permissions(s.key)}) \
.group_by(_['name']) \
.map_values(lambda pks: TOption(
pks.flat_map(_["permissions"]) \
.flat_map(_.space_permissions) \
.map(lambda p: p.user_name.get()) \
.uniq() \
.filter(lambda n: groups_by_member.get(n, TList()).any(_ == args.group.get())) \
or None)
)
)
def test_mergecase_with_tags(self, config_with_tags,
config_mergecase_with_tags):
actual: Config = configmaker.create_config(
TList([config_with_tags, config_mergecase_with_tags]),
TOption(TList(['skip'])))
expected = {
"title": "mergecase_with_tags",
"one": {
"name": "name_one",
"host": "http://host/one",
"proxy": "http://proxy"
},
"other": {
"name": "name_other",
"host": "http://host/other"
},
"output": {
"encoding": "utf8",
"response_dir": "mergecase_with_tags"
},
"threads": 1,
"max_retries": 3,
"addons": {
"log2reqs": {
"name": "addons.log2reqs.csv",
"cls_name": "Executor",
"config": {
"encoding": "utf8"
}
},
"reqs2reqs": [{
"name": "addons.reqs2reqs.head",
"cls_name": "Executor",
"tags": ["no-skip"],
"config": {
"size": 2
}
}],
"res2res": [],
"res2dict": [],
"judgement": [],
"store_criterion": [{
"name": "addons.store_criterion.general",
"cls_name": "Executor",
"tags": ["skip2"],
"config": {
"statuses": ["different"]
}
}],
"dump": [],
"did_challenge": [],
"final": []
}
}
assert actual.to_dict() == expected
def test_includecase1(self, config_only_access_points,
config_includecase_1):
actual: Config = configmaker.create_config(
TList([config_only_access_points, config_includecase_1]),
TOption(None))
expected = {
"title": 'includecase_1',
"one": {
"name": "name_one",
"host": "http://host/one",
"proxy": "http://proxy"
},
"other": {
"name": "name_other",
"host": "http://host/other"
},
"output": {
"encoding": "utf8",
"response_dir": "includecase1"
},
"threads": 1,
"max_retries": 3,
"addons": {
"log2reqs": {
"name": "addons.log2reqs.csv",
"cls_name": "Executor",
"config": {
"encoding": "utf8"
}
},
"reqs2reqs": [{
"name": "addons.reqs2reqs.head",
"cls_name": "Executor",
"config": {
"size": 999
}
}, {
"name": "addons.reqs2reqs.head",
"cls_name": "Executor",
"config": {
"size": 5
}
}],
"res2res": [],
"res2dict": [],
"judgement": [],
"store_criterion": [],
"dump": [],
"did_challenge": [],
"final": []
}
}
assert actual.to_dict() == expected
def _decide_encoding(
cls, res: Any, default_encoding: TOption[str] = TOption(None)) -> str:
content_type = res.headers.get('content-type')
# XXX: See 2.2 in https://tools.ietf.org/html/rfc2616#section-2.2
if res.encoding and not ('text' in content_type
and res.encoding == 'ISO-8859-1'):
return res.encoding
meta_encodings: List[str] = deprecated.get_encodings_from_content(
res.content)
return meta_encodings[0] if meta_encodings else default_encoding.get(
) or res.apparent_encoding
def test_from_requests_decide_encoding(self, title, headers, text, content,
encoding, apparent_encoding,
default_encoding, expected):
Requests = namedtuple(
'Requests',
('headers', 'text', 'content', 'encoding', 'apparent_encoding'))
actual = Response._decide_encoding(
Requests(headers, text, content, encoding, apparent_encoding),
TOption(default_encoding))
assert actual == expected
def test_no_base(self, config_minimum):
actual: Config = configmaker.create_config(TList([config_minimum]),
TOption(None))
expected = {
"one": {
"name": "name_one",
"host": "http://host/one",
"proxy": "http://proxy"
},
"other": {
"name": "name_other",
"host": "http://host/other"
},
"output": {
"encoding": "utf8",
"response_dir": "tmpdir"
},
"threads": 1,
"max_retries": 3,
"addons": {
"log2reqs": {
"name": "addons.log2reqs.csv",
"cls_name": "Executor",
"config": {
"encoding": "utf8"
}
},
"reqs2reqs": [],
"res2res": [],
"res2dict": [],
"judgement": [],
"store_criterion": [{
"name": "addons.store_criterion.general",
"cls_name": "Executor",
"config": {
"statuses": ["different"]
}
}],
"dump": [],
"did_challenge": [],
"final": []
}
}
assert actual.to_dict() == expected
def apply_first_condition(request: Request,
conditions: TList[Condition]) -> Request:
# TODO: remove TOption (owlmixin... find)
condition: TOption[Condition] = TOption(
conditions.find(
lambda c: when_optional_filter(c.when, request.to_dict())))
if condition.is_none():
return request
name: TOption[str] = jinja2_format(condition.get().name, request.to_dict()) \
if when_optional_filter(condition.get().when, request.to_dict()) \
else request.name
return Request.from_dict({
'name': name,
'path': request.path,
'qs': request.qs,
'headers': request.headers,
})
def find_violators(args: Args, config: Config) -> TOption[TList[Violator]]:
global groups_by_member
api = ApiClient(config.base_url, os.environ['USER'],
os.environ['PASSWORD'])
groups_by_member = api.fetch_group() \
.flat_map(lambda g: api.fetch_members(g).map(lambda m: {'member': m, 'group': g})) \
.group_by(_['member'].username) \
.map_values(lambda ds: ds.map(_['group'].name))
violators: TList[Violator] = api.fetch_spaces() \
.map(_.key) \
.map(lambda k: {'key': k, 'permissions': api.fetch_space_permissions(k)}) \
.group_by(_['key']) \
.map_values(grouping_by_names) \
.map(lambda key, violator: find_violator_by_space(key, violator, config.deny)) \
.filter(lambda x: x.group_names or x.user_names or x.anonymous)
return TOption(violators or None)
def test(self, config_only_access_points, config_without_access_points):
actual: Config = configmaker.create_config(
TList([config_only_access_points, config_without_access_points]),
TOption(None))
expected = {
"one": {
"name": "name_one",
"host": "http://host/one",
"proxy": "http://proxy"
},
"other": {
"name": "name_other",
"host": "http://host/other"
},
"output": {
"encoding": "utf8",
"response_dir": "tmpdir"
},
"threads": 3,
"max_retries": 2,
"addons": {
"log2reqs": {
"name": "addons.log2reqs.csv",
"cls_name": "Executor",
"config": {
"encoding": "utf8"
}
},
"reqs2reqs": [],
"res2res": [],
"res2dict": [],
"judgement": [],
"store_criterion": [],
"dump": [],
"did_challenge": [],
"final": []
}
}
assert actual.to_dict() == expected
def apply_first_condition(res: Response, req: Request,
conditions: TList[Condition]) -> Response:
# TODO: remove TOption (owlmixin... find)
condition: TOption[Condition] = TOption(
conditions.find(lambda c: when_optional_filter(c.when, {
'req': req.to_dict(),
'res': res.to_dict()
})))
if condition.is_none():
return res
return Response.from_dict(
{
"body": res.body,
"type": condition.get().type,
"encoding": res.encoding.get(),
"headers": res.headers,
"url": res.url,
"status_code": res.status_code,
"elapsed": res.elapsed,
"elapsed_sec": res.elapsed_sec,
}, )
def from_requests(
cls, res: Any,
default_encoding: TOption[str] = TOption(None)) -> 'Response':
encoding: str = cls._decide_encoding(res, default_encoding)
type: str = cls._to_type(res)
return Response.from_dict({
'body':
res.content,
'encoding':
encoding,
'headers':
res.headers,
'url':
res.url,
'status_code':
res.status_code,
'elapsed':
res.elapsed,
'elapsed_sec':
round(res.elapsed.seconds + res.elapsed.microseconds / 1000000, 2),
'type':
type,
})
def test_from_host_normal(self):
actual = Proxy.from_host(TOption("proxy.net"))
assert actual.http == "http://proxy.net"
assert actual.https == "https://proxy.net"
def get_jinja2_format_error(fmt: str) -> TOption[str]:
try:
ENV.from_string(fmt)
return TOption(None)
except TemplateSyntaxError as err:
return TOption(err.message)
def content_type(self) -> TOption[str]:
return TOption(self.headers.get('content-type'))
def test_from_host_none(self):
actual = Proxy.from_host(TOption(None))
assert actual is None
def send_for_test(message: str, notifier: Notifier) -> TOption[str]:
return TOption(message)