def test_create_flag(self):
def test_help(_next_step, _expected):
self.assertEqual(len(_next_step.flags), len(_expected))
self.assertListEqual([flag.action for flag in _next_step.flags],
[flag['action'] for flag in _expected])
for flag, expected_flag in zip(_next_step.flags, _expected):
self.assertDictEqual(flag.as_json(), expected_flag)
self.assertEqual(flag.parent_name, 'name')
expected_ancestry = list(_next_step.ancestry)
expected_ancestry.append(flag.name)
self.assertListEqual(flag.ancestry, expected_ancestry)
next_step = NextStep(name='name')
next_step.create_flag('1')
expected = [Flag(action='1').as_json()]
test_help(next_step, expected)
filters = [Filter(action='test_filter_action'), Filter()]
next_step.create_flag('2', filters=filters)
expected.append(Flag(action='2', filters=filters).as_json())
test_help(next_step, expected)
args = {'arg1': 'a', 'arg2': 3, 'arg3': u'abc'}
args = {
arg_name: Argument(key=arg_name,
value=arg_value,
format=type(arg_value).__name__)
for arg_name, arg_value in args.items()
}
next_step.create_flag('3', filters=filters, args=args)
expected.append(Flag(action='3', filters=filters, args=args).as_json())
test_help(next_step, expected)
def test_to_from_xml_is_same_with_args_and_filters(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5.4'}),
Filter(action='Top Filter')
]
self.__assert_xml_is_convertible(
Flag(action='mod1_flag2', args={'arg1': '5'}, filters=filters))
def test_get_children(self):
next_step1 = NextStep()
names = ['', 'action1', 'action2']
for name in names:
self.assertIsNone(next_step1.get_children([name]))
self.assertDictEqual(next_step1.get_children([]),
next_step1.as_json(with_children=False))
flags = [Flag(), Flag(action='action1'), Flag(action='action2')]
next_step2 = NextStep(flags=flags)
for i, name in enumerate(names):
self.assertDictEqual(next_step2.get_children([name]),
flags[i].as_json())
self.assertDictEqual(next_step2.get_children([]),
next_step2.as_json(with_children=False))
filters = [
Filter(action='filter1'),
Filter(),
Filter(action='filter2')
]
flags = [Flag(action='action1', filters=filters)]
names = ['filter1', '', 'filter2']
next_step3 = NextStep(flags=flags)
ancestries = [[name, 'action1'] for name in names]
for i, ancestry in enumerate(ancestries):
self.assertDictEqual(next_step3.get_children(ancestry),
filters[i].as_json())
self.assertDictEqual(next_step3.get_children([]),
next_step3.as_json(with_children=False))
def test_to_from_xml(self):
filters = [Filter(action='test_filter_action'), Filter()]
args = {'arg1': 'a', 'arg2': 3, 'arg3': u'abc'}
args = {
arg_name: Argument(key=arg_name,
value=arg_value,
format=type(arg_value).__name__)
for arg_name, arg_value in args.items()
}
flags = [
Flag(),
Flag(parent_name='test_parent'),
Flag(action='test_action'),
Flag(parent_name='test_parent',
action='test_action',
ancestry=['a', 'b']),
Flag(action='test_action', filters=filters),
Flag(action='test_action', args=args),
Flag(parent_name='test_parent',
action='test_action',
ancestry=['a', 'b'],
filters=filters,
args=args)
]
for flag in flags:
original_flag = flag.as_json()
derived_flag = Flag(xml=flag.to_xml()).as_json()
self.assertDictEqual(derived_flag, original_flag)
def url_extrator(self,response): # 从href提取URL队列并加入到target_url中
try:
iter_url = regex.URL_REGEX.finditer(response) # href="/b/a"
filter_data = Filter(iter_url,"url",self.requests_seen) # 初始化过滤器
target_queue = filter_data.extractor(self.logger_type,self.target) # 提取到除特殊文件之外的URL队列
self.target_url.put(target_queue)
except Exception:
pass
def test_get_children_with_filters_no_ancestry(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5.4'}),
Filter(action='Top Filter')
]
flag = Flag(action='Top Flag', filters=filters)
self.assertDictEqual(flag.get_children([]),
flag.as_json(with_children=False))
def test_get_children_with_filters_invalid_ancestry(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5.4'}),
Filter(action='Top Filter')
]
flag = Flag(action='Top Flag', filters=filters)
for ancestry in [['a'], ['a', 'b']]:
self.assertIsNone(flag.get_children(ancestry))
def test_get_children_with_filters_ancestry_too_deep(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5.4'}),
Filter(action='Top Filter')
]
flag = Flag(action='Top Flag', filters=filters)
for name in ['mod1_filter2', 'Top Filter']:
self.assertIsNone(flag.get_children([name, 'too deep']))
def test_init_with_filters(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5.4'}),
Filter(action='Top Filter')
]
flag = Flag(action='Top Flag', filters=filters)
self.__compare_init(flag, 'Top Flag', '', ['', 'Top Flag'], filters,
{})
def url_extrator(self,response):
try:
iter_url = regex.URL_REGEX.finditer(response)
filter_data = Filter(iter_url,"url",self.requests_seen)
target_queue = filter_data.extractor(self.logger_type,self.target)
self.target_url.put(target_queue)
except Exception:
pass
def test_to_xml_with_args_with_routing(self):
xml = Filter(action='mod1_filter2', args={'arg1': '@step1'}).to_xml()
self.assertEqual(xml.tag, 'filter')
self.assertEqual(xml.get('action'), 'mod1_filter2')
arg_xml = xml.findall('args/*')
self.assertEqual(len(arg_xml), 1)
self.assertEqual(arg_xml[0].tag, 'arg1')
self.assertEqual(arg_xml[0].text, '@step1')
def add_filter(self, action='', args=None, index=None):
if index is not None:
self.filters.insert(
index,
Filter(action=action, args=(args if args is not None else {})))
else:
self.filters.append(
Filter(action=action, args=(args if args is not None else {})))
return True
def test_call_action_with_valid_args_and_filters_valid_data(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5'}),
Filter(action='Top Filter')
]
# should go <input = 1> -> <mod1_filter2 = 5+1 = 6> -> <Top Filter 6=6> -> <mod1_flag2 4+6%2==0> -> True
self.assertTrue(
Flag(action='mod1_flag2', args={'arg1': 4}, filters=filters)('1',
{}))
def test_call_action_with_valid_args_and_filters_invalid_data(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5'}),
Filter(action='Top Filter')
]
# should go <input = invalid> -> <mod1_filter2 with error = invalid> -> <Top Filter with error = invalid>
# -> <mod1_flag2 4+invalid throws error> -> False
self.assertFalse(
Flag(action='mod1_flag2', args={'arg1': 4},
filters=filters)('invalid', {}))
def test_get_children_with_filters_valid_ancestry(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5.4'}),
Filter(action='Top Filter')
]
flag = Flag(action='Top Flag', filters=filters)
self.assertDictEqual(flag.get_children(['mod1_filter2']),
filters[0].as_json())
self.assertDictEqual(flag.get_children(['Top Filter']),
filters[1].as_json())
def filters(self, filters):
try:
temp = Filter(self.domain)
temp.stages = filters['stages']
temp.platforms = filters['platforms']
self.__filters = temp
except KeyError as e:
handler(
type(self).__name__, "Unable to properly extract "
"information from filter: {}.".format(e))
raise BadInput
def test_call_action_with_valid_args_and_filters_invalid_data_and_routing(
self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '@step1'}),
Filter(action='Top Filter')
]
# should go <input = invalid> -> <mod1_filter2 with error = invalid> -> <Top Filter with error = invalid>
# -> <mod1_flag2 4+invalid throws error> -> False
accumulator = {'step1': '5', 'step2': 4}
self.assertFalse(
Flag(action='mod1_flag2', args={'arg1': 4},
filters=filters)('invalid', accumulator))
def test_to_from_xml(self):
args = {'arg1': 'a', 'arg2': 3, 'arg3': u'abc'}
input_output = [
Filter(),
Filter(action='test_action'),
Filter(ancestry=['a', 'b'], action="test", args=args)
]
for filter_element in input_output:
original_json = filter_element.as_json()
derived_json = Filter(xml=filter_element.to_xml()).as_json()
self.assertEqual(original_json, derived_json)
def test_validate_invalid_name(self):
filter_elem = Filter(action='junkName')
self.assertFalse(filter_elem.validate_args())
self.test_funcs = {
'filters': {
'func_name1': {
'args': []
},
'func_name2': {
'args': [{
'name': 'arg_name1',
'type': 'str'
}]
},
'func_name3': {
'args': [{
'name': 'arg_name1',
'type': 'str'
}, {
'name': 'arg_name2',
'type': 'int'
}]
}
}
}
corresponding_args = {
'func_name1': {},
'func_name2': {
'arg_name1': 'test_string'
},
'func_name3': {
'arg_name1': 'test_string',
'arg_name2': 6
}
}
actions = ['func_name1', 'func_name2', 'func_name3', 'invalid_name']
for action in actions:
for arg_action, args in corresponding_args.items():
filter_elem = Filter(action=action, args=args)
if action == 'invalid_name':
self.assertFalse(filter_elem.validate_args())
elif not self.test_funcs['filters'][action]['args']:
self.assertTrue(filter_elem.validate_args())
elif action == arg_action:
if len(list(args.keys())) == len(
list(self.test_funcs['filters'][action]['args'])):
self.assertTrue(filter_elem.validate_args())
else:
self.assertFalse(filter_elem.validate_args())
else:
self.assertFalse(filter_elem.validate_args())
def filters(self, filters):
temp = Filter(self.domain)
try:
loadChecker(type(self).__name__, filters, ['platforms'], "filters")
# force upgrade to v4
if 'stages' in filters:
print('[Filters] - V3 Field "stages" detected. Upgrading Filters object to V4.')
temp.platforms = filters['platforms']
self.__filters = temp
except MissingParameters as e:
handler(type(self).__name__, 'Filters {} is missing parameters: '
'{}. Skipping.'
.format(filters, e))
def test_as_json_with_args_with_routing(self):
filter_elem = Filter(action='mod1_filter2', args={'arg1': '@step1'})
expected = {
'action': 'mod1_filter2',
'args': {
'arg1': {
'format': 'number',
'key': 'arg1',
'value': '@step1'
}
}
}
self.assertDictEqual(filter_elem.as_json(), expected)
def test_name_parent_multiple_filter_rename(self):
flag = Flag(ancestry=['flag_parent'], action='test_flag')
filter_one = Filter(action="test_filter_one", ancestry=flag.ancestry)
filter_two = Filter(action="test_filter_two", ancestry=flag.ancestry)
flag.filters = [filter_one, filter_two]
new_ancestry = ["flag_parent_update"]
flag.reconstruct_ancestry(new_ancestry)
new_ancestry.append("test_flag")
new_ancestry.append("test_filter_one")
self.assertListEqual(new_ancestry, flag.filters[0].ancestry)
new_ancestry.remove("test_filter_one")
new_ancestry.append("test_filter_two")
self.assertListEqual(new_ancestry, flag.filters[1].ancestry)
def test_length_filter(self):
filter = Filter(action='length')
self.assertIsNone(filter())
self.assertEqual(filter(output=6), 6)
self.assertEqual(filter(output=5.5), None)
self.assertEqual(filter(output=[3, 4, 5]), 3)
self.assertEqual(filter(output='aaab'), 4)
def test_reconstruct_ancestry_with_filters(self):
filters = [
Filter(action='mod1_filter2', args={'arg1': '5.4'}),
Filter(action='Top Filter')
]
flag = Flag(action='Top Flag',
filters=filters,
ancestry=['flag_parent'])
new_ancestry = ['new_parent']
flag.reconstruct_ancestry(new_ancestry)
new_ancestry.append('Top Flag')
self.assertListEqual(flag.ancestry, new_ancestry)
for filter_element in filters:
ancestry = list(new_ancestry)
ancestry.append(filter_element.action)
self.assertEqual(filter_element.ancestry, ancestry)
def generator_proxies(self):
for name in self.list_name:
if name in self.dic:
url,params = chambering(origin_proxies[name],strike=False)
result = requester(url,params,GET=True,timeout=None)
response = regex.Espace_eliminate.sub("",result.text)
ips, ports, types = self.dic[name]['ip'].finditer(response),\
self.dic[name]['port'].finditer(response),\
self.dic[name]['type'].finditer(response)
for i, j, k in zip(ips,ports,types):
ip = self.dic[name]['sub'].sub(" ", i.group())
port = self.dic[name]['sub'].sub(" ", j.group())
type = self.dic[name]['sub'].sub(" ", k.group())
# self.count = self.count+1
# print((ip, port, type))
if Filter.filter(ip,self.filter_proxy):
proxy = eval(regex.Espace_eliminate.sub("", str((ip, port, type.lower()))))
self.logger.info(f"ip : {proxy[0]} port : {proxy[1]} type : {proxy[2]}")
self.container.put(proxy)
def generator_proxies(self):
''' 从默认代理池(list_name中定义的)获取可用代理 '''
for name in self.list_name:
if name in self.dic:
url, params = chambering(origin_proxies[name], strike=False)
result = requester(url, params, GET=True, timeout=None)
if result == None:
continue
response = regex.Espace_eliminate.sub("", result.text)
ips, ports, types = self.dic[name]['ip'].finditer(response),\
self.dic[name]['port'].finditer(response),\
self.dic[name]['type'].finditer(response)
for i, j, k in zip(ips, ports, types):
ip = self.dic[name]['sub'].sub(" ", i.group())
port = self.dic[name]['sub'].sub(" ", j.group())
type = self.dic[name]['sub'].sub(" ", k.group())
# self.count = self.count+1
# print((ip, port, type))
if Filter.filter(ip,
self.filter_proxy): # 不是特殊文件链接 就加入到代理集合中
proxy = eval(
regex.Espace_eliminate.sub(
"", str((ip, port, type.lower()))))
# temp=r.replace('(','').replace(')','') # 字符串转元组安全方法
# a=tuple([int(i) for i in temp.split(',')])
self.logger.info(
f"ip : {proxy[0]} port : {proxy[1]} type : {proxy[2]}"
)
self.container.put(proxy)
def test_call_with_complex_args(self):
original_filter = Filter(action='sub1_filter1',
args={'arg1': {
'a': '5.4',
'b': 'string_in'
}})
self.assertEqual(original_filter(3, {}), '3.0 5.4 string_in')
def __execute_trigger(conditional, data_in):
filters = [Filter(action=filter_element['action'],
args=Triggers.__to_new_input_format(filter_element['args']))
for filter_element in conditional['filters']]
return Flag(action=conditional['flag'],
args=Triggers.__to_new_input_format(conditional['args']),
filters=filters)(data_in, {})
def test_to_from_json(self):
filter_params = ['test_filter_action', '']
flags_params = [('', []), ('test_action', []),
('test_action', filter_params)]
input_params = [('', '', None, []), ('test_name', '', None, []),
('test_name', 'test_parent', None, []),
('test_name', 'test_parent', ['a', 'b'], []),
('test_name', 'test_parent', ['a', 'b'], flags_params)]
for (name, parent_name, ancestry, flag_params) in input_params:
next_step = NextStep(name=name,
parent_name=parent_name,
ancestry=ancestry)
if flag_params:
flags = []
for flag_action, flag_filter_params in flag_params:
flag = Flag(action=flag_action,
parent_name=next_step.name,
ancestry=next_step.ancestry)
if filter_params:
flag.filters = [
Filter(action=flag_action,
parent_name=flag.name,
ancestry=flag.ancestry)
for flag_action in flag_filter_params
]
flags.append(flag)
next_step.flags = flags
next_step_json = next_step.as_json()
derived_next_step = NextStep.from_json(next_step_json,
parent_name=parent_name,
ancestry=ancestry)
self.assertDictEqual(derived_next_step.as_json(), next_step_json)
self.assertEqual(next_step.parent_name,
derived_next_step.parent_name)
self.assertListEqual(next_step.ancestry,
derived_next_step.ancestry)
derived_json_without_children = next_step_json
derived_json_without_children['flags'] = [
flag['action']
for flag in derived_json_without_children['flags']
]
self.assertDictEqual(
derived_next_step.as_json(with_children=False),
derived_json_without_children)
# check the ancestry of the flags
original_flag_ancestries = [
list(flag.ancestry) for flag in next_step.flags
]
derived_flag_ancestries = [
list(flag.ancestry) for flag in derived_next_step.flags
]
self.assertEqual(len(original_flag_ancestries),
len(derived_flag_ancestries))
for original_flag_ancestry, derived_flag_ancestry in zip(
original_flag_ancestries, derived_flag_ancestries):
self.assertListEqual(derived_flag_ancestry,
original_flag_ancestry)
def from_json(json, parent_name='', ancestry=None):
"""Forms a Flag object from the provided JSON object.
Args:
json (JSON object): The JSON object to convert from.
parent_name (str, optional): The name of the parent for ancestry purposes. Defaults to an empty string.
ancestry (list[str], optional): The ancestry for the new Flag object. Defaults to None.
Returns:
The Flag object parsed from the JSON object.
"""
args = {
arg_name: arguments.Argument.from_json(arg_json)
for arg_name, arg_json in json['args'].items()
}
flag = Flag(action=json['action'],
args=args,
parent_name=parent_name,
ancestry=ancestry)
filters = [
Filter.from_json(filter_element,
parent_name=flag.name,
ancestry=flag.ancestry)
for filter_element in json['filters']
]
flag.filters = filters
return flag
