def test_level_filter(self):
mock_record = MockURLTableRecord()
mock_record.level = 4
url_filter = LevelFilter(0)
self.assertTrue(url_filter.test(None, mock_record))
url_filter = LevelFilter(5)
mock_record.level = 5
self.assertTrue(url_filter.test(None, mock_record))
mock_record.level = 6
self.assertFalse(url_filter.test(None, mock_record))
mock_record.inline = True
self.assertTrue(url_filter.test(None, mock_record))
def _build_url_filters(self):
'''Create the URL filter instances.
Returns:
A list of URL filter instances
'''
args = self._args
filters = [
HTTPSOnlyFilter() if args.https_only else HTTPFilter(),
RecursiveFilter(enabled=args.recursive,
page_requisites=args.page_requisites),
SpanHostsFilter(
self._url_infos,
enabled=args.span_hosts,
page_requisites='page-requisites' in args.span_hosts_allow,
linked_pages='linked-pages' in args.span_hosts_allow,
),
]
if args.no_parent:
filters.append(ParentFilter())
if args.domains or args.exclude_domains:
filters.append(
BackwardDomainFilter(args.domains, args.exclude_domains))
if args.hostnames or args.exclude_hostnames:
filters.append(
HostnameFilter(args.hostnames, args.exclude_hostnames))
if args.tries:
filters.append(TriesFilter(args.tries))
if args.level and args.recursive:
filters.append(LevelFilter(args.level))
if args.accept_regex or args.reject_regex:
filters.append(RegexFilter(args.accept_regex, args.reject_regex))
if args.include_directories or args.exclude_directories:
filters.append(
DirectoryFilter(args.include_directories,
args.exclude_directories))
if args.accept or args.reject:
filters.append(BackwardFilenameFilter(args.accept, args.reject))
return filters
def _build_url_filters(cls, session: AppSession):
'''Create the URL filter instances.
Returns:
A list of URL filter instances
'''
args = session.args
filters = [
HTTPSOnlyFilter() if args.https_only else SchemeFilter(),
RecursiveFilter(enabled=args.recursive,
page_requisites=args.page_requisites),
FollowFTPFilter(follow=args.follow_ftp),
]
if args.no_parent:
filters.append(ParentFilter())
if args.domains or args.exclude_domains:
filters.append(
BackwardDomainFilter(args.domains, args.exclude_domains))
if args.hostnames or args.exclude_hostnames:
filters.append(
HostnameFilter(args.hostnames, args.exclude_hostnames))
if args.tries:
filters.append(TriesFilter(args.tries))
if args.level and args.recursive or args.page_requisites_level:
filters.append(
LevelFilter(args.level,
inline_max_depth=args.page_requisites_level))
if args.accept_regex or args.reject_regex:
filters.append(RegexFilter(args.accept_regex, args.reject_regex))
if args.include_directories or args.exclude_directories:
filters.append(
DirectoryFilter(args.include_directories,
args.exclude_directories))
if args.accept or args.reject:
filters.append(BackwardFilenameFilter(args.accept, args.reject))
return filters
def test_level_filter(self):
mock_record = MockURLTableRecord()
mock_record.level = 4
url_filter = LevelFilter(0)
self.assertTrue(url_filter.test(None, mock_record))
url_filter = LevelFilter(5)
mock_record.level = 5
self.assertTrue(url_filter.test(None, mock_record))
mock_record.level = 6
self.assertFalse(url_filter.test(None, mock_record))
mock_record.inline = True
self.assertTrue(url_filter.test(None, mock_record))
def test_level_filter(self):
mock_record = MockURLTableRecord()
mock_record.level = 4
url_filter = LevelFilter(0)
self.assertTrue(url_filter.test(None, mock_record))
url_filter = LevelFilter(5)
mock_record.level = 5
self.assertTrue(url_filter.test(None, mock_record))
mock_record.level = 6
self.assertFalse(url_filter.test(None, mock_record))
url_filter = LevelFilter(5)
mock_record.inline = 1
mock_record.level = 5
self.assertTrue(url_filter.test(None, mock_record))
mock_record.level = 6
self.assertTrue(url_filter.test(None, mock_record))
mock_record.level = 7
self.assertTrue(url_filter.test(None, mock_record))
mock_record.level = 8
self.assertFalse(url_filter.test(None, mock_record))
url_filter = LevelFilter(0)
mock_record.inline = 1
self.assertTrue(url_filter.test(None, mock_record))
mock_record.inline = 2
self.assertTrue(url_filter.test(None, mock_record))
mock_record.inline = 3
self.assertTrue(url_filter.test(None, mock_record))
mock_record.inline = 4
self.assertTrue(url_filter.test(None, mock_record))
mock_record.inline = 5
self.assertTrue(url_filter.test(None, mock_record))
mock_record.inline = 6
self.assertFalse(url_filter.test(None, mock_record))
mock_record.level = 1
url_filter = LevelFilter(0, inline_max_depth=0)
mock_record.inline = 1000
self.assertTrue(url_filter.test(None, mock_record))
url_filter = LevelFilter(5, inline_max_depth=1)
mock_record.inline = 1
self.assertTrue(url_filter.test(None, mock_record))
mock_record.inline = 2
self.assertFalse(url_filter.test(None, mock_record))
def test_level_filter(self):
record = URLRecord()
record.level = 4
url_filter = LevelFilter(0)
self.assertTrue(url_filter.test(None, record))
url_filter = LevelFilter(5)
record.level = 5
self.assertTrue(url_filter.test(None, record))
record.level = 6
self.assertFalse(url_filter.test(None, record))
url_filter = LevelFilter(5)
record.inline_level = 1
record.level = 5
self.assertTrue(url_filter.test(None, record))
record.level = 6
self.assertTrue(url_filter.test(None, record))
record.level = 7
self.assertTrue(url_filter.test(None, record))
record.level = 8
self.assertFalse(url_filter.test(None, record))
url_filter = LevelFilter(0)
record.inline_level = 1
self.assertTrue(url_filter.test(None, record))
record.inline_level = 2
self.assertTrue(url_filter.test(None, record))
record.inline_level = 3
self.assertTrue(url_filter.test(None, record))
record.inline_level = 4
self.assertTrue(url_filter.test(None, record))
record.inline_level = 5
self.assertTrue(url_filter.test(None, record))
record.inline_level = 6
self.assertFalse(url_filter.test(None, record))
record.level = 1
url_filter = LevelFilter(0, inline_max_depth=0)
record.inline_level = 1000
self.assertTrue(url_filter.test(None, record))
url_filter = LevelFilter(5, inline_max_depth=1)
record.inline_level = 1
self.assertTrue(url_filter.test(None, record))
record.inline_level = 2
self.assertFalse(url_filter.test(None, record))
