def test_recursive_filter_requisites(self):
mock_record = MockURLTableRecord()
mock_record.level = 0
mock_record.inline = True
url_filter = RecursiveFilter(page_requisites=True)
self.assertTrue(url_filter.test(None, mock_record))
def test_recursive_filter_requisites(self):
mock_record = MockURLTableRecord()
mock_record.level = 0
mock_record.inline = 1
url_filter = RecursiveFilter(page_requisites=True)
self.assertTrue(url_filter.test(None, mock_record))
def test_recursive_filter_requisites(self):
record = URLRecord()
record.level = 0
record.inline_level = 1
url_filter = RecursiveFilter(page_requisites=True)
self.assertTrue(url_filter.test(None, record))
def test_recursive_filter_on(self):
mock_record = MockURLTableRecord()
mock_record.level = 0
url_filter = RecursiveFilter(enabled=True)
self.assertTrue(url_filter.test(None, mock_record))
mock_record.level = 1
self.assertTrue(url_filter.test(None, mock_record))
def test_recursive_filter_on(self):
mock_record = MockURLTableRecord()
mock_record.level = 0
url_filter = RecursiveFilter(enabled=True)
self.assertTrue(url_filter.test(None, mock_record))
mock_record.level = 1
self.assertTrue(url_filter.test(None, mock_record))
def test_recursive_filter_on(self):
record = URLRecord()
record.level = 0
url_filter = RecursiveFilter(enabled=True)
self.assertTrue(url_filter.test(None, record))
record.level = 1
self.assertTrue(url_filter.test(None, record))
def test_recursive_filter_off(self):
record = URLRecord()
record.level = 0
url_filter = RecursiveFilter()
self.assertTrue(url_filter.test(None, record))
record.level = 1
self.assertFalse(url_filter.test(None, record))
def _build_url_filters(self):
'''Create the URL filter instances.
Returns:
A list of URL filter instances
'''
args = self._args
filters = [
HTTPSOnlyFilter() if args.https_only else HTTPFilter(),
RecursiveFilter(enabled=args.recursive,
page_requisites=args.page_requisites),
SpanHostsFilter(
self._url_infos,
enabled=args.span_hosts,
page_requisites='page-requisites' in args.span_hosts_allow,
linked_pages='linked-pages' in args.span_hosts_allow,
),
]
if args.no_parent:
filters.append(ParentFilter())
if args.domains or args.exclude_domains:
filters.append(
BackwardDomainFilter(args.domains, args.exclude_domains))
if args.hostnames or args.exclude_hostnames:
filters.append(
HostnameFilter(args.hostnames, args.exclude_hostnames))
if args.tries:
filters.append(TriesFilter(args.tries))
if args.level and args.recursive:
filters.append(LevelFilter(args.level))
if args.accept_regex or args.reject_regex:
filters.append(RegexFilter(args.accept_regex, args.reject_regex))
if args.include_directories or args.exclude_directories:
filters.append(
DirectoryFilter(args.include_directories,
args.exclude_directories))
if args.accept or args.reject:
filters.append(BackwardFilenameFilter(args.accept, args.reject))
return filters
def _build_url_filters(cls, session: AppSession):
'''Create the URL filter instances.
Returns:
A list of URL filter instances
'''
args = session.args
filters = [
HTTPSOnlyFilter() if args.https_only else SchemeFilter(),
RecursiveFilter(enabled=args.recursive,
page_requisites=args.page_requisites),
FollowFTPFilter(follow=args.follow_ftp),
]
if args.no_parent:
filters.append(ParentFilter())
if args.domains or args.exclude_domains:
filters.append(
BackwardDomainFilter(args.domains, args.exclude_domains))
if args.hostnames or args.exclude_hostnames:
filters.append(
HostnameFilter(args.hostnames, args.exclude_hostnames))
if args.tries:
filters.append(TriesFilter(args.tries))
if args.level and args.recursive or args.page_requisites_level:
filters.append(
LevelFilter(args.level,
inline_max_depth=args.page_requisites_level))
if args.accept_regex or args.reject_regex:
filters.append(RegexFilter(args.accept_regex, args.reject_regex))
if args.include_directories or args.exclude_directories:
filters.append(
DirectoryFilter(args.include_directories,
args.exclude_directories))
if args.accept or args.reject:
filters.append(BackwardFilenameFilter(args.accept, args.reject))
return filters