def test_level_filter(self): mock_record = MockURLTableRecord() mock_record.level = 4 url_filter = LevelFilter(0) self.assertTrue(url_filter.test(None, mock_record)) url_filter = LevelFilter(5) mock_record.level = 5 self.assertTrue(url_filter.test(None, mock_record)) mock_record.level = 6 self.assertFalse(url_filter.test(None, mock_record)) mock_record.inline = True self.assertTrue(url_filter.test(None, mock_record))
def _build_url_filters(self): '''Create the URL filter instances. Returns: A list of URL filter instances ''' args = self._args filters = [ HTTPSOnlyFilter() if args.https_only else HTTPFilter(), RecursiveFilter(enabled=args.recursive, page_requisites=args.page_requisites), SpanHostsFilter( self._url_infos, enabled=args.span_hosts, page_requisites='page-requisites' in args.span_hosts_allow, linked_pages='linked-pages' in args.span_hosts_allow, ), ] if args.no_parent: filters.append(ParentFilter()) if args.domains or args.exclude_domains: filters.append( BackwardDomainFilter(args.domains, args.exclude_domains)) if args.hostnames or args.exclude_hostnames: filters.append( HostnameFilter(args.hostnames, args.exclude_hostnames)) if args.tries: filters.append(TriesFilter(args.tries)) if args.level and args.recursive: filters.append(LevelFilter(args.level)) if args.accept_regex or args.reject_regex: filters.append(RegexFilter(args.accept_regex, args.reject_regex)) if args.include_directories or args.exclude_directories: filters.append( DirectoryFilter(args.include_directories, args.exclude_directories)) if args.accept or args.reject: filters.append(BackwardFilenameFilter(args.accept, args.reject)) return filters
def _build_url_filters(cls, session: AppSession): '''Create the URL filter instances. Returns: A list of URL filter instances ''' args = session.args filters = [ HTTPSOnlyFilter() if args.https_only else SchemeFilter(), RecursiveFilter(enabled=args.recursive, page_requisites=args.page_requisites), FollowFTPFilter(follow=args.follow_ftp), ] if args.no_parent: filters.append(ParentFilter()) if args.domains or args.exclude_domains: filters.append( BackwardDomainFilter(args.domains, args.exclude_domains)) if args.hostnames or args.exclude_hostnames: filters.append( HostnameFilter(args.hostnames, args.exclude_hostnames)) if args.tries: filters.append(TriesFilter(args.tries)) if args.level and args.recursive or args.page_requisites_level: filters.append( LevelFilter(args.level, inline_max_depth=args.page_requisites_level)) if args.accept_regex or args.reject_regex: filters.append(RegexFilter(args.accept_regex, args.reject_regex)) if args.include_directories or args.exclude_directories: filters.append( DirectoryFilter(args.include_directories, args.exclude_directories)) if args.accept or args.reject: filters.append(BackwardFilenameFilter(args.accept, args.reject)) return filters
def test_level_filter(self): mock_record = MockURLTableRecord() mock_record.level = 4 url_filter = LevelFilter(0) self.assertTrue(url_filter.test(None, mock_record)) url_filter = LevelFilter(5) mock_record.level = 5 self.assertTrue(url_filter.test(None, mock_record)) mock_record.level = 6 self.assertFalse(url_filter.test(None, mock_record)) url_filter = LevelFilter(5) mock_record.inline = 1 mock_record.level = 5 self.assertTrue(url_filter.test(None, mock_record)) mock_record.level = 6 self.assertTrue(url_filter.test(None, mock_record)) mock_record.level = 7 self.assertTrue(url_filter.test(None, mock_record)) mock_record.level = 8 self.assertFalse(url_filter.test(None, mock_record)) url_filter = LevelFilter(0) mock_record.inline = 1 self.assertTrue(url_filter.test(None, mock_record)) mock_record.inline = 2 self.assertTrue(url_filter.test(None, mock_record)) mock_record.inline = 3 self.assertTrue(url_filter.test(None, mock_record)) mock_record.inline = 4 self.assertTrue(url_filter.test(None, mock_record)) mock_record.inline = 5 self.assertTrue(url_filter.test(None, mock_record)) mock_record.inline = 6 self.assertFalse(url_filter.test(None, mock_record)) mock_record.level = 1 url_filter = LevelFilter(0, inline_max_depth=0) mock_record.inline = 1000 self.assertTrue(url_filter.test(None, mock_record)) url_filter = LevelFilter(5, inline_max_depth=1) mock_record.inline = 1 self.assertTrue(url_filter.test(None, mock_record)) mock_record.inline = 2 self.assertFalse(url_filter.test(None, mock_record))
def test_level_filter(self): record = URLRecord() record.level = 4 url_filter = LevelFilter(0) self.assertTrue(url_filter.test(None, record)) url_filter = LevelFilter(5) record.level = 5 self.assertTrue(url_filter.test(None, record)) record.level = 6 self.assertFalse(url_filter.test(None, record)) url_filter = LevelFilter(5) record.inline_level = 1 record.level = 5 self.assertTrue(url_filter.test(None, record)) record.level = 6 self.assertTrue(url_filter.test(None, record)) record.level = 7 self.assertTrue(url_filter.test(None, record)) record.level = 8 self.assertFalse(url_filter.test(None, record)) url_filter = LevelFilter(0) record.inline_level = 1 self.assertTrue(url_filter.test(None, record)) record.inline_level = 2 self.assertTrue(url_filter.test(None, record)) record.inline_level = 3 self.assertTrue(url_filter.test(None, record)) record.inline_level = 4 self.assertTrue(url_filter.test(None, record)) record.inline_level = 5 self.assertTrue(url_filter.test(None, record)) record.inline_level = 6 self.assertFalse(url_filter.test(None, record)) record.level = 1 url_filter = LevelFilter(0, inline_max_depth=0) record.inline_level = 1000 self.assertTrue(url_filter.test(None, record)) url_filter = LevelFilter(5, inline_max_depth=1) record.inline_level = 1 self.assertTrue(url_filter.test(None, record)) record.inline_level = 2 self.assertFalse(url_filter.test(None, record))