def test_seeker(logger, seed_url, rules):
kwarg = {
"rules": rules,
"parent_node_type": "div",
"parent_node_attrs": {"class":"shop-hesper"}
}
bee.test_seeker(logger, "bee.RuleBasedSeeker", seed_url, **kwarg)
def test_seeker_1(logger):
kwarg = {
"rules": [
[ ".*/cat\d+\.html", 3, 60, "simple_http_get", ["simple_seek"], [], False ],
[ ".*prod\d+\.html", 1, 3600, "simple_http_get", [], ["simple_miner"], False ],
]
}
bee.test_seeker(logger, "bee.RuleBasedSeeker", "file://%s/demosite/index.html" % (os.path.abspath('.'),), **kwarg)
def test_seeker(logger, seed_url, rules):
kwarg = {
"rules": rules,
}
bee.test_seeker(logger, "bee.RuleBasedSeeker", seed_url, **kwarg)