def do_work(argv):
url = argv.get('url')
workmanager = argv.get('workmanager')
max_size = argv.get('max_size')
fetcher = Fetch(url)
for resource in fetcher.get_all_resource():
if len(resourcelist) > max_size:
break
if resource not in resourcelist:
resourcelist.append(resource)
logger.get_logger.info(resource)
for href in fetcher.get_all_link():
if len(resourcelist) > max_size:
break
if href not in urllist:
urllist.append(href)
workmanager.add_job(do_work,
workmanager=workmanager,
url=href,
max_size=max_size)
