def main():
configure = get_args_read()
if not 'host' in configure:
printError('There should be "host" field in your map')
sys.exit(1)
if not 'path' in configure:
printError('There should be "path" field in your map')
sys.exit(1)
if 'max_parallel_pages' in configure:
MAX_PARALLEL_PAGES = configure['max_parallel_pages']
else:
MAX_PARALLEL_PAGES = 1
if 'page_limit' in configure:
page_limit_count = configure['page_limit']
else:
page_limit_count = 0
cj = cookielib.MozillaCookieJar()
cj.load('./cookies.txt')
HttpFetchProcess.start()
main_dict = configure
main_request = PageDelegate.HttpRequest(configure['host'], configure['path'], jar = cj)
main_page_request = Page.PageRequest(main_request, main_dict)
new_page_requests = []
sleeper = SleepForClass()
while True:
for i in range(MAX_PARALLEL_PAGES):
main_page_delegate = PageDelegate.PageDelegate()
if not new_page_requests:
if i == 0:
page_request = main_page_request
if page_limit_count != 0:
page_request.set_limit(Page.PageLimit(page_limit_count))
sleeper.sleep_if_not_empty()
Page.do_page(page_request, main_page_delegate, new_page_requests)
sleeper.inc_count()
break
else:
page_request = new_page_requests.pop(0)
page_limit = page_request.get_limit()
if page_limit:
if page_limit.is_out():
new_page_requests = []
break
page_limit.dec()
Page.do_page(page_request, main_page_delegate, new_page_requests)
sleeper.inc_count()
while HttpFetchProcess.next():
pass
printDebug('<!------------------------------ count = ' + str(sleeper.count_))
sleeper.check()
