def main(): configure = get_args_read() if not 'host' in configure: printError('There should be "host" field in your map') sys.exit(1) if not 'path' in configure: printError('There should be "path" field in your map') sys.exit(1) if 'max_parallel_pages' in configure: MAX_PARALLEL_PAGES = configure['max_parallel_pages'] else: MAX_PARALLEL_PAGES = 1 if 'page_limit' in configure: page_limit_count = configure['page_limit'] else: page_limit_count = 0 cj = cookielib.MozillaCookieJar() cj.load('./cookies.txt') HttpFetchProcess.start() main_dict = configure main_request = PageDelegate.HttpRequest(configure['host'], configure['path'], jar = cj) main_page_request = Page.PageRequest(main_request, main_dict) new_page_requests = [] sleeper = SleepForClass() while True: for i in range(MAX_PARALLEL_PAGES): main_page_delegate = PageDelegate.PageDelegate() if not new_page_requests: if i == 0: page_request = main_page_request if page_limit_count != 0: page_request.set_limit(Page.PageLimit(page_limit_count)) sleeper.sleep_if_not_empty() Page.do_page(page_request, main_page_delegate, new_page_requests) sleeper.inc_count() break else: page_request = new_page_requests.pop(0) page_limit = page_request.get_limit() if page_limit: if page_limit.is_out(): new_page_requests = [] break page_limit.dec() Page.do_page(page_request, main_page_delegate, new_page_requests) sleeper.inc_count() while HttpFetchProcess.next(): pass printDebug('<!------------------------------ count = ' + str(sleeper.count_)) sleeper.check()