def _task_handler(self, url): # 先拿网页源码,再保存,两个都是高阻塞的操作,交给线程处理 url_fetcher = URLFetcher(url) retry = 1 if url_fetcher.fetch(retry): self._save_task_results(url, url_fetcher) self._add_unvisited_hrefs(url_fetcher)
def _start_test_task(self, wpt_base_url, api_key, test_target): test_url = wpt_base_url test_url += "?url=" + quote(test_target) test_url += "&runs=1&fvonly=1&requests=1&f=json" test_url += "&k=" + api_key print test_url url_fetcher = URLFetcher(test_url) json_data_url = "" if url_fetcher.fetch(1): url, page_source = url_fetcher.get_data() response = json.loads(page_source) if response["statusCode"] == 200: json_data_url = response["data"]["jsonUrl"] print "josn_data_url : %s" % json_data_url return json_data_url
def _get_test_result(self, test_target, json_data_url, result_file_path): retry_count = 0 while retry_count <= 100: json_data_fetcher = URLFetcher(json_data_url) if json_data_fetcher.fetch(1): retry_count += 1 url, json_data = json_data_fetcher.get_data() response = json.loads(json_data) formatted_json_data = unicode(json.dumps(response, indent=4)) if response['statusCode'] == 200: self._save_test_result(test_target, formatted_json_data, result_file_path) print "Get test result of %s successfully!" % test_target return True else: print "statusCode: %s, statusText: %s"\ % (response['statusCode'], response['statusText']) time.sleep(5 * 6) return False