def process_request(self, request, spider): save_path = ToCrawlUrl.settings.get( "ROOT_PATH_LOG") + ctime.get_today() + "_to_crawl.log" log_file(save_path, str(ctime.get_now_time()) + "\t" + spider.name + "\t" + request.url + "\n", method="a")
def process_request(self, request, spider): save_path = self.settings.get( "ROOT_PATH_LOG") + Dt.get_today() + "_to_crawl.log" # TODO: logging.info(save_path, str(Dt.get_now_time()) + "\t" + spider.name + "\t" + request.url + "\n", method="a")
def process_response(self, request, response, spider): save_path = CrawledUrl.settings.get( "ROOT_PATH_LOG") + ctime.get_today() + "_carwled.log" log_file(save_path, str(ctime.get_now_time()) + "\t" + spider.name + "\t" + str(response.status) + "\t" + request.url + "\n", method="a") return response
def process_response(self, request, response, spider): save_path = self.settings.get( "ROOT_PATH_LOG") + Dt.get_today() + "_carwled.log" # TODO: logging.info(save_path, str(Dt.get_now_time()) + "\t" + spider.name + "\t" + str(response.status) + "\t" + request.url + "\n", method="a") return response
def print_from_head(s, **kw): print_no_end("\r" + Dt.get_now_time() + "\t" + str(s), **kw)