def process_request(self, request, spider):
save_path = ToCrawlUrl.settings.get(
"ROOT_PATH_LOG") + ctime.get_today() + "_to_crawl.log"
log_file(save_path,
str(ctime.get_now_time()) + "\t" + spider.name + "\t" +
request.url + "\n",
method="a")
def process_request(self, request, spider):
save_path = self.settings.get(
"ROOT_PATH_LOG") + Dt.get_today() + "_to_crawl.log"
# TODO:
logging.info(save_path,
str(Dt.get_now_time()) + "\t" + spider.name + "\t" +
request.url + "\n",
method="a")
def process_response(self, request, response, spider):
save_path = CrawledUrl.settings.get(
"ROOT_PATH_LOG") + ctime.get_today() + "_carwled.log"
log_file(save_path,
str(ctime.get_now_time()) + "\t" + spider.name + "\t" +
str(response.status) + "\t" + request.url + "\n",
method="a")
return response
def process_response(self, request, response, spider):
save_path = self.settings.get(
"ROOT_PATH_LOG") + Dt.get_today() + "_carwled.log"
# TODO:
logging.info(save_path,
str(Dt.get_now_time()) + "\t" + spider.name + "\t" +
str(response.status) + "\t" + request.url + "\n",
method="a")
return response
def print_from_head(s, **kw):
print_no_end("\r" + Dt.get_now_time() + "\t" + str(s), **kw)