def parse_detail(self, response): print("market_current_detail_page:" + response.url) trs = response.css("tbody tr") date_time_str = "2018-" + zhesu.date_time_convert( response.css("#title::text").extract_first("")) date_time = int(time.mktime(time.strptime(date_time_str, '%Y-%m-%d'))) for tr in trs[1:]: tds = tr.css("td") market_item = MarketPriceItem() market_item["breed"] = self.breed market_item["spec"] = tds[0].css("td::text").extract_first( "").strip() market_item["brand"] = tds[1].css("td::text").extract_first( "").strip() market_item["area"] = tds[2].css("td::text").extract_first( "").strip() market_item["price"] = tds[3].css("td::text").extract_first( "").strip() market_item["updown"] = zhesu.market_updown_convert( tds[4].css("td::text").extract_first("").strip()) market_item["release_date"] = date_time * 1000 market_item["release_date_str"] = date_time_str if market_item["spec"] == None or market_item[ "brand"] == None or market_item[ "spec"] == "" or market_item["brand"] == "": continue yield market_item
def parse(self, response): """ """ post_nodes = response.css(".catlist_li") cur_date = time.strftime('%Y-%m-%d', time.localtime(time.time())); for post_node in post_nodes: post_url = post_node.css("a::attr(href)").extract_first("") post_date = "2018-" + zhesu.date_time_convert(post_node.css("a::attr(title)").extract_first("")) if post_date == cur_date: yield Request(url=post_url, callback = self.parse_detail)